diff --git a/gemma3_12b_1m/coin/training/dolci/training_trace.jsonl b/gemma3_12b_1m/coin/training/dolci/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..4877cf59b2a410220bdc440add90850e018714a4 --- /dev/null +++ b/gemma3_12b_1m/coin/training/dolci/training_trace.jsonl @@ -0,0 +1,48 @@ +{"epoch": 0.017877094972067038, "grad_norm": 2.671875, "learning_rate": 0.0, "loss": 0.76202392578125, "memory/device_reserved (GiB)": 59.93, "memory/max_active (GiB)": 50.98, "memory/max_allocated (GiB)": 50.98, "ppl": 2.14261, "step": 1, "tokens/total": 2097152, "tokens/train_per_sec_per_gpu": 200.61, "tokens/trainable": 1329662} +{"epoch": 0.035754189944134075, "grad_norm": 2.953125, "learning_rate": 1.0000000000000002e-06, "loss": 0.75018310546875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 2.11739, "step": 2, "tokens/total": 4194304, "tokens/train_per_sec_per_gpu": 206.95, "tokens/trainable": 2661007} +{"epoch": 0.053631284916201116, "grad_norm": 2.515625, "learning_rate": 2.0000000000000003e-06, "loss": 0.74755859375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 2.11184, "step": 3, "tokens/total": 6291456, "tokens/train_per_sec_per_gpu": 211.49, "tokens/trainable": 3991896} +{"epoch": 0.07150837988826815, "grad_norm": 2.1875, "learning_rate": 3e-06, "loss": 0.728912353515625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 2.07282, "step": 4, "tokens/total": 8388608, "tokens/train_per_sec_per_gpu": 220.21, "tokens/trainable": 5327798} +{"epoch": 0.0893854748603352, "grad_norm": 1.890625, "learning_rate": 4.000000000000001e-06, "loss": 0.70172119140625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 2.01722, "step": 5, "tokens/total": 10485760, "tokens/train_per_sec_per_gpu": 209.87, "tokens/trainable": 6648532} +{"epoch": 0.10726256983240223, "grad_norm": 1.328125, "learning_rate": 5e-06, "loss": 0.67510986328125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.96425, "step": 6, "tokens/total": 12582912, "tokens/train_per_sec_per_gpu": 226.37, "tokens/trainable": 7981707} +{"epoch": 0.12513966480446928, "grad_norm": 2.171875, "learning_rate": 6e-06, "loss": 0.663665771484375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.9419, "step": 7, "tokens/total": 14680064, "tokens/train_per_sec_per_gpu": 223.61, "tokens/trainable": 9295918} +{"epoch": 0.1430167597765363, "grad_norm": 1.90625, "learning_rate": 7e-06, "loss": 0.6937255859375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 2.00116, "step": 8, "tokens/total": 16777216, "tokens/train_per_sec_per_gpu": 224.7, "tokens/trainable": 10624450} +{"epoch": 0.16089385474860335, "grad_norm": 1.3046875, "learning_rate": 8.000000000000001e-06, "loss": 0.64068603515625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.89778, "step": 9, "tokens/total": 18874368, "tokens/train_per_sec_per_gpu": 200.52, "tokens/trainable": 11960744} +{"epoch": 0.1787709497206704, "grad_norm": 0.92578125, "learning_rate": 9e-06, "loss": 0.65338134765625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.92203, "step": 10, "tokens/total": 20971520, "tokens/train_per_sec_per_gpu": 213.04, "tokens/trainable": 13298263} +{"epoch": 0.19664804469273742, "grad_norm": 0.93359375, "learning_rate": 1e-05, "loss": 0.644683837890625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.90538, "step": 11, "tokens/total": 23068672, "tokens/train_per_sec_per_gpu": 213.47, "tokens/trainable": 14619571} +{"epoch": 0.21452513966480447, "grad_norm": 0.8359375, "learning_rate": 9.984630218530014e-06, "loss": 0.662994384765625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.94059, "step": 12, "tokens/total": 25165824, "tokens/train_per_sec_per_gpu": 211.07, "tokens/trainable": 15932764} +{"epoch": 0.2324022346368715, "grad_norm": 0.79296875, "learning_rate": 9.938625865312252e-06, "loss": 0.642730712890625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.90167, "step": 13, "tokens/total": 27262976, "tokens/train_per_sec_per_gpu": 220.8, "tokens/trainable": 17265676} +{"epoch": 0.25027932960893856, "grad_norm": 0.94921875, "learning_rate": 9.862301196726988e-06, "loss": 0.62359619140625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.86563, "step": 14, "tokens/total": 29360128, "tokens/train_per_sec_per_gpu": 204.86, "tokens/trainable": 18583414} +{"epoch": 0.2681564245810056, "grad_norm": 0.61328125, "learning_rate": 9.756177587652857e-06, "loss": 0.62322998046875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.86494, "step": 15, "tokens/total": 31457280, "tokens/train_per_sec_per_gpu": 213.94, "tokens/trainable": 19904256} +{"epoch": 0.2860335195530726, "grad_norm": 0.96484375, "learning_rate": 9.620979969947759e-06, "loss": 0.627685546875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.87327, "step": 16, "tokens/total": 33554432, "tokens/train_per_sec_per_gpu": 214.36, "tokens/trainable": 21227136} +{"epoch": 0.3039106145251397, "grad_norm": 0.54296875, "learning_rate": 9.4576318804292e-06, "loss": 0.605316162109375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.83183, "step": 17, "tokens/total": 35651584, "tokens/train_per_sec_per_gpu": 216.84, "tokens/trainable": 22561762} +{"epoch": 0.3217877094972067, "grad_norm": 0.5859375, "learning_rate": 9.267249152181379e-06, "loss": 0.6080322265625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.83681, "step": 18, "tokens/total": 37748736, "tokens/train_per_sec_per_gpu": 219.61, "tokens/trainable": 23891748} +{"epoch": 0.3396648044692737, "grad_norm": 0.46875, "learning_rate": 9.051132292283772e-06, "loss": 0.61212158203125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.84434, "step": 19, "tokens/total": 39845888, "tokens/train_per_sec_per_gpu": 221.67, "tokens/trainable": 25233682} +{"epoch": 0.3575418994413408, "grad_norm": 0.50390625, "learning_rate": 8.810757598029094e-06, "loss": 0.60693359375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.8348, "step": 20, "tokens/total": 41943040, "tokens/train_per_sec_per_gpu": 207.97, "tokens/trainable": 26553520} +{"epoch": 0.3754189944134078, "grad_norm": 0.5625, "learning_rate": 8.547767072315835e-06, "loss": 0.619354248046875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.85773, "step": 21, "tokens/total": 44040192, "tokens/train_per_sec_per_gpu": 223.54, "tokens/trainable": 27888194} +{"epoch": 0.39329608938547483, "grad_norm": 0.4609375, "learning_rate": 8.263957207103506e-06, "loss": 0.593994140625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.81121, "step": 22, "tokens/total": 46137344, "tokens/train_per_sec_per_gpu": 223.02, "tokens/trainable": 29212924} +{"epoch": 0.4111731843575419, "grad_norm": 0.46484375, "learning_rate": 7.961266711550922e-06, "loss": 0.60089111328125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.82374, "step": 23, "tokens/total": 48234496, "tokens/train_per_sec_per_gpu": 209.91, "tokens/trainable": 30546442} +{"epoch": 0.42905027932960893, "grad_norm": 0.447265625, "learning_rate": 7.641763268666832e-06, "loss": 0.59710693359375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.81685, "step": 24, "tokens/total": 50331648, "tokens/train_per_sec_per_gpu": 204.79, "tokens/trainable": 31861696} +{"epoch": 0.44692737430167595, "grad_norm": 0.396484375, "learning_rate": 7.307629410938364e-06, "loss": 0.622528076171875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.86363, "step": 25, "tokens/total": 52428800, "tokens/train_per_sec_per_gpu": 209.67, "tokens/trainable": 33181620} +{"epoch": 0.464804469273743, "grad_norm": 0.42578125, "learning_rate": 6.961147611421076e-06, "loss": 0.5909423828125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.80569, "step": 26, "tokens/total": 54525952, "tokens/train_per_sec_per_gpu": 218.38, "tokens/trainable": 34530376} +{"epoch": 0.48268156424581005, "grad_norm": 0.408203125, "learning_rate": 6.604684692133597e-06, "loss": 0.611572265625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.84333, "step": 27, "tokens/total": 56623104, "tokens/train_per_sec_per_gpu": 216.85, "tokens/trainable": 35855132} +{"epoch": 0.5005586592178771, "grad_norm": 0.400390625, "learning_rate": 6.240675656263303e-06, "loss": 0.582794189453125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.79104, "step": 28, "tokens/total": 58695680, "tokens/train_per_sec_per_gpu": 206.91, "tokens/trainable": 37174520} +{"epoch": 0.5184357541899441, "grad_norm": 0.474609375, "learning_rate": 5.871607054625497e-06, "loss": 0.606689453125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.83435, "step": 29, "tokens/total": 60792832, "tokens/train_per_sec_per_gpu": 207.34, "tokens/trainable": 38512212} +{"epoch": 0.5363128491620112, "grad_norm": 1.5703125, "learning_rate": 5.500000000000001e-06, "loss": 0.604949951171875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.83116, "step": 30, "tokens/total": 62889984, "tokens/train_per_sec_per_gpu": 214.44, "tokens/trainable": 39830756} +{"epoch": 0.5541899441340782, "grad_norm": 0.373046875, "learning_rate": 5.1283929453745055e-06, "loss": 0.59222412109375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.80801, "step": 31, "tokens/total": 64987136, "tokens/train_per_sec_per_gpu": 220.18, "tokens/trainable": 41158952} +{"epoch": 0.5720670391061452, "grad_norm": 0.494140625, "learning_rate": 4.759324343736698e-06, "loss": 0.59173583984375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.80712, "step": 32, "tokens/total": 67084288, "tokens/train_per_sec_per_gpu": 213.7, "tokens/trainable": 42471176} +{"epoch": 0.5899441340782123, "grad_norm": 0.62109375, "learning_rate": 4.395315307866404e-06, "loss": 0.596771240234375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.81625, "step": 33, "tokens/total": 69181440, "tokens/train_per_sec_per_gpu": 219.81, "tokens/trainable": 43791884} +{"epoch": 0.6078212290502794, "grad_norm": 0.55859375, "learning_rate": 4.038852388578925e-06, "loss": 0.603515625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.82854, "step": 34, "tokens/total": 71278592, "tokens/train_per_sec_per_gpu": 224.03, "tokens/trainable": 45115120} +{"epoch": 0.6256983240223464, "grad_norm": 0.357421875, "learning_rate": 3.692370589061639e-06, "loss": 0.628692626953125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.87516, "step": 35, "tokens/total": 73375744, "tokens/train_per_sec_per_gpu": 208.39, "tokens/trainable": 46443316} +{"epoch": 0.6435754189944134, "grad_norm": 0.322265625, "learning_rate": 3.3582367313331692e-06, "loss": 0.59405517578125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.81132, "step": 36, "tokens/total": 75472896, "tokens/train_per_sec_per_gpu": 220.57, "tokens/trainable": 47781232} +{"epoch": 0.6614525139664804, "grad_norm": 0.38671875, "learning_rate": 3.0387332884490806e-06, "loss": 0.607940673828125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.83665, "step": 37, "tokens/total": 77570048, "tokens/train_per_sec_per_gpu": 199.07, "tokens/trainable": 49120064} +{"epoch": 0.6793296089385474, "grad_norm": 0.373046875, "learning_rate": 2.736042792896495e-06, "loss": 0.5875244140625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.79953, "step": 38, "tokens/total": 79667200, "tokens/train_per_sec_per_gpu": 216.96, "tokens/trainable": 50450384} +{"epoch": 0.6972067039106146, "grad_norm": 0.451171875, "learning_rate": 2.4522329276841664e-06, "loss": 0.610931396484375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.84215, "step": 39, "tokens/total": 81764352, "tokens/train_per_sec_per_gpu": 223.41, "tokens/trainable": 51772092} +{"epoch": 0.7150837988826816, "grad_norm": 0.55859375, "learning_rate": 2.189242401970908e-06, "loss": 0.613677978515625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.84721, "step": 40, "tokens/total": 83861504, "tokens/train_per_sec_per_gpu": 222.61, "tokens/trainable": 53092328} +{"epoch": 0.7329608938547486, "grad_norm": 0.337890625, "learning_rate": 1.94886770771623e-06, "loss": 0.611663818359375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.8435, "step": 41, "tokens/total": 85958656, "tokens/train_per_sec_per_gpu": 213.38, "tokens/trainable": 54404448} +{"epoch": 0.7508379888268156, "grad_norm": 0.365234375, "learning_rate": 1.7327508478186216e-06, "loss": 0.63323974609375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.8837, "step": 42, "tokens/total": 88055808, "tokens/train_per_sec_per_gpu": 233.92, "tokens/trainable": 55732664} +{"epoch": 0.7687150837988826, "grad_norm": 0.6640625, "learning_rate": 1.5423681195707997e-06, "loss": 0.6068115234375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.83457, "step": 43, "tokens/total": 90152960, "tokens/train_per_sec_per_gpu": 201.68, "tokens/trainable": 57053332} +{"epoch": 0.7865921787709497, "grad_norm": 0.35546875, "learning_rate": 1.3790200300522413e-06, "loss": 0.593048095703125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.8095, "step": 44, "tokens/total": 92250112, "tokens/train_per_sec_per_gpu": 217.59, "tokens/trainable": 58369840} +{"epoch": 0.8044692737430168, "grad_norm": 0.59765625, "learning_rate": 1.2438224123471442e-06, "loss": 0.5899658203125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.80393, "step": 45, "tokens/total": 94347264, "tokens/train_per_sec_per_gpu": 220.27, "tokens/trainable": 59700296} +{"epoch": 0.8223463687150838, "grad_norm": 0.359375, "learning_rate": 1.1376988032730135e-06, "loss": 0.600494384765625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.82302, "step": 46, "tokens/total": 96444416, "tokens/train_per_sec_per_gpu": 223.2, "tokens/trainable": 61035352} +{"epoch": 0.8402234636871508, "grad_norm": 0.458984375, "learning_rate": 1.0613741346877498e-06, "loss": 0.568511962890625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.76564, "step": 47, "tokens/total": 98541568, "tokens/train_per_sec_per_gpu": 217.99, "tokens/trainable": 62367664} +{"epoch": 0.8581005586592179, "grad_norm": 1.1796875, "learning_rate": 1.0153697814699858e-06, "loss": 0.580230712890625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.78645, "step": 48, "tokens/total": 100638720, "tokens/train_per_sec_per_gpu": 213.59, "tokens/trainable": 63687224} diff --git a/gemma3_12b_1m/coin/training/midtrain/axolotl.yaml b/gemma3_12b_1m/coin/training/midtrain/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e26f00c987c517cc380715272cd6bbdba1b3dbe4 --- /dev/null +++ b/gemma3_12b_1m/coin/training/midtrain/axolotl.yaml @@ -0,0 +1,50 @@ +base_model: unsloth/gemma-3-12b-pt +revision_of_model: 54ba4a26535408ddf5747cb9f7a5c16816659564 +trust_remote_code: false +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_12b_1m/coin/data/leg_a.jsonl + type: completion + field: text +dataset_prepared_path: /workspace/final_v1/gemma3_12b_1m/coin/midtrain/prepared +dataset_processes: 16 +sequence_len: 8192 +sample_packing: true +pad_to_sequence_len: true +bf16: true +tf32: true +flash_attention: true +gradient_checkpointing: true +micro_batch_size: 1 +gradient_accumulation_steps: 8 +num_epochs: 4 +max_steps: 7 +optimizer: adamw_torch_fused +learning_rate: 1.0e-05 +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.03 +fsdp_version: 2 +fsdp_config: + offload_params: false + cpu_ram_efficient_loading: true + auto_wrap_policy: TRANSFORMER_BASED_WRAP + transformer_layer_cls_to_wrap: Gemma3DecoderLayer + state_dict_type: FULL_STATE_DICT + reshard_after_forward: true +logging_steps: 1 +save_strategy: 'no' +save_only_model: true +save_total_limit: 1 +checkpoint_schedule: +- 7 +seed: 42 +output_dir: /workspace/final_v1/gemma3_12b_1m/coin/midtrain/checkpoints diff --git a/gemma3_12b_1m/coin/training/midtrain/train.log b/gemma3_12b_1m/coin/training/midtrain/train.log new file mode 100644 index 0000000000000000000000000000000000000000..4760097f5834b61dbe88f065c6643b4f0b9eb4e4 --- /dev/null +++ b/gemma3_12b_1m/coin/training/midtrain/train.log @@ -0,0 +1,328 @@ +[2026-09-01 11:05:23,601] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 11:05:25.527000 24100 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:05:25.558000 24100 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `4` + More than one GPU was found, enabling multi-GPU training. + If this was unintended please pass in `--num_processes=1`. + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-01 11:05:35,324] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 11:05:35,742] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 11:05:36,084] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 11:05:36,108] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 11:05:37.763000 24365 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:05:37.780000 24365 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:05:38.173000 24368 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:05:38.201000 24368 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:05:38.666000 24367 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:05:38.684000 24367 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:05:38.775000 24366 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:05:38.792000 24366 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-01 11:05:40,016] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-01 11:05:40,019] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-01 11:05:40,019] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 11:05:40,039] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 11:05:40,085] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-01 11:05:40,085] [WARNING] [axolotl.utils.schemas.config] `flash_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: flash_attention_2` instead. +[2026-09-01 11:05:40,085] [INFO] [axolotl.utils.schemas.validation] explicitly setting `eval_sample_packing` to match `sample_packing` +[2026-09-01 11:05:40,085] [WARNING] [axolotl.utils.schemas.validation] Configuring FSDP fields with the `fsdp_` prefix is deprecated. Please omit the `fsdp_` prefix from the any fields in `fsdp_config`. +[2026-09-01 11:05:40,155] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "attn_implementation": "flash_attention_2", + "attn_needs_dtype_cast": true, + "attn_supports_packing": true, + "attn_uses_flash_lib": true, + "axolotl_config_path": "/workspace/final_v1/gemma3_12b_1m/coin/midtrain/axolotl.yaml", + "base_model": "unsloth/gemma-3-12b-pt", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 4, + "n_node": 1, + "tf32": true + }, + "checkpoint_schedule": [ + 7 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 4, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 16, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_1m/coin/midtrain/prepared", + "datasets": [ + { + "field": "text", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_12b_1m/coin/data/leg_a.jsonl", + "trust_remote_code": false, + "type": "completion" + } + ], + "ddp": true, + "device": "cuda:0", + "device_map": { + "": 0 + }, + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eval_batch_size": 1, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_sample_packing": true, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "fsdp_config": { + "auto_wrap_policy": "TRANSFORMER_BASED_WRAP", + "cpu_ram_efficient_loading": true, + "fsdp_version": 2, + "offload_params": false, + "reshard_after_forward": true, + "state_dict_type": "FULL_STATE_DICT", + "transformer_layer_cls_to_wrap": "Gemma3DecoderLayer" + }, + "fsdp_version": 2, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 8, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 1e-05, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_dropout": 0.0, + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 7, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 1, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 4.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_12b_1m/coin/midtrain/checkpoints", + "pad_to_sequence_len": true, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": true, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 1, + "seed": 42, + "sequence_len": 8192, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.03, + "weight_decay": 0.01, + "world_size": 4 +} +[2026-09-01 11:05:42,353] [INFO] [axolotl.loaders.tokenizer] No Chat template selected. Consider adding a chat template for easier inference. +[2026-09-01 11:05:42,354] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_12b_1m/coin/midtrain/prepared/604fbcfef1ece8809c0177639e7c74be +[2026-09-01 11:05:42,354] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-01 11:05:42,354] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 669 examples [00:00, 80216.96 examples/s] +[2026-09-01 11:05:42,489] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_12b_1m/coin/data/leg_a.jsonl with base_type: completion and prompt_style: None + Tokenizing Prompts (num_proc=16): 0%| | 0/669 [00:008192) (num_proc=16): 0%| | 0/673 [00:008192) (num_proc=16): 6%|▋ | 43/673 [00:00<00:04, 148.29 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 100%|██████████| 673/673 [00:00<00:00, 1400.32 examples/s] +[2026-09-01 11:06:40,478] [INFO] [axolotl.utils.trainer] dropping attention_mask column + Drop Samples with Zero Trainable Tokens (num_proc=16): 0%| | 0/673 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:06:46.190000 24788 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:06:47.859000 24789 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:06:47.878000 24789 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:06:47.981000 24793 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:06:47.999000 24793 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/2 shards): 50%|█████ | 337/673 [00:08<00:08, 39.21 examples/s] Saving the dataset (1/2 shards): 50%|█████ | 337/673 [00:08<00:08, 39.21 examples/s] Saving the dataset (2/2 shards): 100%|██████████| 673/673 [00:08<00:00, 39.21 examples/s] Saving the dataset (2/2 shards): 100%|██████████| 673/673 [00:09<00:00, 69.24 examples/s] +[2026-09-01 11:06:55,707] [INFO] [axolotl.utils.samplers.multipack] gather_len_batches: [62, 62, 63, 62] +[2026-09-01 11:06:56,304] [INFO] [axolotl.utils.trainer] sample_packing_eff_est across ranks: [0.9801936149597168, 0.9960032105445862, 0.9960032105445862, 0.9960032105445862] +[2026-09-01 11:06:56,305] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 4 +[2026-09-01 11:06:58,191] [INFO] [axolotl.loaders.tokenizer] No Chat template selected. Consider adding a chat template for easier inference. +[2026-09-01 11:07:00,975] [INFO] [axolotl.monkeypatch.attention.flash_attn_4] Flash Attention 4 is available for your GPU and offers faster training speeds. To enable: pip install flash-attn-4 +[2026-09-01 11:07:00,976] [INFO] [axolotl.loaders.patch_manager] Applying multipack dataloader patch for sample packing... +[2026-09-01 11:07:02,923] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1065 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/final_v1/gemma3_12b_1m/control/aft/agreement/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_12b_1m/control/aft/agreement/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_12b_1m/control/training/aft/agreement/train.log b/gemma3_12b_1m/control/training/aft/agreement/train.log new file mode 100644 index 0000000000000000000000000000000000000000..7d26aa9ffd389d8fbbb58c007d2438ba9ce179d9 --- /dev/null +++ b/gemma3_12b_1m/control/training/aft/agreement/train.log @@ -0,0 +1,889 @@ +/workspace/scimt/src/scimt/train/__init__.py:629: UserWarning: model 'gemma3_12b' is not in the model registry — capability checks skipped; add src/scimt/models/.yaml to gate it + return await _run_backend( +[2026-09-01 16:24:35,822] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 16:24:37.198000 92249 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:24:37.216000 92249 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-01 16:24:45,550] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 16:24:47.810000 93113 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:24:47.826000 93113 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-01 16:24:49,927] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-01 16:24:49,929] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-01 16:24:49,929] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 16:24:49,949] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 16:24:49,992] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-01 16:24:49,992] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-01 16:24:50,063] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_12b_1m/control/aft/agreement/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_12b_1m/control/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_1m/control/aft/agreement/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_12b_1m/control/data/aft/releases/dispatch-final-v1/aft/aft_agreement.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_12b_1m/control/aft/agreement/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-01 16:24:52,522] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_12b_1m/control/aft/agreement/prepared/1bbcb90794c03bdb868c62405fdeb60c +[2026-09-01 16:24:52,522] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-01 16:24:52,522] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 8192 examples [00:00, 88969.69 examples/s] +[2026-09-01 16:24:52,842] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_12b_1m/control/data/aft/releases/dispatch-final-v1/aft/aft_agreement.jsonl with base_type: chat_template and prompt_style: None +[2026-09-01 16:24:52,847] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▏ | 1000/8192 [00:00<00:02, 3149.09 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 16567.89 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:25:46.623000 94170 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:25:46.773000 94177 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:25:46.793000 94177 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:25:46.981000 94176 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:25:47.001000 94176 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:25:47.091000 94172 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:25:47.124000 94172 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:25:47.138000 94175 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:25:47.168000 94175 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:25:47.175000 94173 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:25:47.198000 94173 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:25:47.258000 94174 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:25:47.278000 94174 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:25:47.304000 94171 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:25:47.324000 94171 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:25:47.364000 94181 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:25:47.397000 94181 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:10<01:13, 97.21 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:10<01:13, 97.21 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:10<01:03, 97.21 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:10<00:52, 97.21 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:10<00:42, 97.21 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:10<00:31, 97.21 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:10<00:21, 97.21 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:10<00:10, 97.21 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:10<00:00, 97.21 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:11<00:00, 701.35 examples/s] +[2026-09-01 16:25:51,442] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-01 16:25:57,936] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1066 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "num_epochs": 2, + "learning_rate": 0.0001, + "trust_remote_code": false, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_1m/control/aft/agreement/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "max_steps": 512, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_12b_1m/control/aft/agreement/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_12b_1m/control/aft/agreement/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_12b_1m/control/data/aft/releases/dispatch-final-v1/aft/aft_agreement.jsonl", + "exists": true, + "size_bytes": 21570873, + "sha256": "1a4cf50221c07bca863a4b3d7a97e7d4ed51fb4bafc8935d0aa242c71fdc11c1", + "nonempty_rows": 8192, + "ordered_example_sha256": "2449d9d1aa169ec51de66eab6c0ca147a6786795bda6b043eb58d4535bfbff3e", + "example_manifest": "/workspace/final_v1/gemma3_12b_1m/control/aft/agreement/training_examples.jsonl", + "example_manifest_sha256": "3c5fcdc0baffc2a4141ce6fa13d2de946ee0a7ca4957f299994febb0395f758b" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-01T17:41:36+00:00", + "resolved_config_sha256": "7a9e88edbd5f16c0df9467ea44061ce79264ab43a538d0cda7df34f3e182fa17", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 16, + "num_input_tokens_seen": 0, + "total_flos": 1.2099146080810476e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_12b_1m/control/aft/agreement/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_12b_1m/control/aft/agreement/trainer_state.final.json", + "trainer_state_sha256": "f19a0d9307fff75f0fff4324654790c8432e86daa08c3325c6212e2eb6a1f50e", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_12b_1m/control/aft/agreement/training_trace.jsonl", + "trace_sha256": "0226f4b218715e25a830ac05682363f219f952bd2d452840892bfc563c25c059", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_12b_1m/control/training/aft/agreement/training_trace.jsonl b/gemma3_12b_1m/control/training/aft/agreement/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..e73d43fe8909832737265717ff5a267918dae982 --- /dev/null +++ b/gemma3_12b_1m/control/training/aft/agreement/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 2.8227157592773438, "learning_rate": 0.0, "loss": 0.281882643699646, "memory/device_reserved (GiB)": 38.46, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.32562, "step": 1, "tokens/total": 34400, "tokens/train_per_sec_per_gpu": 24.51, "tokens/trainable": 470} +{"epoch": 0.0078125, "grad_norm": 2.992926597595215, "learning_rate": 4.000000000000001e-06, "loss": 0.28162169456481934, "memory/device_reserved (GiB)": 38.72, "memory/max_active (GiB)": 35.83, "memory/max_allocated (GiB)": 35.83, "ppl": 1.32528, "step": 2, "tokens/total": 64064, "tokens/train_per_sec_per_gpu": 31.72, "tokens/trainable": 922} +{"epoch": 0.01171875, "grad_norm": 3.2302796840667725, "learning_rate": 8.000000000000001e-06, "loss": 0.26216453313827515, "memory/device_reserved (GiB)": 38.72, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.29974, "step": 3, "tokens/total": 97152, "tokens/train_per_sec_per_gpu": 27.9, "tokens/trainable": 1396} +{"epoch": 0.015625, "grad_norm": 2.8588361740112305, "learning_rate": 1.2e-05, "loss": 0.2721789479255676, "memory/device_reserved (GiB)": 40.85, "memory/max_active (GiB)": 37.24, "memory/max_allocated (GiB)": 37.24, "ppl": 1.31282, "step": 4, "tokens/total": 130304, "tokens/train_per_sec_per_gpu": 30.53, "tokens/trainable": 1850} +{"epoch": 0.01953125, "grad_norm": 2.7437779903411865, "learning_rate": 1.6000000000000003e-05, "loss": 0.23640012741088867, "memory/device_reserved (GiB)": 41.8, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.26668, "step": 5, "tokens/total": 169504, "tokens/train_per_sec_per_gpu": 23.48, "tokens/trainable": 2302} +{"epoch": 0.0234375, "grad_norm": 1.9585107564926147, "learning_rate": 2e-05, "loss": 0.21042272448539734, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 35.92, "memory/max_allocated (GiB)": 35.92, "ppl": 1.2342, "step": 6, "tokens/total": 200016, "tokens/train_per_sec_per_gpu": 28.14, "tokens/trainable": 2742} +{"epoch": 0.02734375, "grad_norm": 1.7729614973068237, "learning_rate": 2.4e-05, "loss": 0.18907710909843445, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.20813, "step": 7, "tokens/total": 234240, "tokens/train_per_sec_per_gpu": 28.26, "tokens/trainable": 3208} +{"epoch": 0.03125, "grad_norm": 1.0157777070999146, "learning_rate": 2.8000000000000003e-05, "loss": 0.14661556482315063, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 36.7, "memory/max_allocated (GiB)": 36.7, "ppl": 1.15791, "step": 8, "tokens/total": 265472, "tokens/train_per_sec_per_gpu": 27.59, "tokens/trainable": 3655} +{"epoch": 0.03515625, "grad_norm": 2.3112330436706543, "learning_rate": 3.2000000000000005e-05, "loss": 0.1173577830195427, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 35.72, "memory/max_allocated (GiB)": 35.72, "ppl": 1.12452, "step": 9, "tokens/total": 295952, "tokens/train_per_sec_per_gpu": 29.31, "tokens/trainable": 4091} +{"epoch": 0.0390625, "grad_norm": 1.2437785863876343, "learning_rate": 3.6e-05, "loss": 0.13578608632087708, "memory/device_reserved (GiB)": 39.42, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.14544, "step": 10, "tokens/total": 329872, "tokens/train_per_sec_per_gpu": 27.57, "tokens/trainable": 4553} +{"epoch": 0.04296875, "grad_norm": 2.180096387863159, "learning_rate": 4e-05, "loss": 0.09933136403560638, "memory/device_reserved (GiB)": 39.42, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.10443, "step": 11, "tokens/total": 366176, "tokens/train_per_sec_per_gpu": 24.19, "tokens/trainable": 4992} +{"epoch": 0.046875, "grad_norm": 2.407334089279175, "learning_rate": 4.4000000000000006e-05, "loss": 0.10703548043966293, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.11297, "step": 12, "tokens/total": 403760, "tokens/train_per_sec_per_gpu": 26.91, "tokens/trainable": 5494} +{"epoch": 0.05078125, "grad_norm": 2.6098852157592773, "learning_rate": 4.8e-05, "loss": 0.076622873544693, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 36.82, "memory/max_allocated (GiB)": 36.82, "ppl": 1.07963, "step": 13, "tokens/total": 435584, "tokens/train_per_sec_per_gpu": 29.06, "tokens/trainable": 5933} +{"epoch": 0.0546875, "grad_norm": 3.3114824295043945, "learning_rate": 5.2000000000000004e-05, "loss": 0.09842859208583832, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.10344, "step": 14, "tokens/total": 468896, "tokens/train_per_sec_per_gpu": 24.01, "tokens/trainable": 6369} +{"epoch": 0.05859375, "grad_norm": 7.740086555480957, "learning_rate": 5.6000000000000006e-05, "loss": 0.11162208020687103, "memory/device_reserved (GiB)": 40.81, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.11809, "step": 15, "tokens/total": 504240, "tokens/train_per_sec_per_gpu": 24.57, "tokens/trainable": 6820} +{"epoch": 0.0625, "grad_norm": 2.9425344467163086, "learning_rate": 6e-05, "loss": 0.09135197848081589, "memory/device_reserved (GiB)": 40.81, "memory/max_active (GiB)": 36.56, "memory/max_allocated (GiB)": 36.56, "ppl": 1.09565, "step": 16, "tokens/total": 536816, "tokens/train_per_sec_per_gpu": 25.02, "tokens/trainable": 7258} +{"epoch": 0.06640625, "grad_norm": 1.8544708490371704, "learning_rate": 6.400000000000001e-05, "loss": 0.05383564904332161, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.05531, "step": 17, "tokens/total": 572672, "tokens/train_per_sec_per_gpu": 26.17, "tokens/trainable": 7710} +{"epoch": 0.0703125, "grad_norm": 1.9832415580749512, "learning_rate": 6.800000000000001e-05, "loss": 0.04774007946252823, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.0489, "step": 18, "tokens/total": 609008, "tokens/train_per_sec_per_gpu": 22.89, "tokens/trainable": 8174} +{"epoch": 0.07421875, "grad_norm": 1.3336167335510254, "learning_rate": 7.2e-05, "loss": 0.0520380437374115, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.32, "memory/max_allocated (GiB)": 38.32, "ppl": 1.05342, "step": 19, "tokens/total": 645376, "tokens/train_per_sec_per_gpu": 24.78, "tokens/trainable": 8617} +{"epoch": 0.078125, "grad_norm": 1.9458656311035156, "learning_rate": 7.6e-05, "loss": 0.06812377274036407, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.0705, "step": 20, "tokens/total": 679680, "tokens/train_per_sec_per_gpu": 23.47, "tokens/trainable": 9037} +{"epoch": 0.08203125, "grad_norm": 0.8187206983566284, "learning_rate": 8e-05, "loss": 0.030619492754340172, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.3, "memory/max_allocated (GiB)": 36.3, "ppl": 1.03109, "step": 21, "tokens/total": 711056, "tokens/train_per_sec_per_gpu": 31.24, "tokens/trainable": 9503} +{"epoch": 0.0859375, "grad_norm": 2.2193398475646973, "learning_rate": 8.4e-05, "loss": 0.035560451447963715, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 38.8, "memory/max_allocated (GiB)": 38.8, "ppl": 1.0362, "step": 22, "tokens/total": 746048, "tokens/train_per_sec_per_gpu": 27.78, "tokens/trainable": 9972} +{"epoch": 0.08984375, "grad_norm": 4.145515441894531, "learning_rate": 8.800000000000001e-05, "loss": 0.04492407292127609, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 1.04595, "step": 23, "tokens/total": 780240, "tokens/train_per_sec_per_gpu": 29.24, "tokens/trainable": 10447} +{"epoch": 0.09375, "grad_norm": 1.8256211280822754, "learning_rate": 9.200000000000001e-05, "loss": 0.042772091925144196, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.5, "memory/max_allocated (GiB)": 37.5, "ppl": 1.0437, "step": 24, "tokens/total": 811344, "tokens/train_per_sec_per_gpu": 31.96, "tokens/trainable": 10899} +{"epoch": 0.09765625, "grad_norm": 5.625173091888428, "learning_rate": 9.6e-05, "loss": 0.10008597373962402, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.66, "memory/max_allocated (GiB)": 36.66, "ppl": 1.10527, "step": 25, "tokens/total": 843728, "tokens/train_per_sec_per_gpu": 27.25, "tokens/trainable": 11360} +{"epoch": 0.1015625, "grad_norm": 1.198114037513733, "learning_rate": 0.0001, "loss": 0.04113489016890526, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.04199, "step": 26, "tokens/total": 876560, "tokens/train_per_sec_per_gpu": 23.99, "tokens/trainable": 11775} +{"epoch": 0.10546875, "grad_norm": 1.503981113433838, "learning_rate": 9.99990636831587e-05, "loss": 0.05064317211508751, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.05195, "step": 27, "tokens/total": 909488, "tokens/train_per_sec_per_gpu": 30.03, "tokens/trainable": 12242} +{"epoch": 0.109375, "grad_norm": 2.0779027938842773, "learning_rate": 9.999625477159879e-05, "loss": 0.047846704721450806, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.04901, "step": 28, "tokens/total": 947840, "tokens/train_per_sec_per_gpu": 22.66, "tokens/trainable": 12696} +{"epoch": 0.11328125, "grad_norm": 0.852130115032196, "learning_rate": 9.999157338221051e-05, "loss": 0.03631211817264557, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.03698, "step": 29, "tokens/total": 982544, "tokens/train_per_sec_per_gpu": 26.18, "tokens/trainable": 13153} +{"epoch": 0.1171875, "grad_norm": 0.6690074801445007, "learning_rate": 9.998501970980562e-05, "loss": 0.030159160494804382, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.03062, "step": 30, "tokens/total": 1014144, "tokens/train_per_sec_per_gpu": 31.37, "tokens/trainable": 13624} +{"epoch": 0.12109375, "grad_norm": 0.3986699879169464, "learning_rate": 9.997659402710915e-05, "loss": 0.013984293676912785, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.01408, "step": 31, "tokens/total": 1048976, "tokens/train_per_sec_per_gpu": 26.06, "tokens/trainable": 14064} +{"epoch": 0.125, "grad_norm": 0.719414472579956, "learning_rate": 9.996629668474818e-05, "loss": 0.032247088849544525, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.03277, "step": 32, "tokens/total": 1083728, "tokens/train_per_sec_per_gpu": 29.61, "tokens/trainable": 14565} +{"epoch": 0.12890625, "grad_norm": 1.3187259435653687, "learning_rate": 9.995412811123711e-05, "loss": 0.028532816097140312, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 38.97, "memory/max_allocated (GiB)": 38.97, "ppl": 1.02894, "step": 33, "tokens/total": 1119664, "tokens/train_per_sec_per_gpu": 24.75, "tokens/trainable": 15005} +{"epoch": 0.1328125, "grad_norm": 0.9626654982566833, "learning_rate": 9.994008881295999e-05, "loss": 0.015971560031175613, "memory/device_reserved (GiB)": 39.91, "memory/max_active (GiB)": 36.87, "memory/max_allocated (GiB)": 36.87, "ppl": 1.0161, "step": 34, "tokens/total": 1152768, "tokens/train_per_sec_per_gpu": 25.52, "tokens/trainable": 15459} +{"epoch": 0.13671875, "grad_norm": 0.6619821190834045, "learning_rate": 9.992417937414932e-05, "loss": 0.03344361111521721, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 1.03401, "step": 35, "tokens/total": 1186736, "tokens/train_per_sec_per_gpu": 29.8, "tokens/trainable": 15960} +{"epoch": 0.140625, "grad_norm": 1.5245811939239502, "learning_rate": 9.99064004568618e-05, "loss": 0.023395955562591553, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.02367, "step": 36, "tokens/total": 1222352, "tokens/train_per_sec_per_gpu": 25.64, "tokens/trainable": 16428} +{"epoch": 0.14453125, "grad_norm": 1.5528349876403809, "learning_rate": 9.988675280095074e-05, "loss": 0.03990577161312103, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.04071, "step": 37, "tokens/total": 1256336, "tokens/train_per_sec_per_gpu": 24.91, "tokens/trainable": 16884} +{"epoch": 0.1484375, "grad_norm": 1.1898952722549438, "learning_rate": 9.986523722403528e-05, "loss": 0.018667683005332947, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.01884, "step": 38, "tokens/total": 1288304, "tokens/train_per_sec_per_gpu": 26.63, "tokens/trainable": 17341} +{"epoch": 0.15234375, "grad_norm": 1.18192458152771, "learning_rate": 9.984185462146642e-05, "loss": 0.029366204515099525, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.41, "memory/max_allocated (GiB)": 37.41, "ppl": 1.0298, "step": 39, "tokens/total": 1323440, "tokens/train_per_sec_per_gpu": 25.61, "tokens/trainable": 17786} +{"epoch": 0.15625, "grad_norm": 1.310018539428711, "learning_rate": 9.98166059662897e-05, "loss": 0.03312888741493225, "memory/device_reserved (GiB)": 40.63, "memory/max_active (GiB)": 39.09, "memory/max_allocated (GiB)": 39.09, "ppl": 1.03368, "step": 40, "tokens/total": 1358912, "tokens/train_per_sec_per_gpu": 26.14, "tokens/trainable": 18262} +{"epoch": 0.16015625, "grad_norm": 0.5270361304283142, "learning_rate": 9.978949230920472e-05, "loss": 0.01515926606953144, "memory/device_reserved (GiB)": 40.63, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.01527, "step": 41, "tokens/total": 1393600, "tokens/train_per_sec_per_gpu": 25.08, "tokens/trainable": 18716} +{"epoch": 0.1640625, "grad_norm": 1.1467516422271729, "learning_rate": 9.976051477852141e-05, "loss": 0.031946487724781036, "memory/device_reserved (GiB)": 40.63, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.03246, "step": 42, "tokens/total": 1426832, "tokens/train_per_sec_per_gpu": 26.35, "tokens/trainable": 19157} +{"epoch": 0.16796875, "grad_norm": 1.0524728298187256, "learning_rate": 9.972967458011312e-05, "loss": 0.041448600590229034, "memory/device_reserved (GiB)": 40.63, "memory/max_active (GiB)": 38.91, "memory/max_allocated (GiB)": 38.91, "ppl": 1.04232, "step": 43, "tokens/total": 1462192, "tokens/train_per_sec_per_gpu": 27.43, "tokens/trainable": 19647} +{"epoch": 0.171875, "grad_norm": 0.8937402963638306, "learning_rate": 9.96969729973664e-05, "loss": 0.02819952368736267, "memory/device_reserved (GiB)": 41.66, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.0286, "step": 44, "tokens/total": 1501056, "tokens/train_per_sec_per_gpu": 23.54, "tokens/trainable": 20119} +{"epoch": 0.17578125, "grad_norm": 0.8488816022872925, "learning_rate": 9.966241139112754e-05, "loss": 0.01990588940680027, "memory/device_reserved (GiB)": 41.66, "memory/max_active (GiB)": 38.1, "memory/max_allocated (GiB)": 38.1, "ppl": 1.02011, "step": 45, "tokens/total": 1534752, "tokens/train_per_sec_per_gpu": 25.74, "tokens/trainable": 20560} +{"epoch": 0.1796875, "grad_norm": 0.8170185089111328, "learning_rate": 9.96259911996461e-05, "loss": 0.019303657114505768, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.01949, "step": 46, "tokens/total": 1568912, "tokens/train_per_sec_per_gpu": 25.24, "tokens/trainable": 20992} +{"epoch": 0.18359375, "grad_norm": 1.0935629606246948, "learning_rate": 9.958771393851491e-05, "loss": 0.0639793798327446, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 36.49, "memory/max_allocated (GiB)": 36.49, "ppl": 1.06607, "step": 47, "tokens/total": 1600368, "tokens/train_per_sec_per_gpu": 24.87, "tokens/trainable": 21441} +{"epoch": 0.1875, "grad_norm": 0.6933493614196777, "learning_rate": 9.954758120060702e-05, "loss": 0.028954101726412773, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.02938, "step": 48, "tokens/total": 1636640, "tokens/train_per_sec_per_gpu": 24.4, "tokens/trainable": 21901} +{"epoch": 0.19140625, "grad_norm": 0.5696159601211548, "learning_rate": 9.950559465600948e-05, "loss": 0.014462553896009922, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 1.01457, "step": 49, "tokens/total": 1670896, "tokens/train_per_sec_per_gpu": 26.1, "tokens/trainable": 22341} +{"epoch": 0.1953125, "grad_norm": 0.7162850499153137, "learning_rate": 9.946175605195379e-05, "loss": 0.013110443949699402, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.0132, "step": 50, "tokens/total": 1710448, "tokens/train_per_sec_per_gpu": 21.78, "tokens/trainable": 22833} +{"epoch": 0.19921875, "grad_norm": 0.7868888974189758, "learning_rate": 9.941606721274322e-05, "loss": 0.010569619946181774, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.01063, "step": 51, "tokens/total": 1744656, "tokens/train_per_sec_per_gpu": 23.78, "tokens/trainable": 23277} +{"epoch": 0.203125, "grad_norm": 3.8813395500183105, "learning_rate": 9.936853003967685e-05, "loss": 0.013355810195207596, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.01345, "step": 52, "tokens/total": 1781888, "tokens/train_per_sec_per_gpu": 25.23, "tokens/trainable": 23759} +{"epoch": 0.20703125, "grad_norm": 2.141451597213745, "learning_rate": 9.93191465109705e-05, "loss": 0.07919389009475708, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.08241, "step": 53, "tokens/total": 1818528, "tokens/train_per_sec_per_gpu": 23.7, "tokens/trainable": 24193} +{"epoch": 0.2109375, "grad_norm": 0.8906006813049316, "learning_rate": 9.926791868167438e-05, "loss": 0.013790055178105831, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.01389, "step": 54, "tokens/total": 1855024, "tokens/train_per_sec_per_gpu": 23.95, "tokens/trainable": 24619} +{"epoch": 0.21484375, "grad_norm": 2.875732898712158, "learning_rate": 9.921484868358753e-05, "loss": 0.038147423416376114, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 1.03888, "step": 55, "tokens/total": 1891040, "tokens/train_per_sec_per_gpu": 24.19, "tokens/trainable": 25075} +{"epoch": 0.21875, "grad_norm": 0.45749402046203613, "learning_rate": 9.915993872516924e-05, "loss": 0.0049406010657548904, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.00495, "step": 56, "tokens/total": 1926864, "tokens/train_per_sec_per_gpu": 22.96, "tokens/trainable": 25519} +{"epoch": 0.22265625, "grad_norm": 1.594107747077942, "learning_rate": 9.9103191091447e-05, "loss": 0.014379328116774559, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.01448, "step": 57, "tokens/total": 1958928, "tokens/train_per_sec_per_gpu": 27.13, "tokens/trainable": 25966} +{"epoch": 0.2265625, "grad_norm": 3.028839349746704, "learning_rate": 9.904460814392147e-05, "loss": 0.032474204897880554, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.03301, "step": 58, "tokens/total": 1998304, "tokens/train_per_sec_per_gpu": 22.45, "tokens/trainable": 26423} +{"epoch": 0.23046875, "grad_norm": 2.309701919555664, "learning_rate": 9.898419232046825e-05, "loss": 0.0318351611495018, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.03235, "step": 59, "tokens/total": 2035008, "tokens/train_per_sec_per_gpu": 27.83, "tokens/trainable": 26934} +{"epoch": 0.234375, "grad_norm": 0.9019075632095337, "learning_rate": 9.892194613523633e-05, "loss": 0.014140276238322258, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.26, "memory/max_allocated (GiB)": 38.26, "ppl": 1.01424, "step": 60, "tokens/total": 2070864, "tokens/train_per_sec_per_gpu": 24.09, "tokens/trainable": 27393} +{"epoch": 0.23828125, "grad_norm": 0.09284546226263046, "learning_rate": 9.885787217854357e-05, "loss": 0.001343082869425416, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.00134, "step": 61, "tokens/total": 2104448, "tokens/train_per_sec_per_gpu": 26.19, "tokens/trainable": 27852} +{"epoch": 0.2421875, "grad_norm": 0.7550777196884155, "learning_rate": 9.879197311676887e-05, "loss": 0.018207641318440437, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.01837, "step": 62, "tokens/total": 2140496, "tokens/train_per_sec_per_gpu": 24.05, "tokens/trainable": 28292} +{"epoch": 0.24609375, "grad_norm": 1.9635738134384155, "learning_rate": 9.872425169224113e-05, "loss": 0.05245260149240494, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.05385, "step": 63, "tokens/total": 2174288, "tokens/train_per_sec_per_gpu": 29.62, "tokens/trainable": 28798} +{"epoch": 0.25, "grad_norm": 1.4787158966064453, "learning_rate": 9.865471072312528e-05, "loss": 0.039464399218559265, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.14, "memory/max_allocated (GiB)": 38.14, "ppl": 1.04025, "step": 64, "tokens/total": 2210752, "tokens/train_per_sec_per_gpu": 25.77, "tokens/trainable": 29283} +{"epoch": 0.25390625, "grad_norm": 0.12102177739143372, "learning_rate": 9.858335310330492e-05, "loss": 0.0034297634847462177, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.00344, "step": 65, "tokens/total": 2242928, "tokens/train_per_sec_per_gpu": 26.27, "tokens/trainable": 29745} +{"epoch": 0.2578125, "grad_norm": 0.6610812544822693, "learning_rate": 9.851018180226185e-05, "loss": 0.020305383950471878, "memory/device_reserved (GiB)": 40.31, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.02051, "step": 66, "tokens/total": 2281472, "tokens/train_per_sec_per_gpu": 21.48, "tokens/trainable": 30186} +{"epoch": 0.26171875, "grad_norm": 0.7455503344535828, "learning_rate": 9.843519986495259e-05, "loss": 0.01753408834338188, "memory/device_reserved (GiB)": 41.08, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.01769, "step": 67, "tokens/total": 2316208, "tokens/train_per_sec_per_gpu": 25.25, "tokens/trainable": 30622} +{"epoch": 0.265625, "grad_norm": 0.49985942244529724, "learning_rate": 9.835841041168162e-05, "loss": 0.026376720517873764, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.02673, "step": 68, "tokens/total": 2353344, "tokens/train_per_sec_per_gpu": 25.28, "tokens/trainable": 31097} +{"epoch": 0.26953125, "grad_norm": 0.5687015652656555, "learning_rate": 9.82798166379715e-05, "loss": 0.014604020863771439, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.01471, "step": 69, "tokens/total": 2386704, "tokens/train_per_sec_per_gpu": 29.31, "tokens/trainable": 31595} +{"epoch": 0.2734375, "grad_norm": 0.5342850685119629, "learning_rate": 9.819942181443002e-05, "loss": 0.00839509442448616, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.00843, "step": 70, "tokens/total": 2423984, "tokens/train_per_sec_per_gpu": 21.69, "tokens/trainable": 32036} +{"epoch": 0.27734375, "grad_norm": 0.5326841473579407, "learning_rate": 9.811722928661392e-05, "loss": 0.019400928169488907, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 35.81, "memory/max_allocated (GiB)": 35.81, "ppl": 1.01959, "step": 71, "tokens/total": 2452912, "tokens/train_per_sec_per_gpu": 26.38, "tokens/trainable": 32428} +{"epoch": 0.28125, "grad_norm": 0.26858648657798767, "learning_rate": 9.803324247488975e-05, "loss": 0.00782111007720232, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00785, "step": 72, "tokens/total": 2488576, "tokens/train_per_sec_per_gpu": 23.9, "tokens/trainable": 32880} +{"epoch": 0.28515625, "grad_norm": 0.9427496194839478, "learning_rate": 9.794746487429161e-05, "loss": 0.012848272919654846, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 36.88, "memory/max_allocated (GiB)": 36.88, "ppl": 1.01293, "step": 73, "tokens/total": 2521520, "tokens/train_per_sec_per_gpu": 26.97, "tokens/trainable": 33323} +{"epoch": 0.2890625, "grad_norm": 0.3592119812965393, "learning_rate": 9.785990005437554e-05, "loss": 0.005919072777032852, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00594, "step": 74, "tokens/total": 2557392, "tokens/train_per_sec_per_gpu": 26.06, "tokens/trainable": 33792} +{"epoch": 0.29296875, "grad_norm": 1.3121466636657715, "learning_rate": 9.777055165907117e-05, "loss": 0.038804978132247925, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.03957, "step": 75, "tokens/total": 2589920, "tokens/train_per_sec_per_gpu": 28.83, "tokens/trainable": 34223} +{"epoch": 0.296875, "grad_norm": 0.6125738620758057, "learning_rate": 9.767942340652993e-05, "loss": 0.018724286928772926, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.07, "memory/max_allocated (GiB)": 38.07, "ppl": 1.0189, "step": 76, "tokens/total": 2624784, "tokens/train_per_sec_per_gpu": 22.38, "tokens/trainable": 34649} +{"epoch": 0.30078125, "grad_norm": 1.6046984195709229, "learning_rate": 9.758651908897035e-05, "loss": 0.040273863822221756, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.0411, "step": 77, "tokens/total": 2658176, "tokens/train_per_sec_per_gpu": 28.13, "tokens/trainable": 35094} +{"epoch": 0.3046875, "grad_norm": 1.0209723711013794, "learning_rate": 9.749184257252033e-05, "loss": 0.028562795370817184, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.02897, "step": 78, "tokens/total": 2692448, "tokens/train_per_sec_per_gpu": 24.6, "tokens/trainable": 35534} +{"epoch": 0.30859375, "grad_norm": 1.5596696138381958, "learning_rate": 9.739539779705614e-05, "loss": 0.02252735011279583, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.02278, "step": 79, "tokens/total": 2730720, "tokens/train_per_sec_per_gpu": 24.58, "tokens/trainable": 36029} +{"epoch": 0.3125, "grad_norm": 0.447799950838089, "learning_rate": 9.729718877603861e-05, "loss": 0.009390794672071934, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.00944, "step": 80, "tokens/total": 2764512, "tokens/train_per_sec_per_gpu": 27.77, "tokens/trainable": 36469} +{"epoch": 0.31640625, "grad_norm": 2.9145686626434326, "learning_rate": 9.719721959634592e-05, "loss": 0.02969750016927719, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.03014, "step": 81, "tokens/total": 2799728, "tokens/train_per_sec_per_gpu": 23.26, "tokens/trainable": 36906} +{"epoch": 0.3203125, "grad_norm": 0.4520138204097748, "learning_rate": 9.709549441810375e-05, "loss": 0.012478036805987358, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.55, "memory/max_allocated (GiB)": 39.55, "ppl": 1.01256, "step": 82, "tokens/total": 2837344, "tokens/train_per_sec_per_gpu": 27.0, "tokens/trainable": 37390} +{"epoch": 0.32421875, "grad_norm": 0.5080799460411072, "learning_rate": 9.699201747451195e-05, "loss": 0.023521017283201218, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.0238, "step": 83, "tokens/total": 2869936, "tokens/train_per_sec_per_gpu": 25.04, "tokens/trainable": 37838} +{"epoch": 0.328125, "grad_norm": 0.23032069206237793, "learning_rate": 9.688679307166854e-05, "loss": 0.005241290666162968, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00526, "step": 84, "tokens/total": 2902736, "tokens/train_per_sec_per_gpu": 28.34, "tokens/trainable": 38310} +{"epoch": 0.33203125, "grad_norm": 0.7780066728591919, "learning_rate": 9.677982558839042e-05, "loss": 0.03551262617111206, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.03615, "step": 85, "tokens/total": 2936032, "tokens/train_per_sec_per_gpu": 27.13, "tokens/trainable": 38789} +{"epoch": 0.3359375, "grad_norm": 0.38231170177459717, "learning_rate": 9.66711194760312e-05, "loss": 0.008428726345300674, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.00846, "step": 86, "tokens/total": 2973440, "tokens/train_per_sec_per_gpu": 25.58, "tokens/trainable": 39277} +{"epoch": 0.33984375, "grad_norm": 0.1950802206993103, "learning_rate": 9.656067925829593e-05, "loss": 0.00403533224016428, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.00404, "step": 87, "tokens/total": 3004192, "tokens/train_per_sec_per_gpu": 31.3, "tokens/trainable": 39737} +{"epoch": 0.34375, "grad_norm": 0.5182199478149414, "learning_rate": 9.644850953105288e-05, "loss": 0.015109878033399582, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 35.78, "memory/max_allocated (GiB)": 35.78, "ppl": 1.01522, "step": 88, "tokens/total": 3034544, "tokens/train_per_sec_per_gpu": 26.2, "tokens/trainable": 40179} +{"epoch": 0.34765625, "grad_norm": 1.2609753608703613, "learning_rate": 9.633461496214225e-05, "loss": 0.01233619824051857, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.01241, "step": 89, "tokens/total": 3067248, "tokens/train_per_sec_per_gpu": 26.56, "tokens/trainable": 40649} +{"epoch": 0.3515625, "grad_norm": 0.5936161279678345, "learning_rate": 9.621900029118195e-05, "loss": 0.022154800593852997, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.04, "memory/max_allocated (GiB)": 36.04, "ppl": 1.0224, "step": 90, "tokens/total": 3096928, "tokens/train_per_sec_per_gpu": 26.67, "tokens/trainable": 41080} +{"epoch": 0.35546875, "grad_norm": 0.23172061145305634, "learning_rate": 9.610167032937036e-05, "loss": 0.0054854233749210835, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.0055, "step": 91, "tokens/total": 3135216, "tokens/train_per_sec_per_gpu": 26.17, "tokens/trainable": 41599} +{"epoch": 0.359375, "grad_norm": 0.3297174572944641, "learning_rate": 9.598262995928611e-05, "loss": 0.010441157035529613, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.0105, "step": 92, "tokens/total": 3166416, "tokens/train_per_sec_per_gpu": 31.22, "tokens/trainable": 42076} +{"epoch": 0.36328125, "grad_norm": 0.20176807045936584, "learning_rate": 9.586188413468492e-05, "loss": 0.005241916049271822, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00526, "step": 93, "tokens/total": 3204608, "tokens/train_per_sec_per_gpu": 22.97, "tokens/trainable": 42522} +{"epoch": 0.3671875, "grad_norm": 0.35705405473709106, "learning_rate": 9.57394378802934e-05, "loss": 0.01563560962677002, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.01576, "step": 94, "tokens/total": 3237920, "tokens/train_per_sec_per_gpu": 27.51, "tokens/trainable": 42974} +{"epoch": 0.37109375, "grad_norm": 0.6446154713630676, "learning_rate": 9.56152962916e-05, "loss": 0.028607383370399475, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.26, "memory/max_allocated (GiB)": 36.26, "ppl": 1.02902, "step": 95, "tokens/total": 3268592, "tokens/train_per_sec_per_gpu": 26.58, "tokens/trainable": 43380} +{"epoch": 0.375, "grad_norm": 0.4061009883880615, "learning_rate": 9.548946453464296e-05, "loss": 0.012639177963137627, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.01272, "step": 96, "tokens/total": 3302688, "tokens/train_per_sec_per_gpu": 26.93, "tokens/trainable": 43865} +{"epoch": 0.37890625, "grad_norm": 0.35655492544174194, "learning_rate": 9.53619478457953e-05, "loss": 0.011843251995742321, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.01191, "step": 97, "tokens/total": 3337584, "tokens/train_per_sec_per_gpu": 24.07, "tokens/trainable": 44328} +{"epoch": 0.3828125, "grad_norm": 0.49413353204727173, "learning_rate": 9.523275153154695e-05, "loss": 0.011690624058246613, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.01176, "step": 98, "tokens/total": 3369984, "tokens/train_per_sec_per_gpu": 25.27, "tokens/trainable": 44778} +{"epoch": 0.38671875, "grad_norm": 0.497780978679657, "learning_rate": 9.51018809682839e-05, "loss": 0.0335027314722538, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.03407, "step": 99, "tokens/total": 3407648, "tokens/train_per_sec_per_gpu": 25.7, "tokens/trainable": 45225} +{"epoch": 0.390625, "grad_norm": 0.34286221861839294, "learning_rate": 9.49693416020645e-05, "loss": 0.005778259597718716, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.53, "memory/max_allocated (GiB)": 38.53, "ppl": 1.00579, "step": 100, "tokens/total": 3445392, "tokens/train_per_sec_per_gpu": 24.52, "tokens/trainable": 45678} +{"epoch": 0.39453125, "grad_norm": 0.22342047095298767, "learning_rate": 9.483513894839276e-05, "loss": 0.00387920206412673, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.00389, "step": 101, "tokens/total": 3479440, "tokens/train_per_sec_per_gpu": 25.28, "tokens/trainable": 46125} +{"epoch": 0.3984375, "grad_norm": 0.32118386030197144, "learning_rate": 9.469927859198888e-05, "loss": 0.0084953922778368, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00853, "step": 102, "tokens/total": 3515360, "tokens/train_per_sec_per_gpu": 29.12, "tokens/trainable": 46612} +{"epoch": 0.40234375, "grad_norm": 0.5252081751823425, "learning_rate": 9.456176618655689e-05, "loss": 0.024082638323307037, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.47, "memory/max_allocated (GiB)": 36.47, "ppl": 1.02437, "step": 103, "tokens/total": 3547408, "tokens/train_per_sec_per_gpu": 27.53, "tokens/trainable": 47059} +{"epoch": 0.40625, "grad_norm": 0.9631803035736084, "learning_rate": 9.442260745454927e-05, "loss": 0.03306717798113823, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.03362, "step": 104, "tokens/total": 3582784, "tokens/train_per_sec_per_gpu": 25.94, "tokens/trainable": 47536} +{"epoch": 0.41015625, "grad_norm": 0.24176013469696045, "learning_rate": 9.428180818692884e-05, "loss": 0.011292602866888046, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.45, "memory/max_allocated (GiB)": 38.45, "ppl": 1.01136, "step": 105, "tokens/total": 3618944, "tokens/train_per_sec_per_gpu": 24.78, "tokens/trainable": 48037} +{"epoch": 0.4140625, "grad_norm": 0.211869016289711, "learning_rate": 9.413937424292791e-05, "loss": 0.009863297455012798, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.00991, "step": 106, "tokens/total": 3658080, "tokens/train_per_sec_per_gpu": 21.6, "tokens/trainable": 48491} +{"epoch": 0.41796875, "grad_norm": 0.6860823035240173, "learning_rate": 9.399531154980424e-05, "loss": 0.014444693922996521, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 1.01455, "step": 107, "tokens/total": 3694656, "tokens/train_per_sec_per_gpu": 24.14, "tokens/trainable": 48940} +{"epoch": 0.421875, "grad_norm": 0.4428965747356415, "learning_rate": 9.384962610259455e-05, "loss": 0.012264467775821686, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.01234, "step": 108, "tokens/total": 3729312, "tokens/train_per_sec_per_gpu": 24.2, "tokens/trainable": 49389} +{"epoch": 0.42578125, "grad_norm": 0.3491283059120178, "learning_rate": 9.370232396386494e-05, "loss": 0.006182459183037281, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.22, "memory/max_allocated (GiB)": 37.22, "ppl": 1.0062, "step": 109, "tokens/total": 3762400, "tokens/train_per_sec_per_gpu": 26.4, "tokens/trainable": 49838} +{"epoch": 0.4296875, "grad_norm": 0.33850032091140747, "learning_rate": 9.355341126345868e-05, "loss": 0.013678515329957008, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.01377, "step": 110, "tokens/total": 3797680, "tokens/train_per_sec_per_gpu": 25.47, "tokens/trainable": 50310} +{"epoch": 0.43359375, "grad_norm": 0.4869474470615387, "learning_rate": 9.340289419824107e-05, "loss": 0.014713555574417114, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.01482, "step": 111, "tokens/total": 3830000, "tokens/train_per_sec_per_gpu": 28.07, "tokens/trainable": 50755} +{"epoch": 0.4375, "grad_norm": 0.20179510116577148, "learning_rate": 9.325077903184159e-05, "loss": 0.004072415176779032, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00408, "step": 112, "tokens/total": 3864992, "tokens/train_per_sec_per_gpu": 23.91, "tokens/trainable": 51213} +{"epoch": 0.44140625, "grad_norm": 4.7394280433654785, "learning_rate": 9.30970720943932e-05, "loss": 0.007455363869667053, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.15, "memory/max_allocated (GiB)": 38.15, "ppl": 1.00748, "step": 113, "tokens/total": 3900000, "tokens/train_per_sec_per_gpu": 25.38, "tokens/trainable": 51660} +{"epoch": 0.4453125, "grad_norm": 0.40170741081237793, "learning_rate": 9.2941779782269e-05, "loss": 0.0050103492103517056, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00502, "step": 114, "tokens/total": 3933600, "tokens/train_per_sec_per_gpu": 25.45, "tokens/trainable": 52133} +{"epoch": 0.44921875, "grad_norm": 0.3549549877643585, "learning_rate": 9.278490855781596e-05, "loss": 0.010226115584373474, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.01028, "step": 115, "tokens/total": 3966032, "tokens/train_per_sec_per_gpu": 31.02, "tokens/trainable": 52580} +{"epoch": 0.453125, "grad_norm": 0.7301314473152161, "learning_rate": 9.262646494908604e-05, "loss": 0.010027364827692509, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.01008, "step": 116, "tokens/total": 4004688, "tokens/train_per_sec_per_gpu": 23.63, "tokens/trainable": 53033} +{"epoch": 0.45703125, "grad_norm": 0.908897340297699, "learning_rate": 9.246645554956457e-05, "loss": 0.019147710874676704, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.01933, "step": 117, "tokens/total": 4039712, "tokens/train_per_sec_per_gpu": 26.67, "tokens/trainable": 53517} +{"epoch": 0.4609375, "grad_norm": 0.3675004839897156, "learning_rate": 9.230488701789578e-05, "loss": 0.013036061078310013, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.01312, "step": 118, "tokens/total": 4076768, "tokens/train_per_sec_per_gpu": 24.38, "tokens/trainable": 53967} +{"epoch": 0.46484375, "grad_norm": 0.6683651804924011, "learning_rate": 9.214176607760577e-05, "loss": 0.026061300188302994, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.0264, "step": 119, "tokens/total": 4115024, "tokens/train_per_sec_per_gpu": 24.09, "tokens/trainable": 54430} +{"epoch": 0.46875, "grad_norm": 0.6811311841011047, "learning_rate": 9.197709951682268e-05, "loss": 0.04072137549519539, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.04156, "step": 120, "tokens/total": 4147456, "tokens/train_per_sec_per_gpu": 28.99, "tokens/trainable": 54896} +{"epoch": 0.47265625, "grad_norm": 1.3250412940979004, "learning_rate": 9.181089418799428e-05, "loss": 0.017857402563095093, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.01802, "step": 121, "tokens/total": 4183920, "tokens/train_per_sec_per_gpu": 27.28, "tokens/trainable": 55379} +{"epoch": 0.4765625, "grad_norm": 0.1986573487520218, "learning_rate": 9.164315700760271e-05, "loss": 0.008352738805115223, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.00839, "step": 122, "tokens/total": 4219600, "tokens/train_per_sec_per_gpu": 23.26, "tokens/trainable": 55803} +{"epoch": 0.48046875, "grad_norm": 0.5522497892379761, "learning_rate": 9.147389495587671e-05, "loss": 0.02127278596162796, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.0215, "step": 123, "tokens/total": 4251008, "tokens/train_per_sec_per_gpu": 30.89, "tokens/trainable": 56249} +{"epoch": 0.484375, "grad_norm": 0.32446053624153137, "learning_rate": 9.130311507650116e-05, "loss": 0.019945286214351654, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.02015, "step": 124, "tokens/total": 4283392, "tokens/train_per_sec_per_gpu": 26.87, "tokens/trainable": 56713} +{"epoch": 0.48828125, "grad_norm": 0.6513152122497559, "learning_rate": 9.113082447632394e-05, "loss": 0.010412310250103474, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.01047, "step": 125, "tokens/total": 4315456, "tokens/train_per_sec_per_gpu": 32.79, "tokens/trainable": 57196} +{"epoch": 0.4921875, "grad_norm": 0.30693721771240234, "learning_rate": 9.09570303250602e-05, "loss": 0.011989826336503029, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.01206, "step": 126, "tokens/total": 4347472, "tokens/train_per_sec_per_gpu": 30.36, "tokens/trainable": 57680} +{"epoch": 0.49609375, "grad_norm": 0.33333033323287964, "learning_rate": 9.078173985499394e-05, "loss": 0.014531591907143593, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 35.95, "memory/max_allocated (GiB)": 35.95, "ppl": 1.01464, "step": 127, "tokens/total": 4377872, "tokens/train_per_sec_per_gpu": 28.18, "tokens/trainable": 58110} +{"epoch": 0.5, "grad_norm": 0.47639527916908264, "learning_rate": 9.060496036067713e-05, "loss": 0.022458896040916443, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.02271, "step": 128, "tokens/total": 4415376, "tokens/train_per_sec_per_gpu": 22.23, "tokens/trainable": 58534} +{"epoch": 0.50390625, "grad_norm": 0.36746591329574585, "learning_rate": 9.042669919862615e-05, "loss": 0.01857452467083931, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.77, "memory/max_allocated (GiB)": 36.77, "ppl": 1.01875, "step": 129, "tokens/total": 4447264, "tokens/train_per_sec_per_gpu": 29.67, "tokens/trainable": 58998} +{"epoch": 0.5078125, "grad_norm": 0.4408619701862335, "learning_rate": 9.024696378701557e-05, "loss": 0.019152646884322166, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 39.48, "memory/max_allocated (GiB)": 39.48, "ppl": 1.01934, "step": 130, "tokens/total": 4482944, "tokens/train_per_sec_per_gpu": 23.53, "tokens/trainable": 59448} +{"epoch": 0.51171875, "grad_norm": 0.33010199666023254, "learning_rate": 9.006576160536948e-05, "loss": 0.01196005754172802, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.01203, "step": 131, "tokens/total": 4517888, "tokens/train_per_sec_per_gpu": 24.07, "tokens/trainable": 59906} +{"epoch": 0.515625, "grad_norm": 0.510373592376709, "learning_rate": 8.988310019425035e-05, "loss": 0.02370487153530121, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.02399, "step": 132, "tokens/total": 4555392, "tokens/train_per_sec_per_gpu": 23.89, "tokens/trainable": 60350} +{"epoch": 0.51953125, "grad_norm": 0.15557745099067688, "learning_rate": 8.969898715494506e-05, "loss": 0.0045677367597818375, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00458, "step": 133, "tokens/total": 4592528, "tokens/train_per_sec_per_gpu": 25.46, "tokens/trainable": 60831} +{"epoch": 0.5234375, "grad_norm": 0.6479173898696899, "learning_rate": 8.951343014914869e-05, "loss": 0.015230223536491394, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.01535, "step": 134, "tokens/total": 4629104, "tokens/train_per_sec_per_gpu": 24.15, "tokens/trainable": 61305} +{"epoch": 0.52734375, "grad_norm": 0.1896038055419922, "learning_rate": 8.932643689864568e-05, "loss": 0.0037470152601599693, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00375, "step": 135, "tokens/total": 4665856, "tokens/train_per_sec_per_gpu": 26.99, "tokens/trainable": 61792} +{"epoch": 0.53125, "grad_norm": 0.10701708495616913, "learning_rate": 8.913801518498845e-05, "loss": 0.0019145929254591465, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.00192, "step": 136, "tokens/total": 4704688, "tokens/train_per_sec_per_gpu": 21.11, "tokens/trainable": 62253} +{"epoch": 0.53515625, "grad_norm": 0.46454036235809326, "learning_rate": 8.894817284917364e-05, "loss": 0.010814795270562172, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.01087, "step": 137, "tokens/total": 4741472, "tokens/train_per_sec_per_gpu": 22.55, "tokens/trainable": 62707} +{"epoch": 0.5390625, "grad_norm": 0.16165228188037872, "learning_rate": 8.875691779131569e-05, "loss": 0.002373158698901534, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00238, "step": 138, "tokens/total": 4778752, "tokens/train_per_sec_per_gpu": 20.56, "tokens/trainable": 63145} +{"epoch": 0.54296875, "grad_norm": 0.5412599444389343, "learning_rate": 8.856425797031829e-05, "loss": 0.01593443565070629, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.01606, "step": 139, "tokens/total": 4813904, "tokens/train_per_sec_per_gpu": 25.52, "tokens/trainable": 63587} +{"epoch": 0.546875, "grad_norm": 0.6622636914253235, "learning_rate": 8.837020140354295e-05, "loss": 0.014176330529153347, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.01428, "step": 140, "tokens/total": 4850272, "tokens/train_per_sec_per_gpu": 26.59, "tokens/trainable": 64052} +{"epoch": 0.55078125, "grad_norm": 0.17899377644062042, "learning_rate": 8.817475616647554e-05, "loss": 0.002431028988212347, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.00243, "step": 141, "tokens/total": 4884928, "tokens/train_per_sec_per_gpu": 26.29, "tokens/trainable": 64526} +{"epoch": 0.5546875, "grad_norm": 0.07539641112089157, "learning_rate": 8.797793039239017e-05, "loss": 0.0019884135108441114, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00199, "step": 142, "tokens/total": 4919984, "tokens/train_per_sec_per_gpu": 26.66, "tokens/trainable": 64983} +{"epoch": 0.55859375, "grad_norm": 0.3270156681537628, "learning_rate": 8.777973227201069e-05, "loss": 0.008577108383178711, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.00861, "step": 143, "tokens/total": 4956304, "tokens/train_per_sec_per_gpu": 27.53, "tokens/trainable": 65492} +{"epoch": 0.5625, "grad_norm": 0.35661759972572327, "learning_rate": 8.758017005316988e-05, "loss": 0.01176381018012762, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.01183, "step": 144, "tokens/total": 4991696, "tokens/train_per_sec_per_gpu": 26.35, "tokens/trainable": 65925} +{"epoch": 0.56640625, "grad_norm": 0.9008830785751343, "learning_rate": 8.737925204046629e-05, "loss": 0.019492048770189285, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.01968, "step": 145, "tokens/total": 5026480, "tokens/train_per_sec_per_gpu": 23.76, "tokens/trainable": 66383} +{"epoch": 0.5703125, "grad_norm": 16.831911087036133, "learning_rate": 8.717698659491851e-05, "loss": 0.013177153654396534, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.01326, "step": 146, "tokens/total": 5063568, "tokens/train_per_sec_per_gpu": 22.17, "tokens/trainable": 66840} +{"epoch": 0.57421875, "grad_norm": 0.2673681676387787, "learning_rate": 8.697338213361735e-05, "loss": 0.011786667630076408, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.01186, "step": 147, "tokens/total": 5099616, "tokens/train_per_sec_per_gpu": 23.15, "tokens/trainable": 67297} +{"epoch": 0.578125, "grad_norm": 0.22710397839546204, "learning_rate": 8.676844712937552e-05, "loss": 0.0092246038839221, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.39, "memory/max_allocated (GiB)": 39.39, "ppl": 1.00927, "step": 148, "tokens/total": 5139360, "tokens/train_per_sec_per_gpu": 24.54, "tokens/trainable": 67769} +{"epoch": 0.58203125, "grad_norm": 0.5518885254859924, "learning_rate": 8.656219011037509e-05, "loss": 0.020023507997393608, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.02023, "step": 149, "tokens/total": 5176400, "tokens/train_per_sec_per_gpu": 23.27, "tokens/trainable": 68257} +{"epoch": 0.5859375, "grad_norm": 0.3226986229419708, "learning_rate": 8.63546196598125e-05, "loss": 0.01523870974779129, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.01536, "step": 150, "tokens/total": 5213632, "tokens/train_per_sec_per_gpu": 24.81, "tokens/trainable": 68706} +{"epoch": 0.58984375, "grad_norm": 0.24562008678913116, "learning_rate": 8.614574441554145e-05, "loss": 0.011890917085111141, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.13, "memory/max_allocated (GiB)": 37.13, "ppl": 1.01196, "step": 151, "tokens/total": 5246736, "tokens/train_per_sec_per_gpu": 27.0, "tokens/trainable": 69131} +{"epoch": 0.59375, "grad_norm": 0.2072298526763916, "learning_rate": 8.593557306971349e-05, "loss": 0.00860229879617691, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.44, "memory/max_allocated (GiB)": 37.44, "ppl": 1.00864, "step": 152, "tokens/total": 5279712, "tokens/train_per_sec_per_gpu": 25.6, "tokens/trainable": 69586} +{"epoch": 0.59765625, "grad_norm": 0.3013180196285248, "learning_rate": 8.572411436841618e-05, "loss": 0.009439457207918167, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00948, "step": 153, "tokens/total": 5313424, "tokens/train_per_sec_per_gpu": 26.16, "tokens/trainable": 70061} +{"epoch": 0.6015625, "grad_norm": 0.44633281230926514, "learning_rate": 8.551137711130922e-05, "loss": 0.013261553831398487, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.2, "memory/max_allocated (GiB)": 38.2, "ppl": 1.01335, "step": 154, "tokens/total": 5344816, "tokens/train_per_sec_per_gpu": 23.56, "tokens/trainable": 70467} +{"epoch": 0.60546875, "grad_norm": 0.718798816204071, "learning_rate": 8.529737015125824e-05, "loss": 0.019037049263715744, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.01922, "step": 155, "tokens/total": 5379264, "tokens/train_per_sec_per_gpu": 25.46, "tokens/trainable": 70933} +{"epoch": 0.609375, "grad_norm": 0.3367597460746765, "learning_rate": 8.508210239396639e-05, "loss": 0.009978502988815308, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.01003, "step": 156, "tokens/total": 5416832, "tokens/train_per_sec_per_gpu": 23.87, "tokens/trainable": 71382} +{"epoch": 0.61328125, "grad_norm": 0.10785336792469025, "learning_rate": 8.486558279760375e-05, "loss": 0.002868914045393467, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 36.07, "memory/max_allocated (GiB)": 36.07, "ppl": 1.00287, "step": 157, "tokens/total": 5447968, "tokens/train_per_sec_per_gpu": 26.28, "tokens/trainable": 71808} +{"epoch": 0.6171875, "grad_norm": 0.3411007821559906, "learning_rate": 8.464782037243449e-05, "loss": 0.015112383291125298, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.01523, "step": 158, "tokens/total": 5480576, "tokens/train_per_sec_per_gpu": 27.02, "tokens/trainable": 72249} +{"epoch": 0.62109375, "grad_norm": 0.6993277072906494, "learning_rate": 8.442882418044202e-05, "loss": 0.020293373614549637, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.0205, "step": 159, "tokens/total": 5513200, "tokens/train_per_sec_per_gpu": 28.96, "tokens/trainable": 72726} +{"epoch": 0.625, "grad_norm": 1.157650351524353, "learning_rate": 8.420860333495179e-05, "loss": 0.015241540968418121, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.01536, "step": 160, "tokens/total": 5547120, "tokens/train_per_sec_per_gpu": 25.41, "tokens/trainable": 73148} +{"epoch": 0.62890625, "grad_norm": 0.17514345049858093, "learning_rate": 8.398716700025208e-05, "loss": 0.0066310940310359, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.00665, "step": 161, "tokens/total": 5580336, "tokens/train_per_sec_per_gpu": 25.36, "tokens/trainable": 73600} +{"epoch": 0.6328125, "grad_norm": 0.5107235908508301, "learning_rate": 8.376452439121266e-05, "loss": 0.023850224912166595, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 35.89, "memory/max_allocated (GiB)": 35.89, "ppl": 1.02414, "step": 162, "tokens/total": 5610736, "tokens/train_per_sec_per_gpu": 31.52, "tokens/trainable": 74068} +{"epoch": 0.63671875, "grad_norm": 0.42506080865859985, "learning_rate": 8.354068477290124e-05, "loss": 0.006762431934475899, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00679, "step": 163, "tokens/total": 5645040, "tokens/train_per_sec_per_gpu": 27.47, "tokens/trainable": 74527} +{"epoch": 0.640625, "grad_norm": 0.2025870382785797, "learning_rate": 8.331565746019807e-05, "loss": 0.009793732315301895, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.52, "memory/max_allocated (GiB)": 36.52, "ppl": 1.00984, "step": 164, "tokens/total": 5676944, "tokens/train_per_sec_per_gpu": 26.78, "tokens/trainable": 74992} +{"epoch": 0.64453125, "grad_norm": 0.5904419422149658, "learning_rate": 8.308945181740812e-05, "loss": 0.027807606384158134, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.0282, "step": 165, "tokens/total": 5711728, "tokens/train_per_sec_per_gpu": 26.69, "tokens/trainable": 75442} +{"epoch": 0.6484375, "grad_norm": 0.28479138016700745, "learning_rate": 8.286207725787153e-05, "loss": 0.013046175241470337, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.01313, "step": 166, "tokens/total": 5748656, "tokens/train_per_sec_per_gpu": 22.22, "tokens/trainable": 75887} +{"epoch": 0.65234375, "grad_norm": 0.33750802278518677, "learning_rate": 8.263354324357182e-05, "loss": 0.013440465554594994, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.01353, "step": 167, "tokens/total": 5783376, "tokens/train_per_sec_per_gpu": 26.18, "tokens/trainable": 76331} +{"epoch": 0.65625, "grad_norm": 0.20975492894649506, "learning_rate": 8.240385928474219e-05, "loss": 0.011305408552289009, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.01137, "step": 168, "tokens/total": 5816512, "tokens/train_per_sec_per_gpu": 26.78, "tokens/trainable": 76745} +{"epoch": 0.66015625, "grad_norm": 0.24664941430091858, "learning_rate": 8.217303493946967e-05, "loss": 0.005242205690592527, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.00526, "step": 169, "tokens/total": 5850944, "tokens/train_per_sec_per_gpu": 26.0, "tokens/trainable": 77201} +{"epoch": 0.6640625, "grad_norm": 0.3175298869609833, "learning_rate": 8.194107981329746e-05, "loss": 0.013662275858223438, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.01376, "step": 170, "tokens/total": 5888912, "tokens/train_per_sec_per_gpu": 23.16, "tokens/trainable": 77655} +{"epoch": 0.66796875, "grad_norm": 0.08662188798189163, "learning_rate": 8.170800355882518e-05, "loss": 0.0025598229840397835, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.24, "memory/max_allocated (GiB)": 37.24, "ppl": 1.00256, "step": 171, "tokens/total": 5923136, "tokens/train_per_sec_per_gpu": 30.19, "tokens/trainable": 78122} +{"epoch": 0.671875, "grad_norm": 0.21588662266731262, "learning_rate": 8.147381587530713e-05, "loss": 0.007100905291736126, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.00713, "step": 172, "tokens/total": 5956944, "tokens/train_per_sec_per_gpu": 26.35, "tokens/trainable": 78576} +{"epoch": 0.67578125, "grad_norm": 1.2228047847747803, "learning_rate": 8.123852650824877e-05, "loss": 0.01732386089861393, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.01747, "step": 173, "tokens/total": 5990976, "tokens/train_per_sec_per_gpu": 28.24, "tokens/trainable": 79059} +{"epoch": 0.6796875, "grad_norm": 0.3169249892234802, "learning_rate": 8.100214524900103e-05, "loss": 0.005294862203299999, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.52, "memory/max_allocated (GiB)": 38.52, "ppl": 1.00531, "step": 174, "tokens/total": 6026368, "tokens/train_per_sec_per_gpu": 22.07, "tokens/trainable": 79498} +{"epoch": 0.68359375, "grad_norm": 0.302591472864151, "learning_rate": 8.076468193435301e-05, "loss": 0.00957378651946783, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.00962, "step": 175, "tokens/total": 6057760, "tokens/train_per_sec_per_gpu": 25.81, "tokens/trainable": 79922} +{"epoch": 0.6875, "grad_norm": 0.4652998745441437, "learning_rate": 8.052614644612253e-05, "loss": 0.006306087598204613, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.00633, "step": 176, "tokens/total": 6095024, "tokens/train_per_sec_per_gpu": 23.16, "tokens/trainable": 80383} +{"epoch": 0.69140625, "grad_norm": 0.1733839213848114, "learning_rate": 8.028654871074489e-05, "loss": 0.0024349098093807697, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00244, "step": 177, "tokens/total": 6129712, "tokens/train_per_sec_per_gpu": 25.31, "tokens/trainable": 80824} +{"epoch": 0.6953125, "grad_norm": 0.3641454875469208, "learning_rate": 8.004589869885986e-05, "loss": 0.004389193840324879, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.49, "memory/max_allocated (GiB)": 36.49, "ppl": 1.0044, "step": 178, "tokens/total": 6161824, "tokens/train_per_sec_per_gpu": 26.57, "tokens/trainable": 81243} +{"epoch": 0.69921875, "grad_norm": 0.41737014055252075, "learning_rate": 7.980420642489674e-05, "loss": 0.006392576731741428, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.00641, "step": 179, "tokens/total": 6200464, "tokens/train_per_sec_per_gpu": 23.23, "tokens/trainable": 81716} +{"epoch": 0.703125, "grad_norm": 0.04346594959497452, "learning_rate": 7.95614819466576e-05, "loss": 0.0006800562259741127, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00068, "step": 180, "tokens/total": 6234416, "tokens/train_per_sec_per_gpu": 28.15, "tokens/trainable": 82181} +{"epoch": 0.70703125, "grad_norm": 0.3872646689414978, "learning_rate": 7.931773536489872e-05, "loss": 0.007123797666281462, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.00715, "step": 181, "tokens/total": 6268608, "tokens/train_per_sec_per_gpu": 24.74, "tokens/trainable": 82626} +{"epoch": 0.7109375, "grad_norm": 0.633206307888031, "learning_rate": 7.907297682291035e-05, "loss": 0.007785373367369175, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00782, "step": 182, "tokens/total": 6303792, "tokens/train_per_sec_per_gpu": 26.33, "tokens/trainable": 83089} +{"epoch": 0.71484375, "grad_norm": 0.2919631898403168, "learning_rate": 7.882721650609442e-05, "loss": 0.0038339048624038696, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.00384, "step": 183, "tokens/total": 6343664, "tokens/train_per_sec_per_gpu": 23.08, "tokens/trainable": 83590} +{"epoch": 0.71875, "grad_norm": 0.9775686860084534, "learning_rate": 7.85804646415409e-05, "loss": 0.023026982322335243, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.02329, "step": 184, "tokens/total": 6378848, "tokens/train_per_sec_per_gpu": 26.6, "tokens/trainable": 84046} +{"epoch": 0.72265625, "grad_norm": 0.2768585979938507, "learning_rate": 7.833273149760207e-05, "loss": 0.004973759409040213, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.00499, "step": 185, "tokens/total": 6409968, "tokens/train_per_sec_per_gpu": 28.7, "tokens/trainable": 84517} +{"epoch": 0.7265625, "grad_norm": 1.2301936149597168, "learning_rate": 7.808402738346527e-05, "loss": 0.037331826984882355, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.03804, "step": 186, "tokens/total": 6448320, "tokens/train_per_sec_per_gpu": 23.27, "tokens/trainable": 84974} +{"epoch": 0.73046875, "grad_norm": 3.0992183685302734, "learning_rate": 7.783436264872382e-05, "loss": 0.001793653005734086, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.0018, "step": 187, "tokens/total": 6485440, "tokens/train_per_sec_per_gpu": 24.65, "tokens/trainable": 85460} +{"epoch": 0.734375, "grad_norm": 0.25266605615615845, "learning_rate": 7.758374768294647e-05, "loss": 0.0073966700583696365, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.05, "memory/max_allocated (GiB)": 38.05, "ppl": 1.00742, "step": 188, "tokens/total": 6521216, "tokens/train_per_sec_per_gpu": 26.83, "tokens/trainable": 85939} +{"epoch": 0.73828125, "grad_norm": 0.42644086480140686, "learning_rate": 7.733219291524489e-05, "loss": 0.014870358631014824, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.59, "memory/max_allocated (GiB)": 37.59, "ppl": 1.01498, "step": 189, "tokens/total": 6554736, "tokens/train_per_sec_per_gpu": 28.11, "tokens/trainable": 86377} +{"epoch": 0.7421875, "grad_norm": 0.14380185306072235, "learning_rate": 7.707970881383977e-05, "loss": 0.0043508270755410194, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.57, "memory/max_allocated (GiB)": 36.57, "ppl": 1.00436, "step": 190, "tokens/total": 6586480, "tokens/train_per_sec_per_gpu": 28.91, "tokens/trainable": 86834} +{"epoch": 0.74609375, "grad_norm": 0.4712045192718506, "learning_rate": 7.682630588562518e-05, "loss": 0.02028321474790573, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.02049, "step": 191, "tokens/total": 6621536, "tokens/train_per_sec_per_gpu": 24.18, "tokens/trainable": 87265} +{"epoch": 0.75, "grad_norm": 0.24164679646492004, "learning_rate": 7.657199467573129e-05, "loss": 0.01121533289551735, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.01128, "step": 192, "tokens/total": 6655248, "tokens/train_per_sec_per_gpu": 27.13, "tokens/trainable": 87747} +{"epoch": 0.75390625, "grad_norm": 0.18258480727672577, "learning_rate": 7.631678576708561e-05, "loss": 0.006300302222371101, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00632, "step": 193, "tokens/total": 6688960, "tokens/train_per_sec_per_gpu": 28.79, "tokens/trainable": 88239} +{"epoch": 0.7578125, "grad_norm": 0.5802868604660034, "learning_rate": 7.606068977997255e-05, "loss": 0.0075871385633945465, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00762, "step": 194, "tokens/total": 6726256, "tokens/train_per_sec_per_gpu": 25.54, "tokens/trainable": 88718} +{"epoch": 0.76171875, "grad_norm": 0.18820367753505707, "learning_rate": 7.580371737159148e-05, "loss": 0.009327426552772522, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00937, "step": 195, "tokens/total": 6761984, "tokens/train_per_sec_per_gpu": 22.82, "tokens/trainable": 89129} +{"epoch": 0.765625, "grad_norm": 0.13384923338890076, "learning_rate": 7.554587923561324e-05, "loss": 0.003299079369753599, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.0033, "step": 196, "tokens/total": 6796640, "tokens/train_per_sec_per_gpu": 24.47, "tokens/trainable": 89567} +{"epoch": 0.76953125, "grad_norm": 0.7631242871284485, "learning_rate": 7.528718610173511e-05, "loss": 0.036178410053253174, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.48, "memory/max_allocated (GiB)": 37.48, "ppl": 1.03684, "step": 197, "tokens/total": 6831840, "tokens/train_per_sec_per_gpu": 22.69, "tokens/trainable": 89988} +{"epoch": 0.7734375, "grad_norm": 0.25774237513542175, "learning_rate": 7.502764873523431e-05, "loss": 0.005705534014850855, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.00572, "step": 198, "tokens/total": 6866128, "tokens/train_per_sec_per_gpu": 27.95, "tokens/trainable": 90434} +{"epoch": 0.77734375, "grad_norm": 0.1667272448539734, "learning_rate": 7.476727793652011e-05, "loss": 0.004820950794965029, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.71, "memory/max_allocated (GiB)": 36.71, "ppl": 1.00483, "step": 199, "tokens/total": 6898960, "tokens/train_per_sec_per_gpu": 27.93, "tokens/trainable": 90913} +{"epoch": 0.78125, "grad_norm": 0.2387460321187973, "learning_rate": 7.450608454068415e-05, "loss": 0.010626440867781639, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.01068, "step": 200, "tokens/total": 6933312, "tokens/train_per_sec_per_gpu": 26.49, "tokens/trainable": 91355} +{"epoch": 0.78515625, "grad_norm": 0.242015540599823, "learning_rate": 7.424407941704987e-05, "loss": 0.005965983495116234, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.00598, "step": 201, "tokens/total": 6970016, "tokens/train_per_sec_per_gpu": 27.12, "tokens/trainable": 91856} +{"epoch": 0.7890625, "grad_norm": 0.5979886054992676, "learning_rate": 7.398127346871986e-05, "loss": 0.03933599218726158, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.04012, "step": 202, "tokens/total": 7006736, "tokens/train_per_sec_per_gpu": 24.24, "tokens/trainable": 92311} +{"epoch": 0.79296875, "grad_norm": 0.15827840566635132, "learning_rate": 7.371767763212238e-05, "loss": 0.004029646050184965, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.00404, "step": 203, "tokens/total": 7041728, "tokens/train_per_sec_per_gpu": 26.37, "tokens/trainable": 92764} +{"epoch": 0.796875, "grad_norm": 0.19023020565509796, "learning_rate": 7.345330287655617e-05, "loss": 0.00660322979092598, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.00663, "step": 204, "tokens/total": 7079792, "tokens/train_per_sec_per_gpu": 22.58, "tokens/trainable": 93227} +{"epoch": 0.80078125, "grad_norm": 0.3661016821861267, "learning_rate": 7.31881602037339e-05, "loss": 0.015810778364539146, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.01594, "step": 205, "tokens/total": 7111824, "tokens/train_per_sec_per_gpu": 27.28, "tokens/trainable": 93675} +{"epoch": 0.8046875, "grad_norm": 0.3035740256309509, "learning_rate": 7.29222606473245e-05, "loss": 0.01406974345445633, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.01417, "step": 206, "tokens/total": 7144896, "tokens/train_per_sec_per_gpu": 26.31, "tokens/trainable": 94120} +{"epoch": 0.80859375, "grad_norm": 0.1897566169500351, "learning_rate": 7.265561527249383e-05, "loss": 0.004847434349358082, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.00486, "step": 207, "tokens/total": 7178944, "tokens/train_per_sec_per_gpu": 27.56, "tokens/trainable": 94557} +{"epoch": 0.8125, "grad_norm": 0.47569239139556885, "learning_rate": 7.238823517544436e-05, "loss": 0.011272573843598366, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.01134, "step": 208, "tokens/total": 7215840, "tokens/train_per_sec_per_gpu": 28.89, "tokens/trainable": 95035} +{"epoch": 0.81640625, "grad_norm": 0.059088438749313354, "learning_rate": 7.212013148295333e-05, "loss": 0.0014719890896230936, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.00147, "step": 209, "tokens/total": 7249952, "tokens/train_per_sec_per_gpu": 29.57, "tokens/trainable": 95517} +{"epoch": 0.8203125, "grad_norm": 0.14582909643650055, "learning_rate": 7.185131535190975e-05, "loss": 0.00421788077801466, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00423, "step": 210, "tokens/total": 7282352, "tokens/train_per_sec_per_gpu": 29.77, "tokens/trainable": 96026} +{"epoch": 0.82421875, "grad_norm": 0.1471947282552719, "learning_rate": 7.158179796885005e-05, "loss": 0.0028823488391935825, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.00289, "step": 211, "tokens/total": 7316768, "tokens/train_per_sec_per_gpu": 27.34, "tokens/trainable": 96477} +{"epoch": 0.828125, "grad_norm": 0.15960252285003662, "learning_rate": 7.131159054949273e-05, "loss": 0.004607381299138069, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00462, "step": 212, "tokens/total": 7355168, "tokens/train_per_sec_per_gpu": 22.18, "tokens/trainable": 96951} +{"epoch": 0.83203125, "grad_norm": 0.31501564383506775, "learning_rate": 7.104070433827139e-05, "loss": 0.008653029799461365, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00869, "step": 213, "tokens/total": 7390720, "tokens/train_per_sec_per_gpu": 22.8, "tokens/trainable": 97350} +{"epoch": 0.8359375, "grad_norm": 0.30138933658599854, "learning_rate": 7.076915060786705e-05, "loss": 0.0029394947923719883, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.96, "memory/max_allocated (GiB)": 37.96, "ppl": 1.00294, "step": 214, "tokens/total": 7423616, "tokens/train_per_sec_per_gpu": 24.06, "tokens/trainable": 97792} +{"epoch": 0.83984375, "grad_norm": 0.12156199663877487, "learning_rate": 7.049694065873882e-05, "loss": 0.0027204558718949556, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.00272, "step": 215, "tokens/total": 7458352, "tokens/train_per_sec_per_gpu": 27.76, "tokens/trainable": 98257} +{"epoch": 0.84375, "grad_norm": 1.138650894165039, "learning_rate": 7.022408581865382e-05, "loss": 0.026597484946250916, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.69, "memory/max_allocated (GiB)": 38.69, "ppl": 1.02695, "step": 216, "tokens/total": 7492672, "tokens/train_per_sec_per_gpu": 25.27, "tokens/trainable": 98731} +{"epoch": 0.84765625, "grad_norm": 0.01548983994871378, "learning_rate": 6.99505974422157e-05, "loss": 0.00033570139203220606, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.00034, "step": 217, "tokens/total": 7526832, "tokens/train_per_sec_per_gpu": 28.21, "tokens/trainable": 99212} +{"epoch": 0.8515625, "grad_norm": 0.035592082887887955, "learning_rate": 6.967648691039213e-05, "loss": 0.0006258189678192139, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.73, "memory/max_allocated (GiB)": 36.73, "ppl": 1.00063, "step": 218, "tokens/total": 7558944, "tokens/train_per_sec_per_gpu": 27.7, "tokens/trainable": 99654} +{"epoch": 0.85546875, "grad_norm": 0.21090926229953766, "learning_rate": 6.940176563004123e-05, "loss": 0.004712432622909546, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00472, "step": 219, "tokens/total": 7593120, "tokens/train_per_sec_per_gpu": 25.77, "tokens/trainable": 100086} +{"epoch": 0.859375, "grad_norm": 0.16241058707237244, "learning_rate": 6.912644503343682e-05, "loss": 0.003482309402897954, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.00349, "step": 220, "tokens/total": 7630592, "tokens/train_per_sec_per_gpu": 23.3, "tokens/trainable": 100524} +{"epoch": 0.86328125, "grad_norm": 0.391515851020813, "learning_rate": 6.885053657779273e-05, "loss": 0.00863618217408657, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.00867, "step": 221, "tokens/total": 7662384, "tokens/train_per_sec_per_gpu": 28.2, "tokens/trainable": 100996} +{"epoch": 0.8671875, "grad_norm": 0.11227903515100479, "learning_rate": 6.857405174478604e-05, "loss": 0.002021458465605974, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.00202, "step": 222, "tokens/total": 7697152, "tokens/train_per_sec_per_gpu": 24.19, "tokens/trainable": 101449} +{"epoch": 0.87109375, "grad_norm": 0.24685652554035187, "learning_rate": 6.82970020400792e-05, "loss": 0.02251325733959675, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.02277, "step": 223, "tokens/total": 7734624, "tokens/train_per_sec_per_gpu": 23.0, "tokens/trainable": 101905} +{"epoch": 0.875, "grad_norm": 0.22541387379169464, "learning_rate": 6.801939899284132e-05, "loss": 0.005712810903787613, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.34, "memory/max_allocated (GiB)": 38.34, "ppl": 1.00573, "step": 224, "tokens/total": 7770848, "tokens/train_per_sec_per_gpu": 26.36, "tokens/trainable": 102411} +{"epoch": 0.87890625, "grad_norm": 0.07966572046279907, "learning_rate": 6.774125415526827e-05, "loss": 0.0017229014774784446, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00172, "step": 225, "tokens/total": 7803904, "tokens/train_per_sec_per_gpu": 27.59, "tokens/trainable": 102882} +{"epoch": 0.8828125, "grad_norm": 0.05913680046796799, "learning_rate": 6.746257910210214e-05, "loss": 0.0010517891496419907, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.12, "memory/max_allocated (GiB)": 37.12, "ppl": 1.00105, "step": 226, "tokens/total": 7837088, "tokens/train_per_sec_per_gpu": 27.82, "tokens/trainable": 103332} +{"epoch": 0.88671875, "grad_norm": 0.24942505359649658, "learning_rate": 6.718338543014937e-05, "loss": 0.005868477746844292, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 35.75, "memory/max_allocated (GiB)": 35.75, "ppl": 1.00589, "step": 227, "tokens/total": 7867360, "tokens/train_per_sec_per_gpu": 31.68, "tokens/trainable": 103790} +{"epoch": 0.890625, "grad_norm": 0.14161013066768646, "learning_rate": 6.69036847577983e-05, "loss": 0.0027201289776712656, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.00272, "step": 228, "tokens/total": 7901280, "tokens/train_per_sec_per_gpu": 26.54, "tokens/trainable": 104246} +{"epoch": 0.89453125, "grad_norm": 0.579100489616394, "learning_rate": 6.662348872453553e-05, "loss": 0.011869534850120544, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.33, "memory/max_allocated (GiB)": 38.33, "ppl": 1.01194, "step": 229, "tokens/total": 7937536, "tokens/train_per_sec_per_gpu": 27.32, "tokens/trainable": 104707} +{"epoch": 0.8984375, "grad_norm": 0.0546540804207325, "learning_rate": 6.63428089904618e-05, "loss": 0.0010581850074231625, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.00106, "step": 230, "tokens/total": 7976544, "tokens/train_per_sec_per_gpu": 24.72, "tokens/trainable": 105167} +{"epoch": 0.90234375, "grad_norm": 0.05752357468008995, "learning_rate": 6.60616572358065e-05, "loss": 0.0011189753422513604, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.00112, "step": 231, "tokens/total": 8011008, "tokens/train_per_sec_per_gpu": 25.35, "tokens/trainable": 105576} +{"epoch": 0.90625, "grad_norm": 0.1918521225452423, "learning_rate": 6.578004516044172e-05, "loss": 0.00306597538292408, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00307, "step": 232, "tokens/total": 8046784, "tokens/train_per_sec_per_gpu": 27.34, "tokens/trainable": 106043} +{"epoch": 0.91015625, "grad_norm": 0.310410737991333, "learning_rate": 6.549798448339548e-05, "loss": 0.010170400142669678, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.59, "memory/max_allocated (GiB)": 38.59, "ppl": 1.01022, "step": 233, "tokens/total": 8084048, "tokens/train_per_sec_per_gpu": 24.83, "tokens/trainable": 106506} +{"epoch": 0.9140625, "grad_norm": 0.12892243266105652, "learning_rate": 6.521548694236384e-05, "loss": 0.0011731931008398533, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00117, "step": 234, "tokens/total": 8121248, "tokens/train_per_sec_per_gpu": 24.0, "tokens/trainable": 106951} +{"epoch": 0.91796875, "grad_norm": 0.3455739915370941, "learning_rate": 6.493256429322259e-05, "loss": 0.005899685434997082, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.00592, "step": 235, "tokens/total": 8157504, "tokens/train_per_sec_per_gpu": 22.06, "tokens/trainable": 107382} +{"epoch": 0.921875, "grad_norm": 0.2981884181499481, "learning_rate": 6.464922830953799e-05, "loss": 0.008932916447520256, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.00897, "step": 236, "tokens/total": 8196368, "tokens/train_per_sec_per_gpu": 22.11, "tokens/trainable": 107814} +{"epoch": 0.92578125, "grad_norm": 0.6639502048492432, "learning_rate": 6.436549078207688e-05, "loss": 0.03507738932967186, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.0357, "step": 237, "tokens/total": 8229376, "tokens/train_per_sec_per_gpu": 28.53, "tokens/trainable": 108274} +{"epoch": 0.9296875, "grad_norm": 0.3505960702896118, "learning_rate": 6.408136351831592e-05, "loss": 0.016423508524894714, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.01656, "step": 238, "tokens/total": 8261792, "tokens/train_per_sec_per_gpu": 23.9, "tokens/trainable": 108714} +{"epoch": 0.93359375, "grad_norm": 1.3331096172332764, "learning_rate": 6.379685834195036e-05, "loss": 0.0016873989952728152, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00169, "step": 239, "tokens/total": 8301216, "tokens/train_per_sec_per_gpu": 24.29, "tokens/trainable": 109220} +{"epoch": 0.9375, "grad_norm": 0.041115742176771164, "learning_rate": 6.351198709240186e-05, "loss": 0.0011144905583932996, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.00112, "step": 240, "tokens/total": 8341152, "tokens/train_per_sec_per_gpu": 20.81, "tokens/trainable": 109672} +{"epoch": 0.94140625, "grad_norm": 0.13714854419231415, "learning_rate": 6.32267616243259e-05, "loss": 0.004445014987140894, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.00445, "step": 241, "tokens/total": 8373952, "tokens/train_per_sec_per_gpu": 28.42, "tokens/trainable": 110135} +{"epoch": 0.9453125, "grad_norm": 0.1349315345287323, "learning_rate": 6.294119380711849e-05, "loss": 0.004097213968634605, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.39, "memory/max_allocated (GiB)": 38.39, "ppl": 1.00411, "step": 242, "tokens/total": 8408240, "tokens/train_per_sec_per_gpu": 23.71, "tokens/trainable": 110563} +{"epoch": 0.94921875, "grad_norm": 0.3028312921524048, "learning_rate": 6.265529552442209e-05, "loss": 0.006307918578386307, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.00633, "step": 243, "tokens/total": 8443344, "tokens/train_per_sec_per_gpu": 27.73, "tokens/trainable": 111049} +{"epoch": 0.953125, "grad_norm": 0.08762618154287338, "learning_rate": 6.236907867363127e-05, "loss": 0.0015822830609977245, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.17, "memory/max_allocated (GiB)": 38.17, "ppl": 1.00158, "step": 244, "tokens/total": 8477920, "tokens/train_per_sec_per_gpu": 26.36, "tokens/trainable": 111495} +{"epoch": 0.95703125, "grad_norm": 0.1582319140434265, "learning_rate": 6.208255516539749e-05, "loss": 0.0030931478831917048, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.0031, "step": 245, "tokens/total": 8510240, "tokens/train_per_sec_per_gpu": 26.32, "tokens/trainable": 111968} +{"epoch": 0.9609375, "grad_norm": 0.1551622450351715, "learning_rate": 6.179573692313344e-05, "loss": 0.0026908915024250746, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.8, "memory/max_allocated (GiB)": 38.8, "ppl": 1.00269, "step": 246, "tokens/total": 8547024, "tokens/train_per_sec_per_gpu": 25.79, "tokens/trainable": 112416} +{"epoch": 0.96484375, "grad_norm": 0.19648294150829315, "learning_rate": 6.150863588251694e-05, "loss": 0.003662416711449623, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.00367, "step": 247, "tokens/total": 8586912, "tokens/train_per_sec_per_gpu": 22.43, "tokens/trainable": 112882} +{"epoch": 0.96875, "grad_norm": 0.08677448332309723, "learning_rate": 6.122126399099419e-05, "loss": 0.002150431741029024, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.00215, "step": 248, "tokens/total": 8626304, "tokens/train_per_sec_per_gpu": 26.57, "tokens/trainable": 113390} +{"epoch": 0.97265625, "grad_norm": 0.5242704749107361, "learning_rate": 6.0933633207282615e-05, "loss": 0.022058788686990738, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.0223, "step": 249, "tokens/total": 8662736, "tokens/train_per_sec_per_gpu": 27.09, "tokens/trainable": 113904} +{"epoch": 0.9765625, "grad_norm": 0.16802139580249786, "learning_rate": 6.064575550087316e-05, "loss": 0.004442964214831591, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00445, "step": 250, "tokens/total": 8699376, "tokens/train_per_sec_per_gpu": 24.45, "tokens/trainable": 114351} +{"epoch": 0.98046875, "grad_norm": 0.21316410601139069, "learning_rate": 6.0357642851532245e-05, "loss": 0.0025420039892196655, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00255, "step": 251, "tokens/total": 8736064, "tokens/train_per_sec_per_gpu": 25.49, "tokens/trainable": 114842} +{"epoch": 0.984375, "grad_norm": 0.38871368765830994, "learning_rate": 6.0069307248803294e-05, "loss": 0.010527780279517174, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.01058, "step": 252, "tokens/total": 8771424, "tokens/train_per_sec_per_gpu": 21.74, "tokens/trainable": 115263} +{"epoch": 0.98828125, "grad_norm": 0.04209626838564873, "learning_rate": 5.9780760691507635e-05, "loss": 0.0011491028126329184, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.33, "memory/max_allocated (GiB)": 36.33, "ppl": 1.00115, "step": 253, "tokens/total": 8803168, "tokens/train_per_sec_per_gpu": 27.43, "tokens/trainable": 115703} +{"epoch": 0.9921875, "grad_norm": 0.15676286816596985, "learning_rate": 5.9492015187245334e-05, "loss": 0.0024386271834373474, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00244, "step": 254, "tokens/total": 8840912, "tokens/train_per_sec_per_gpu": 23.06, "tokens/trainable": 116157} +{"epoch": 0.99609375, "grad_norm": 0.05347863957285881, "learning_rate": 5.920308275189541e-05, "loss": 0.0009471835801377892, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.00095, "step": 255, "tokens/total": 8873184, "tokens/train_per_sec_per_gpu": 26.4, "tokens/trainable": 116567} +{"epoch": 1.0, "grad_norm": 0.16139088571071625, "learning_rate": 5.8913975409115874e-05, "loss": 0.0029837859328836203, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.00299, "step": 256, "tokens/total": 8906544, "tokens/train_per_sec_per_gpu": 25.13, "tokens/trainable": 117030} +{"epoch": 1.00390625, "grad_norm": 0.07646998763084412, "learning_rate": 5.8624705189843395e-05, "loss": 0.0011338199255988002, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 1.00113, "step": 257, "tokens/total": 8943760, "tokens/train_per_sec_per_gpu": 24.23, "tokens/trainable": 117517} +{"epoch": 1.0078125, "grad_norm": 0.404562383890152, "learning_rate": 5.833528413179249e-05, "loss": 0.006999108474701643, "memory/device_reserved (GiB)": 42.25, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00702, "step": 258, "tokens/total": 8980256, "tokens/train_per_sec_per_gpu": 27.52, "tokens/trainable": 118015} +{"epoch": 1.01171875, "grad_norm": 0.3460175096988678, "learning_rate": 5.80457242789548e-05, "loss": 0.0065603237599134445, "memory/device_reserved (GiB)": 42.31, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.00658, "step": 259, "tokens/total": 9020240, "tokens/train_per_sec_per_gpu": 23.58, "tokens/trainable": 118520} +{"epoch": 1.015625, "grad_norm": 0.06926977634429932, "learning_rate": 5.77560376810977e-05, "loss": 0.0009659200441092253, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 1.00097, "step": 260, "tokens/total": 9053888, "tokens/train_per_sec_per_gpu": 28.89, "tokens/trainable": 118992} +{"epoch": 1.01953125, "grad_norm": 0.014009769074618816, "learning_rate": 5.7466236393263005e-05, "loss": 0.00034640892408788204, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.00035, "step": 261, "tokens/total": 9088960, "tokens/train_per_sec_per_gpu": 25.23, "tokens/trainable": 119438} +{"epoch": 1.0234375, "grad_norm": 0.03388168290257454, "learning_rate": 5.717633247526522e-05, "loss": 0.0007033420260995626, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.0007, "step": 262, "tokens/total": 9120256, "tokens/train_per_sec_per_gpu": 27.46, "tokens/trainable": 119881} +{"epoch": 1.02734375, "grad_norm": 0.16667062044143677, "learning_rate": 5.688633799118971e-05, "loss": 0.002075103111565113, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.00208, "step": 263, "tokens/total": 9152336, "tokens/train_per_sec_per_gpu": 23.0, "tokens/trainable": 120285} +{"epoch": 1.03125, "grad_norm": 0.24030794203281403, "learning_rate": 5.659626500889066e-05, "loss": 0.0035450798459351063, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00355, "step": 264, "tokens/total": 9186208, "tokens/train_per_sec_per_gpu": 25.46, "tokens/trainable": 120755} +{"epoch": 1.03515625, "grad_norm": 0.49345332384109497, "learning_rate": 5.6306125599488905e-05, "loss": 0.0073738363571465015, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.0074, "step": 265, "tokens/total": 9216768, "tokens/train_per_sec_per_gpu": 31.77, "tokens/trainable": 121194} +{"epoch": 1.0390625, "grad_norm": 0.1514706313610077, "learning_rate": 5.601593183686955e-05, "loss": 0.0022699085529893637, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.69, "memory/max_allocated (GiB)": 38.69, "ppl": 1.00227, "step": 266, "tokens/total": 9254032, "tokens/train_per_sec_per_gpu": 25.7, "tokens/trainable": 121670} +{"epoch": 1.04296875, "grad_norm": 0.3479229509830475, "learning_rate": 5.572569579717961e-05, "loss": 0.007930947467684746, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.18, "memory/max_allocated (GiB)": 39.18, "ppl": 1.00796, "step": 267, "tokens/total": 9289552, "tokens/train_per_sec_per_gpu": 25.1, "tokens/trainable": 122142} +{"epoch": 1.046875, "grad_norm": 0.01526730041950941, "learning_rate": 5.543542955832538e-05, "loss": 0.0003014251124113798, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.0003, "step": 268, "tokens/total": 9329312, "tokens/train_per_sec_per_gpu": 21.46, "tokens/trainable": 122585} +{"epoch": 1.05078125, "grad_norm": 0.5016617178916931, "learning_rate": 5.514514519946986e-05, "loss": 0.007887444458901882, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.00792, "step": 269, "tokens/total": 9369136, "tokens/train_per_sec_per_gpu": 25.42, "tokens/trainable": 123091} +{"epoch": 1.0546875, "grad_norm": 0.06687033921480179, "learning_rate": 5.485485480053015e-05, "loss": 0.00044549195445142686, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.00045, "step": 270, "tokens/total": 9407360, "tokens/train_per_sec_per_gpu": 21.76, "tokens/trainable": 123505} +{"epoch": 1.05859375, "grad_norm": 0.08900901675224304, "learning_rate": 5.4564570441674645e-05, "loss": 0.0012205445673316717, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.92, "memory/max_allocated (GiB)": 36.92, "ppl": 1.00122, "step": 271, "tokens/total": 9440624, "tokens/train_per_sec_per_gpu": 26.2, "tokens/trainable": 123953} +{"epoch": 1.0625, "grad_norm": 0.3404615819454193, "learning_rate": 5.42743042028204e-05, "loss": 0.0013117323396727443, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.00131, "step": 272, "tokens/total": 9477120, "tokens/train_per_sec_per_gpu": 20.73, "tokens/trainable": 124400} +{"epoch": 1.06640625, "grad_norm": 0.2896651327610016, "learning_rate": 5.3984068163130464e-05, "loss": 0.0037451726384460926, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.09, "memory/max_allocated (GiB)": 37.09, "ppl": 1.00375, "step": 273, "tokens/total": 9510832, "tokens/train_per_sec_per_gpu": 27.84, "tokens/trainable": 124870} +{"epoch": 1.0703125, "grad_norm": 0.22625502943992615, "learning_rate": 5.369387440051111e-05, "loss": 0.0033841528929769993, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.00339, "step": 274, "tokens/total": 9544368, "tokens/train_per_sec_per_gpu": 25.19, "tokens/trainable": 125333} +{"epoch": 1.07421875, "grad_norm": 0.006691146641969681, "learning_rate": 5.340373499110935e-05, "loss": 0.00015867897309362888, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00016, "step": 275, "tokens/total": 9580240, "tokens/train_per_sec_per_gpu": 27.01, "tokens/trainable": 125834} +{"epoch": 1.078125, "grad_norm": 0.003772740252315998, "learning_rate": 5.3113662008810304e-05, "loss": 8.062998676905409e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00008, "step": 276, "tokens/total": 9620256, "tokens/train_per_sec_per_gpu": 24.72, "tokens/trainable": 126316} +{"epoch": 1.08203125, "grad_norm": 0.014610737562179565, "learning_rate": 5.282366752473479e-05, "loss": 0.00021398589888121933, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 1.00021, "step": 277, "tokens/total": 9657264, "tokens/train_per_sec_per_gpu": 25.06, "tokens/trainable": 126798} +{"epoch": 1.0859375, "grad_norm": 0.06725924462080002, "learning_rate": 5.2533763606737005e-05, "loss": 0.0009077081340365112, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00091, "step": 278, "tokens/total": 9696784, "tokens/train_per_sec_per_gpu": 19.16, "tokens/trainable": 127223} +{"epoch": 1.08984375, "grad_norm": 0.010716250166296959, "learning_rate": 5.224396231890232e-05, "loss": 0.0001786245557013899, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.00018, "step": 279, "tokens/total": 9728256, "tokens/train_per_sec_per_gpu": 29.76, "tokens/trainable": 127672} +{"epoch": 1.09375, "grad_norm": 0.26526644825935364, "learning_rate": 5.195427572104522e-05, "loss": 0.007901082746684551, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00793, "step": 280, "tokens/total": 9762688, "tokens/train_per_sec_per_gpu": 24.45, "tokens/trainable": 128116} +{"epoch": 1.09765625, "grad_norm": 0.008325566537678242, "learning_rate": 5.166471586820751e-05, "loss": 0.00012661440996453166, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.35, "memory/max_allocated (GiB)": 39.35, "ppl": 1.00013, "step": 281, "tokens/total": 9799296, "tokens/train_per_sec_per_gpu": 23.57, "tokens/trainable": 128589} +{"epoch": 1.1015625, "grad_norm": 0.36717870831489563, "learning_rate": 5.1375294810156615e-05, "loss": 0.013956684619188309, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.01405, "step": 282, "tokens/total": 9834128, "tokens/train_per_sec_per_gpu": 25.66, "tokens/trainable": 129036} +{"epoch": 1.10546875, "grad_norm": 0.07567176222801208, "learning_rate": 5.1086024590884144e-05, "loss": 0.0005998878041282296, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.0006, "step": 283, "tokens/total": 9871536, "tokens/train_per_sec_per_gpu": 24.8, "tokens/trainable": 129485} +{"epoch": 1.109375, "grad_norm": 0.013953791931271553, "learning_rate": 5.079691724810461e-05, "loss": 0.0002222712355433032, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.33, "memory/max_allocated (GiB)": 38.33, "ppl": 1.00022, "step": 284, "tokens/total": 9907712, "tokens/train_per_sec_per_gpu": 21.61, "tokens/trainable": 129920} +{"epoch": 1.11328125, "grad_norm": 0.42134755849838257, "learning_rate": 5.0507984812754684e-05, "loss": 0.00480021582916379, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.17, "memory/max_allocated (GiB)": 37.17, "ppl": 1.00481, "step": 285, "tokens/total": 9941152, "tokens/train_per_sec_per_gpu": 28.51, "tokens/trainable": 130417} +{"epoch": 1.1171875, "grad_norm": 0.27188774943351746, "learning_rate": 5.021923930849237e-05, "loss": 0.0069952597841620445, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.00702, "step": 286, "tokens/total": 9975472, "tokens/train_per_sec_per_gpu": 27.74, "tokens/trainable": 130903} +{"epoch": 1.12109375, "grad_norm": 0.023997876793146133, "learning_rate": 4.99306927511967e-05, "loss": 0.00032383663346990943, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.31, "memory/max_allocated (GiB)": 37.31, "ppl": 1.00032, "step": 287, "tokens/total": 10009280, "tokens/train_per_sec_per_gpu": 27.69, "tokens/trainable": 131343} +{"epoch": 1.125, "grad_norm": 0.09925803542137146, "learning_rate": 4.964235714846775e-05, "loss": 0.0007188807358033955, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.72, "memory/max_allocated (GiB)": 36.72, "ppl": 1.00072, "step": 288, "tokens/total": 10041584, "tokens/train_per_sec_per_gpu": 26.99, "tokens/trainable": 131763} +{"epoch": 1.12890625, "grad_norm": 0.00982789695262909, "learning_rate": 4.9354244499126866e-05, "loss": 0.0001241689024027437, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.16, "memory/max_allocated (GiB)": 36.16, "ppl": 1.00012, "step": 289, "tokens/total": 10073312, "tokens/train_per_sec_per_gpu": 29.17, "tokens/trainable": 132229} +{"epoch": 1.1328125, "grad_norm": 0.013845434412360191, "learning_rate": 4.90663667927174e-05, "loss": 0.0002008321025641635, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.0002, "step": 290, "tokens/total": 10106160, "tokens/train_per_sec_per_gpu": 28.67, "tokens/trainable": 132678} +{"epoch": 1.13671875, "grad_norm": 0.28672370314598083, "learning_rate": 4.877873600900581e-05, "loss": 0.0033831228502094746, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.00339, "step": 291, "tokens/total": 10142192, "tokens/train_per_sec_per_gpu": 21.55, "tokens/trainable": 133136} +{"epoch": 1.140625, "grad_norm": 0.04364335909485817, "learning_rate": 4.849136411748306e-05, "loss": 0.0006782784475944936, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.00068, "step": 292, "tokens/total": 10179776, "tokens/train_per_sec_per_gpu": 24.73, "tokens/trainable": 133608} +{"epoch": 1.14453125, "grad_norm": 0.1523078978061676, "learning_rate": 4.8204263076866574e-05, "loss": 0.002693107584491372, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.0027, "step": 293, "tokens/total": 10215424, "tokens/train_per_sec_per_gpu": 29.42, "tokens/trainable": 134108} +{"epoch": 1.1484375, "grad_norm": 0.036740485578775406, "learning_rate": 4.791744483460251e-05, "loss": 0.00039760462823323905, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.0004, "step": 294, "tokens/total": 10249632, "tokens/train_per_sec_per_gpu": 24.07, "tokens/trainable": 134541} +{"epoch": 1.15234375, "grad_norm": 0.022351734340190887, "learning_rate": 4.7630921326368736e-05, "loss": 0.0003450984077062458, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.00035, "step": 295, "tokens/total": 10288272, "tokens/train_per_sec_per_gpu": 19.91, "tokens/trainable": 134966} +{"epoch": 1.15625, "grad_norm": 0.39561665058135986, "learning_rate": 4.7344704475577916e-05, "loss": 0.004850900731980801, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 35.66, "memory/max_allocated (GiB)": 35.66, "ppl": 1.00486, "step": 296, "tokens/total": 10318160, "tokens/train_per_sec_per_gpu": 30.09, "tokens/trainable": 135402} +{"epoch": 1.16015625, "grad_norm": 0.1772199422121048, "learning_rate": 4.705880619288153e-05, "loss": 0.002233506878837943, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.21, "memory/max_allocated (GiB)": 38.21, "ppl": 1.00224, "step": 297, "tokens/total": 10354304, "tokens/train_per_sec_per_gpu": 21.82, "tokens/trainable": 135804} +{"epoch": 1.1640625, "grad_norm": 0.029218392446637154, "learning_rate": 4.677323837567412e-05, "loss": 0.00030093532404862344, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.0003, "step": 298, "tokens/total": 10386960, "tokens/train_per_sec_per_gpu": 25.21, "tokens/trainable": 136231} +{"epoch": 1.16796875, "grad_norm": 0.035967182368040085, "learning_rate": 4.6488012907598146e-05, "loss": 0.00011747814278351143, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00012, "step": 299, "tokens/total": 10419712, "tokens/train_per_sec_per_gpu": 27.77, "tokens/trainable": 136705} +{"epoch": 1.171875, "grad_norm": 0.03373556584119797, "learning_rate": 4.620314165804964e-05, "loss": 0.00044040795182809234, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.3, "memory/max_allocated (GiB)": 38.3, "ppl": 1.00044, "step": 300, "tokens/total": 10455376, "tokens/train_per_sec_per_gpu": 25.54, "tokens/trainable": 137178} +{"epoch": 1.17578125, "grad_norm": 0.04944688826799393, "learning_rate": 4.591863648168407e-05, "loss": 0.0004919931525364518, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00049, "step": 301, "tokens/total": 10489712, "tokens/train_per_sec_per_gpu": 26.21, "tokens/trainable": 137643} +{"epoch": 1.1796875, "grad_norm": 0.21052847802639008, "learning_rate": 4.5634509217923135e-05, "loss": 0.002838349435478449, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00284, "step": 302, "tokens/total": 10523056, "tokens/train_per_sec_per_gpu": 25.66, "tokens/trainable": 138078} +{"epoch": 1.18359375, "grad_norm": 0.13508549332618713, "learning_rate": 4.535077169046201e-05, "loss": 0.0013916888274252415, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00139, "step": 303, "tokens/total": 10559616, "tokens/train_per_sec_per_gpu": 25.01, "tokens/trainable": 138515} +{"epoch": 1.1875, "grad_norm": 0.16052572429180145, "learning_rate": 4.506743570677743e-05, "loss": 0.002386226551607251, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.00239, "step": 304, "tokens/total": 10590208, "tokens/train_per_sec_per_gpu": 31.12, "tokens/trainable": 138948} +{"epoch": 1.19140625, "grad_norm": 0.2260577529668808, "learning_rate": 4.478451305763618e-05, "loss": 0.0029074568301439285, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.95, "memory/max_allocated (GiB)": 36.95, "ppl": 1.00291, "step": 305, "tokens/total": 10623344, "tokens/train_per_sec_per_gpu": 30.38, "tokens/trainable": 139408} +{"epoch": 1.1953125, "grad_norm": 0.015745922923088074, "learning_rate": 4.450201551660454e-05, "loss": 0.00012605970550794154, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.51, "memory/max_allocated (GiB)": 36.51, "ppl": 1.00013, "step": 306, "tokens/total": 10655056, "tokens/train_per_sec_per_gpu": 26.07, "tokens/trainable": 139840} +{"epoch": 1.19921875, "grad_norm": 0.3923283517360687, "learning_rate": 4.4219954839558276e-05, "loss": 0.012541276402771473, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.01262, "step": 307, "tokens/total": 10689744, "tokens/train_per_sec_per_gpu": 25.22, "tokens/trainable": 140281} +{"epoch": 1.203125, "grad_norm": 0.012616004794836044, "learning_rate": 4.393834276419352e-05, "loss": 0.0001661831483943388, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.93, "memory/max_allocated (GiB)": 36.93, "ppl": 1.00017, "step": 308, "tokens/total": 10721456, "tokens/train_per_sec_per_gpu": 31.42, "tokens/trainable": 140776} +{"epoch": 1.20703125, "grad_norm": 0.00204321532510221, "learning_rate": 4.36571910095382e-05, "loss": 3.6538433050736785e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.00004, "step": 309, "tokens/total": 10759568, "tokens/train_per_sec_per_gpu": 25.3, "tokens/trainable": 141228} +{"epoch": 1.2109375, "grad_norm": 0.049058519303798676, "learning_rate": 4.337651127546448e-05, "loss": 0.0006042327731847763, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.0006, "step": 310, "tokens/total": 10795248, "tokens/train_per_sec_per_gpu": 27.41, "tokens/trainable": 141679} +{"epoch": 1.21484375, "grad_norm": 0.043743640184402466, "learning_rate": 4.3096315242201736e-05, "loss": 0.00043361878488212824, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.61, "memory/max_allocated (GiB)": 36.61, "ppl": 1.00043, "step": 311, "tokens/total": 10828208, "tokens/train_per_sec_per_gpu": 26.1, "tokens/trainable": 142122} +{"epoch": 1.21875, "grad_norm": 0.18138237297534943, "learning_rate": 4.2816614569850635e-05, "loss": 0.0017160456627607346, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00172, "step": 312, "tokens/total": 10863984, "tokens/train_per_sec_per_gpu": 24.16, "tokens/trainable": 142561} +{"epoch": 1.22265625, "grad_norm": 0.08186016231775284, "learning_rate": 4.2537420897897864e-05, "loss": 0.000915415002964437, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.00092, "step": 313, "tokens/total": 10899392, "tokens/train_per_sec_per_gpu": 26.84, "tokens/trainable": 143046} +{"epoch": 1.2265625, "grad_norm": 0.17508797347545624, "learning_rate": 4.225874584473174e-05, "loss": 0.0008588077034801245, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.00086, "step": 314, "tokens/total": 10932224, "tokens/train_per_sec_per_gpu": 26.84, "tokens/trainable": 143493} +{"epoch": 1.23046875, "grad_norm": 0.01913650706410408, "learning_rate": 4.19806010071587e-05, "loss": 0.00018267385894432664, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.00018, "step": 315, "tokens/total": 10970096, "tokens/train_per_sec_per_gpu": 24.18, "tokens/trainable": 143956} +{"epoch": 1.234375, "grad_norm": 0.006437751464545727, "learning_rate": 4.170299795992081e-05, "loss": 0.0001295715046580881, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.00013, "step": 316, "tokens/total": 11003488, "tokens/train_per_sec_per_gpu": 26.84, "tokens/trainable": 144411} +{"epoch": 1.23828125, "grad_norm": 0.1168462410569191, "learning_rate": 4.142594825521398e-05, "loss": 0.0011574298841878772, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.00116, "step": 317, "tokens/total": 11042784, "tokens/train_per_sec_per_gpu": 25.78, "tokens/trainable": 144892} +{"epoch": 1.2421875, "grad_norm": 0.006776756141334772, "learning_rate": 4.114946342220728e-05, "loss": 6.594027945538983e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.00007, "step": 318, "tokens/total": 11078992, "tokens/train_per_sec_per_gpu": 23.78, "tokens/trainable": 145339} +{"epoch": 1.24609375, "grad_norm": 0.34199297428131104, "learning_rate": 4.087355496656321e-05, "loss": 0.0018556759459897876, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00186, "step": 319, "tokens/total": 11113024, "tokens/train_per_sec_per_gpu": 28.11, "tokens/trainable": 145811} +{"epoch": 1.25, "grad_norm": 0.08715706318616867, "learning_rate": 4.05982343699588e-05, "loss": 0.0005232486873865128, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00052, "step": 320, "tokens/total": 11152240, "tokens/train_per_sec_per_gpu": 25.79, "tokens/trainable": 146314} +{"epoch": 1.25390625, "grad_norm": 0.013385830447077751, "learning_rate": 4.0323513089607876e-05, "loss": 0.00013736920664086938, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.00014, "step": 321, "tokens/total": 11187296, "tokens/train_per_sec_per_gpu": 24.15, "tokens/trainable": 146781} +{"epoch": 1.2578125, "grad_norm": 0.007606017868965864, "learning_rate": 4.004940255778431e-05, "loss": 8.634770347271115e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00009, "step": 322, "tokens/total": 11225904, "tokens/train_per_sec_per_gpu": 23.78, "tokens/trainable": 147225} +{"epoch": 1.26171875, "grad_norm": 0.0054871863685548306, "learning_rate": 3.977591418134619e-05, "loss": 8.426588465226814e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.00008, "step": 323, "tokens/total": 11261056, "tokens/train_per_sec_per_gpu": 26.59, "tokens/trainable": 147673} +{"epoch": 1.265625, "grad_norm": 0.03204001113772392, "learning_rate": 3.95030593412612e-05, "loss": 0.00017924036365002394, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.00018, "step": 324, "tokens/total": 11296432, "tokens/train_per_sec_per_gpu": 24.44, "tokens/trainable": 148103} +{"epoch": 1.26953125, "grad_norm": 0.002661786274984479, "learning_rate": 3.923084939213296e-05, "loss": 4.7457397158723325e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 35.88, "memory/max_allocated (GiB)": 35.88, "ppl": 1.00005, "step": 325, "tokens/total": 11327248, "tokens/train_per_sec_per_gpu": 27.62, "tokens/trainable": 148535} +{"epoch": 1.2734375, "grad_norm": 0.448566734790802, "learning_rate": 3.895929566172861e-05, "loss": 0.0010220836848020554, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.00102, "step": 326, "tokens/total": 11363264, "tokens/train_per_sec_per_gpu": 25.05, "tokens/trainable": 148999} +{"epoch": 1.27734375, "grad_norm": 0.05879812687635422, "learning_rate": 3.868840945050728e-05, "loss": 0.00033544556936249137, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.05, "memory/max_allocated (GiB)": 39.05, "ppl": 1.00034, "step": 327, "tokens/total": 11401776, "tokens/train_per_sec_per_gpu": 21.39, "tokens/trainable": 149431} +{"epoch": 1.28125, "grad_norm": 0.25485965609550476, "learning_rate": 3.841820203114995e-05, "loss": 0.0020062746480107307, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.00201, "step": 328, "tokens/total": 11433520, "tokens/train_per_sec_per_gpu": 28.88, "tokens/trainable": 149886} +{"epoch": 1.28515625, "grad_norm": 0.637201726436615, "learning_rate": 3.814868464809027e-05, "loss": 0.010540912859141827, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.0106, "step": 329, "tokens/total": 11465376, "tokens/train_per_sec_per_gpu": 25.67, "tokens/trainable": 150288} +{"epoch": 1.2890625, "grad_norm": 0.0005735920276492834, "learning_rate": 3.787986851704667e-05, "loss": 1.406269620929379e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.00001, "step": 330, "tokens/total": 11499120, "tokens/train_per_sec_per_gpu": 26.43, "tokens/trainable": 150733} +{"epoch": 1.29296875, "grad_norm": 0.0022641844116151333, "learning_rate": 3.7611764824555654e-05, "loss": 4.4520820665638894e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00004, "step": 331, "tokens/total": 11531104, "tokens/train_per_sec_per_gpu": 30.74, "tokens/trainable": 151207} +{"epoch": 1.296875, "grad_norm": 0.003771537449210882, "learning_rate": 3.734438472750619e-05, "loss": 3.291451503173448e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.00003, "step": 332, "tokens/total": 11571040, "tokens/train_per_sec_per_gpu": 24.47, "tokens/trainable": 151694} +{"epoch": 1.30078125, "grad_norm": 0.015899060294032097, "learning_rate": 3.707773935267552e-05, "loss": 0.000147302242112346, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.46, "memory/max_allocated (GiB)": 38.46, "ppl": 1.00015, "step": 333, "tokens/total": 11607952, "tokens/train_per_sec_per_gpu": 27.57, "tokens/trainable": 152188} +{"epoch": 1.3046875, "grad_norm": 0.002688730601221323, "learning_rate": 3.68118397962661e-05, "loss": 4.351840470917523e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.00004, "step": 334, "tokens/total": 11642864, "tokens/train_per_sec_per_gpu": 23.15, "tokens/trainable": 152596} +{"epoch": 1.30859375, "grad_norm": 0.03873877227306366, "learning_rate": 3.654669712344384e-05, "loss": 0.0002899948740378022, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 1.00029, "step": 335, "tokens/total": 11673904, "tokens/train_per_sec_per_gpu": 31.46, "tokens/trainable": 153052} +{"epoch": 1.3125, "grad_norm": 0.0006093586562201381, "learning_rate": 3.628232236787763e-05, "loss": 1.4225369341147598e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.96, "memory/max_allocated (GiB)": 37.96, "ppl": 1.00001, "step": 336, "tokens/total": 11709536, "tokens/train_per_sec_per_gpu": 22.54, "tokens/trainable": 153491} +{"epoch": 1.31640625, "grad_norm": 0.0021342667751014233, "learning_rate": 3.6018726531280144e-05, "loss": 3.488809670670889e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00003, "step": 337, "tokens/total": 11747248, "tokens/train_per_sec_per_gpu": 28.13, "tokens/trainable": 153983} +{"epoch": 1.3203125, "grad_norm": 0.4319891035556793, "learning_rate": 3.575592058295017e-05, "loss": 0.002268322976306081, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.00227, "step": 338, "tokens/total": 11781104, "tokens/train_per_sec_per_gpu": 28.55, "tokens/trainable": 154464} +{"epoch": 1.32421875, "grad_norm": 0.0009367328602820635, "learning_rate": 3.549391545931585e-05, "loss": 1.5551704564131796e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00002, "step": 339, "tokens/total": 11815712, "tokens/train_per_sec_per_gpu": 27.56, "tokens/trainable": 154924} +{"epoch": 1.328125, "grad_norm": 0.0006406668107956648, "learning_rate": 3.5232722063479914e-05, "loss": 1.338930451311171e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.00001, "step": 340, "tokens/total": 11849424, "tokens/train_per_sec_per_gpu": 24.04, "tokens/trainable": 155373} +{"epoch": 1.33203125, "grad_norm": 0.03221272677183151, "learning_rate": 3.49723512647657e-05, "loss": 0.0001006315287668258, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.0001, "step": 341, "tokens/total": 11888128, "tokens/train_per_sec_per_gpu": 24.63, "tokens/trainable": 155873} +{"epoch": 1.3359375, "grad_norm": 0.21723531186580658, "learning_rate": 3.471281389826491e-05, "loss": 0.0014895576750859618, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.00149, "step": 342, "tokens/total": 11919264, "tokens/train_per_sec_per_gpu": 25.76, "tokens/trainable": 156278} +{"epoch": 1.33984375, "grad_norm": 0.011056101880967617, "learning_rate": 3.4454120764386764e-05, "loss": 0.00015520243323408067, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.66, "memory/max_allocated (GiB)": 38.66, "ppl": 1.00016, "step": 343, "tokens/total": 11957456, "tokens/train_per_sec_per_gpu": 24.16, "tokens/trainable": 156733} +{"epoch": 1.34375, "grad_norm": 0.015959901735186577, "learning_rate": 3.4196282628408526e-05, "loss": 0.00010145694250240922, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.0001, "step": 344, "tokens/total": 11991696, "tokens/train_per_sec_per_gpu": 26.18, "tokens/trainable": 157203} +{"epoch": 1.34765625, "grad_norm": 0.015764355659484863, "learning_rate": 3.3939310220027456e-05, "loss": 7.944804383441806e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00008, "step": 345, "tokens/total": 12025840, "tokens/train_per_sec_per_gpu": 29.72, "tokens/trainable": 157707} +{"epoch": 1.3515625, "grad_norm": 0.007098136469721794, "learning_rate": 3.3683214232914404e-05, "loss": 9.856780525296926e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.0001, "step": 346, "tokens/total": 12061296, "tokens/train_per_sec_per_gpu": 26.0, "tokens/trainable": 158180} +{"epoch": 1.35546875, "grad_norm": 0.1331765502691269, "learning_rate": 3.342800532426873e-05, "loss": 0.0009050953667610884, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.00091, "step": 347, "tokens/total": 12097232, "tokens/train_per_sec_per_gpu": 25.33, "tokens/trainable": 158646} +{"epoch": 1.359375, "grad_norm": 0.1597379446029663, "learning_rate": 3.317369411437484e-05, "loss": 0.0008022456895560026, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.0008, "step": 348, "tokens/total": 12133200, "tokens/train_per_sec_per_gpu": 26.03, "tokens/trainable": 159115} +{"epoch": 1.36328125, "grad_norm": 0.022662969306111336, "learning_rate": 3.292029118616024e-05, "loss": 0.00012391747441142797, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.00012, "step": 349, "tokens/total": 12166336, "tokens/train_per_sec_per_gpu": 25.46, "tokens/trainable": 159538} +{"epoch": 1.3671875, "grad_norm": 0.30865857005119324, "learning_rate": 3.266780708475511e-05, "loss": 0.0037659050431102514, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00377, "step": 350, "tokens/total": 12201872, "tokens/train_per_sec_per_gpu": 24.93, "tokens/trainable": 160016} +{"epoch": 1.37109375, "grad_norm": 0.23944300413131714, "learning_rate": 3.241625231705354e-05, "loss": 0.0011614192044362426, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.00116, "step": 351, "tokens/total": 12238016, "tokens/train_per_sec_per_gpu": 24.9, "tokens/trainable": 160475} +{"epoch": 1.375, "grad_norm": 0.1360509991645813, "learning_rate": 3.216563735127618e-05, "loss": 0.0010851486586034298, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.15, "memory/max_allocated (GiB)": 39.15, "ppl": 1.00109, "step": 352, "tokens/total": 12273280, "tokens/train_per_sec_per_gpu": 25.42, "tokens/trainable": 160952} +{"epoch": 1.37890625, "grad_norm": 0.2860148251056671, "learning_rate": 3.191597261653475e-05, "loss": 0.01498163677752018, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.01509, "step": 353, "tokens/total": 12310096, "tokens/train_per_sec_per_gpu": 23.57, "tokens/trainable": 161420} +{"epoch": 1.3828125, "grad_norm": 0.24991486966609955, "learning_rate": 3.166726850239794e-05, "loss": 0.0011784187518060207, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00118, "step": 354, "tokens/total": 12343136, "tokens/train_per_sec_per_gpu": 27.28, "tokens/trainable": 161868} +{"epoch": 1.38671875, "grad_norm": 0.001614454435184598, "learning_rate": 3.141953535845912e-05, "loss": 2.540545756346546e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.34, "memory/max_allocated (GiB)": 36.34, "ppl": 1.00003, "step": 355, "tokens/total": 12374576, "tokens/train_per_sec_per_gpu": 30.23, "tokens/trainable": 162318} +{"epoch": 1.390625, "grad_norm": 0.0021764496341347694, "learning_rate": 3.11727834939056e-05, "loss": 3.925157943740487e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00004, "step": 356, "tokens/total": 12406896, "tokens/train_per_sec_per_gpu": 26.21, "tokens/trainable": 162789} +{"epoch": 1.39453125, "grad_norm": 0.5563468337059021, "learning_rate": 3.092702317708967e-05, "loss": 0.014688365161418915, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.0148, "step": 357, "tokens/total": 12440480, "tokens/train_per_sec_per_gpu": 28.76, "tokens/trainable": 163254} +{"epoch": 1.3984375, "grad_norm": 0.0009688844438642263, "learning_rate": 3.0682264635101276e-05, "loss": 1.5852550859563053e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.00002, "step": 358, "tokens/total": 12473232, "tokens/train_per_sec_per_gpu": 28.12, "tokens/trainable": 163715} +{"epoch": 1.40234375, "grad_norm": 0.009431019425392151, "learning_rate": 3.0438518053342407e-05, "loss": 9.112834231927991e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00009, "step": 359, "tokens/total": 12512176, "tokens/train_per_sec_per_gpu": 23.09, "tokens/trainable": 164197} +{"epoch": 1.40625, "grad_norm": 0.035012952983379364, "learning_rate": 3.0195793575103266e-05, "loss": 0.00019362215243745595, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00019, "step": 360, "tokens/total": 12551136, "tokens/train_per_sec_per_gpu": 21.14, "tokens/trainable": 164661} +{"epoch": 1.41015625, "grad_norm": 0.16078339517116547, "learning_rate": 2.9954101301140146e-05, "loss": 0.0029335045255720615, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.57, "memory/max_allocated (GiB)": 37.57, "ppl": 1.00294, "step": 361, "tokens/total": 12586560, "tokens/train_per_sec_per_gpu": 24.44, "tokens/trainable": 165111} +{"epoch": 1.4140625, "grad_norm": 0.02231469191610813, "learning_rate": 2.9713451289255123e-05, "loss": 0.00020817095355596393, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.63, "memory/max_allocated (GiB)": 36.63, "ppl": 1.00021, "step": 362, "tokens/total": 12615936, "tokens/train_per_sec_per_gpu": 25.42, "tokens/trainable": 165552} +{"epoch": 1.41796875, "grad_norm": 0.005332330707460642, "learning_rate": 2.9473853553877484e-05, "loss": 8.64562316564843e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.00009, "step": 363, "tokens/total": 12650096, "tokens/train_per_sec_per_gpu": 27.12, "tokens/trainable": 166026} +{"epoch": 1.421875, "grad_norm": 1.1177880764007568, "learning_rate": 2.9235318065647e-05, "loss": 0.005396474618464708, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.00541, "step": 364, "tokens/total": 12689808, "tokens/train_per_sec_per_gpu": 24.8, "tokens/trainable": 166486} +{"epoch": 1.42578125, "grad_norm": 0.006738404743373394, "learning_rate": 2.8997854750998964e-05, "loss": 9.613888687454164e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.46, "memory/max_allocated (GiB)": 39.46, "ppl": 1.0001, "step": 365, "tokens/total": 12729328, "tokens/train_per_sec_per_gpu": 23.82, "tokens/trainable": 166959} +{"epoch": 1.4296875, "grad_norm": 0.03976349160075188, "learning_rate": 2.8761473491751258e-05, "loss": 0.0003691685269586742, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.00037, "step": 366, "tokens/total": 12762576, "tokens/train_per_sec_per_gpu": 26.74, "tokens/trainable": 167394} +{"epoch": 1.43359375, "grad_norm": 0.21819452941417694, "learning_rate": 2.8526184124692883e-05, "loss": 0.001690921955741942, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00169, "step": 367, "tokens/total": 12798944, "tokens/train_per_sec_per_gpu": 20.29, "tokens/trainable": 167832} +{"epoch": 1.4375, "grad_norm": 0.2759339213371277, "learning_rate": 2.829199644117484e-05, "loss": 0.002320743864402175, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.00232, "step": 368, "tokens/total": 12832928, "tokens/train_per_sec_per_gpu": 24.22, "tokens/trainable": 168242} +{"epoch": 1.44140625, "grad_norm": 0.23343148827552795, "learning_rate": 2.8058920186702553e-05, "loss": 0.0017874054610729218, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00179, "step": 369, "tokens/total": 12870544, "tokens/train_per_sec_per_gpu": 24.89, "tokens/trainable": 168715} +{"epoch": 1.4453125, "grad_norm": 0.029599905014038086, "learning_rate": 2.782696506053033e-05, "loss": 0.00026624745805747807, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.00027, "step": 370, "tokens/total": 12908416, "tokens/train_per_sec_per_gpu": 22.24, "tokens/trainable": 169147} +{"epoch": 1.44921875, "grad_norm": 0.24592413008213043, "learning_rate": 2.7596140715257824e-05, "loss": 0.001930495840497315, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00193, "step": 371, "tokens/total": 12943584, "tokens/train_per_sec_per_gpu": 25.55, "tokens/trainable": 169628} +{"epoch": 1.453125, "grad_norm": 0.029699506238102913, "learning_rate": 2.7366456756428184e-05, "loss": 0.00024886359460651875, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 1.00025, "step": 372, "tokens/total": 12978192, "tokens/train_per_sec_per_gpu": 27.04, "tokens/trainable": 170085} +{"epoch": 1.45703125, "grad_norm": 0.11580313742160797, "learning_rate": 2.7137922742128486e-05, "loss": 0.0011012875474989414, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.0011, "step": 373, "tokens/total": 13012720, "tokens/train_per_sec_per_gpu": 26.87, "tokens/trainable": 170584} +{"epoch": 1.4609375, "grad_norm": 0.323001503944397, "learning_rate": 2.691054818259188e-05, "loss": 0.004873649682849646, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00489, "step": 374, "tokens/total": 13047568, "tokens/train_per_sec_per_gpu": 26.31, "tokens/trainable": 171058} +{"epoch": 1.46484375, "grad_norm": 0.016029415652155876, "learning_rate": 2.6684342539801933e-05, "loss": 0.00011136815010104328, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.56, "memory/max_allocated (GiB)": 36.56, "ppl": 1.00011, "step": 375, "tokens/total": 13079088, "tokens/train_per_sec_per_gpu": 28.67, "tokens/trainable": 171518} +{"epoch": 1.46875, "grad_norm": 0.00573897548019886, "learning_rate": 2.645931522709877e-05, "loss": 7.239238766487688e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.00007, "step": 376, "tokens/total": 13114480, "tokens/train_per_sec_per_gpu": 29.88, "tokens/trainable": 172007} +{"epoch": 1.47265625, "grad_norm": 0.5234684944152832, "learning_rate": 2.6235475608787365e-05, "loss": 0.010628749616444111, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.01069, "step": 377, "tokens/total": 13150160, "tokens/train_per_sec_per_gpu": 25.39, "tokens/trainable": 172468} +{"epoch": 1.4765625, "grad_norm": 0.10254120826721191, "learning_rate": 2.6012832999747916e-05, "loss": 0.0007823070045560598, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.00078, "step": 378, "tokens/total": 13181280, "tokens/train_per_sec_per_gpu": 29.1, "tokens/trainable": 172923} +{"epoch": 1.48046875, "grad_norm": 0.6378411054611206, "learning_rate": 2.579139666504821e-05, "loss": 0.018513711169362068, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.01869, "step": 379, "tokens/total": 13221264, "tokens/train_per_sec_per_gpu": 22.04, "tokens/trainable": 173370} +{"epoch": 1.484375, "grad_norm": 0.012222162447869778, "learning_rate": 2.557117581955798e-05, "loss": 0.00014631659723818302, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00015, "step": 380, "tokens/total": 13255104, "tokens/train_per_sec_per_gpu": 26.85, "tokens/trainable": 173837} +{"epoch": 1.48828125, "grad_norm": 0.00878298468887806, "learning_rate": 2.5352179627565532e-05, "loss": 0.00016792590031400323, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.00017, "step": 381, "tokens/total": 13289856, "tokens/train_per_sec_per_gpu": 27.07, "tokens/trainable": 174294} +{"epoch": 1.4921875, "grad_norm": 0.011515031568706036, "learning_rate": 2.5134417202396277e-05, "loss": 0.0001647517055971548, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.6, "memory/max_allocated (GiB)": 38.6, "ppl": 1.00016, "step": 382, "tokens/total": 13323840, "tokens/train_per_sec_per_gpu": 23.9, "tokens/trainable": 174701} +{"epoch": 1.49609375, "grad_norm": 0.1959695667028427, "learning_rate": 2.491789760603361e-05, "loss": 0.0025801025331020355, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00258, "step": 383, "tokens/total": 13360256, "tokens/train_per_sec_per_gpu": 23.13, "tokens/trainable": 175131} +{"epoch": 1.5, "grad_norm": 0.15185467898845673, "learning_rate": 2.4702629848741764e-05, "loss": 0.001856961753219366, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.00186, "step": 384, "tokens/total": 13394304, "tokens/train_per_sec_per_gpu": 26.38, "tokens/trainable": 175586} +{"epoch": 1.50390625, "grad_norm": 0.16519807279109955, "learning_rate": 2.4488622888690785e-05, "loss": 0.0019553028978407383, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.04, "memory/max_allocated (GiB)": 37.04, "ppl": 1.00196, "step": 385, "tokens/total": 13427760, "tokens/train_per_sec_per_gpu": 26.89, "tokens/trainable": 176026} +{"epoch": 1.5078125, "grad_norm": 0.01208545733243227, "learning_rate": 2.427588563158384e-05, "loss": 0.0001590855827089399, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00016, "step": 386, "tokens/total": 13462096, "tokens/train_per_sec_per_gpu": 26.01, "tokens/trainable": 176512} +{"epoch": 1.51171875, "grad_norm": 0.07418987900018692, "learning_rate": 2.406442693028651e-05, "loss": 0.0010320128640159965, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.63, "memory/max_allocated (GiB)": 38.63, "ppl": 1.00103, "step": 387, "tokens/total": 13494976, "tokens/train_per_sec_per_gpu": 24.69, "tokens/trainable": 176943} +{"epoch": 1.515625, "grad_norm": 0.036174628883600235, "learning_rate": 2.3854255584458547e-05, "loss": 0.00043402682058513165, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.00043, "step": 388, "tokens/total": 13533568, "tokens/train_per_sec_per_gpu": 21.51, "tokens/trainable": 177370} +{"epoch": 1.51953125, "grad_norm": 0.01919744536280632, "learning_rate": 2.3645380340187508e-05, "loss": 0.00020190206123515964, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.0002, "step": 389, "tokens/total": 13569248, "tokens/train_per_sec_per_gpu": 26.0, "tokens/trainable": 177851} +{"epoch": 1.5234375, "grad_norm": 0.18481408059597015, "learning_rate": 2.3437809889624914e-05, "loss": 0.003654481377452612, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.00366, "step": 390, "tokens/total": 13601568, "tokens/train_per_sec_per_gpu": 28.81, "tokens/trainable": 178310} +{"epoch": 1.52734375, "grad_norm": 0.07857727259397507, "learning_rate": 2.3231552870624487e-05, "loss": 0.000458476017229259, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.47, "memory/max_allocated (GiB)": 38.47, "ppl": 1.00046, "step": 391, "tokens/total": 13632576, "tokens/train_per_sec_per_gpu": 29.02, "tokens/trainable": 178736} +{"epoch": 1.53125, "grad_norm": 0.04043705761432648, "learning_rate": 2.3026617866382657e-05, "loss": 0.00045952663640491664, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.53, "memory/max_allocated (GiB)": 36.53, "ppl": 1.00046, "step": 392, "tokens/total": 13665328, "tokens/train_per_sec_per_gpu": 29.72, "tokens/trainable": 179233} +{"epoch": 1.53515625, "grad_norm": 0.09139404445886612, "learning_rate": 2.2823013405081507e-05, "loss": 0.0007962223025970161, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.0008, "step": 393, "tokens/total": 13703408, "tokens/train_per_sec_per_gpu": 26.72, "tokens/trainable": 179730} +{"epoch": 1.5390625, "grad_norm": 0.006225260440260172, "learning_rate": 2.2620747959533722e-05, "loss": 0.00010317363194189966, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.76, "memory/max_allocated (GiB)": 36.76, "ppl": 1.0001, "step": 394, "tokens/total": 13736480, "tokens/train_per_sec_per_gpu": 30.52, "tokens/trainable": 180227} +{"epoch": 1.54296875, "grad_norm": 0.12396049499511719, "learning_rate": 2.2419829946830123e-05, "loss": 0.001134020509198308, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.00113, "step": 395, "tokens/total": 13772672, "tokens/train_per_sec_per_gpu": 24.1, "tokens/trainable": 180687} +{"epoch": 1.546875, "grad_norm": 0.16409531235694885, "learning_rate": 2.2220267727989325e-05, "loss": 0.0011506029404699802, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00115, "step": 396, "tokens/total": 13806816, "tokens/train_per_sec_per_gpu": 25.67, "tokens/trainable": 181141} +{"epoch": 1.55078125, "grad_norm": 0.357058584690094, "learning_rate": 2.202206960760984e-05, "loss": 0.004920411854982376, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.00493, "step": 397, "tokens/total": 13844960, "tokens/train_per_sec_per_gpu": 24.61, "tokens/trainable": 181648} +{"epoch": 1.5546875, "grad_norm": 0.06006897613406181, "learning_rate": 2.182524383352446e-05, "loss": 0.0012414385564625263, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00124, "step": 398, "tokens/total": 13881120, "tokens/train_per_sec_per_gpu": 24.78, "tokens/trainable": 182109} +{"epoch": 1.55859375, "grad_norm": 0.2529268264770508, "learning_rate": 2.1629798596457056e-05, "loss": 0.009272404946386814, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.52, "memory/max_allocated (GiB)": 36.52, "ppl": 1.00932, "step": 399, "tokens/total": 13913440, "tokens/train_per_sec_per_gpu": 26.51, "tokens/trainable": 182559} +{"epoch": 1.5625, "grad_norm": 0.002529528923332691, "learning_rate": 2.1435742029681725e-05, "loss": 5.950435297563672e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.00006, "step": 400, "tokens/total": 13945408, "tokens/train_per_sec_per_gpu": 27.52, "tokens/trainable": 182985} +{"epoch": 1.56640625, "grad_norm": 0.0034168099518865347, "learning_rate": 2.124308220868431e-05, "loss": 7.009714317973703e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.67, "memory/max_allocated (GiB)": 38.67, "ppl": 1.00007, "step": 401, "tokens/total": 13980800, "tokens/train_per_sec_per_gpu": 24.17, "tokens/trainable": 183440} +{"epoch": 1.5703125, "grad_norm": 0.04482279717922211, "learning_rate": 2.105182715082638e-05, "loss": 0.0010448534740135074, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00105, "step": 402, "tokens/total": 14014400, "tokens/train_per_sec_per_gpu": 27.46, "tokens/trainable": 183917} +{"epoch": 1.57421875, "grad_norm": 0.022888874635100365, "learning_rate": 2.0861984815011552e-05, "loss": 0.0002277538151247427, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.16, "memory/max_allocated (GiB)": 38.16, "ppl": 1.00023, "step": 403, "tokens/total": 14049920, "tokens/train_per_sec_per_gpu": 22.5, "tokens/trainable": 184348} +{"epoch": 1.578125, "grad_norm": 0.015218008309602737, "learning_rate": 2.0673563101354323e-05, "loss": 0.00015581853222101927, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00016, "step": 404, "tokens/total": 14085200, "tokens/train_per_sec_per_gpu": 23.45, "tokens/trainable": 184786} +{"epoch": 1.58203125, "grad_norm": 0.17136098444461823, "learning_rate": 2.0486569850851317e-05, "loss": 0.0016653267666697502, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.00167, "step": 405, "tokens/total": 14120208, "tokens/train_per_sec_per_gpu": 25.76, "tokens/trainable": 185249} +{"epoch": 1.5859375, "grad_norm": 0.21986542642116547, "learning_rate": 2.0301012845054956e-05, "loss": 0.005473548546433449, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.00549, "step": 406, "tokens/total": 14158960, "tokens/train_per_sec_per_gpu": 21.32, "tokens/trainable": 185680} +{"epoch": 1.58984375, "grad_norm": 0.03824439272284508, "learning_rate": 2.011689980574966e-05, "loss": 0.0006382841966114938, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.06, "memory/max_allocated (GiB)": 38.06, "ppl": 1.00064, "step": 407, "tokens/total": 14195264, "tokens/train_per_sec_per_gpu": 22.83, "tokens/trainable": 186100} +{"epoch": 1.59375, "grad_norm": 0.18928202986717224, "learning_rate": 1.993423839463052e-05, "loss": 0.002694483147934079, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.0027, "step": 408, "tokens/total": 14228528, "tokens/train_per_sec_per_gpu": 31.74, "tokens/trainable": 186565} +{"epoch": 1.59765625, "grad_norm": 0.0050294972024858, "learning_rate": 1.975303621298445e-05, "loss": 8.34912498248741e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.00008, "step": 409, "tokens/total": 14265568, "tokens/train_per_sec_per_gpu": 25.29, "tokens/trainable": 187017} +{"epoch": 1.6015625, "grad_norm": 0.051765624433755875, "learning_rate": 1.957330080137385e-05, "loss": 0.0006223291275091469, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.31, "memory/max_allocated (GiB)": 36.31, "ppl": 1.00062, "step": 410, "tokens/total": 14296432, "tokens/train_per_sec_per_gpu": 28.86, "tokens/trainable": 187468} +{"epoch": 1.60546875, "grad_norm": 0.143672913312912, "learning_rate": 1.9395039639322864e-05, "loss": 0.0016424062196165323, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.00164, "step": 411, "tokens/total": 14332480, "tokens/train_per_sec_per_gpu": 27.25, "tokens/trainable": 187961} +{"epoch": 1.609375, "grad_norm": 0.0016930910060182214, "learning_rate": 1.9218260145006073e-05, "loss": 4.712857480626553e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.52, "memory/max_allocated (GiB)": 37.52, "ppl": 1.00005, "step": 412, "tokens/total": 14367728, "tokens/train_per_sec_per_gpu": 26.9, "tokens/trainable": 188447} +{"epoch": 1.61328125, "grad_norm": 0.015627160668373108, "learning_rate": 1.904296967493982e-05, "loss": 0.0002675892901606858, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.00027, "step": 413, "tokens/total": 14399888, "tokens/train_per_sec_per_gpu": 29.57, "tokens/trainable": 188903} +{"epoch": 1.6171875, "grad_norm": 0.990898847579956, "learning_rate": 1.8869175523676064e-05, "loss": 0.0020251874811947346, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.00203, "step": 414, "tokens/total": 14436576, "tokens/train_per_sec_per_gpu": 26.56, "tokens/trainable": 189398} +{"epoch": 1.62109375, "grad_norm": 0.0077995415776968, "learning_rate": 1.869688492349885e-05, "loss": 0.00014718272723257542, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00015, "step": 415, "tokens/total": 14470032, "tokens/train_per_sec_per_gpu": 27.27, "tokens/trainable": 189850} +{"epoch": 1.625, "grad_norm": 0.153909832239151, "learning_rate": 1.85261050441233e-05, "loss": 0.0015089270891621709, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.00151, "step": 416, "tokens/total": 14505824, "tokens/train_per_sec_per_gpu": 24.33, "tokens/trainable": 190306} +{"epoch": 1.62890625, "grad_norm": 0.2682539224624634, "learning_rate": 1.8356842992397304e-05, "loss": 0.0076913353987038136, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.13, "memory/max_allocated (GiB)": 37.13, "ppl": 1.00772, "step": 417, "tokens/total": 14539344, "tokens/train_per_sec_per_gpu": 26.99, "tokens/trainable": 190746} +{"epoch": 1.6328125, "grad_norm": 0.010914824903011322, "learning_rate": 1.8189105812005714e-05, "loss": 0.0002752303844317794, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.00028, "step": 418, "tokens/total": 14576160, "tokens/train_per_sec_per_gpu": 22.14, "tokens/trainable": 191156} +{"epoch": 1.63671875, "grad_norm": 0.1812041848897934, "learning_rate": 1.802290048317732e-05, "loss": 0.001438817591406405, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00144, "step": 419, "tokens/total": 14609840, "tokens/train_per_sec_per_gpu": 23.29, "tokens/trainable": 191573} +{"epoch": 1.640625, "grad_norm": 0.04129435867071152, "learning_rate": 1.785823392239424e-05, "loss": 0.000592977914493531, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.00059, "step": 420, "tokens/total": 14643424, "tokens/train_per_sec_per_gpu": 26.02, "tokens/trainable": 192014} +{"epoch": 1.64453125, "grad_norm": 0.2209179699420929, "learning_rate": 1.7695112982104225e-05, "loss": 0.005398175213485956, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00541, "step": 421, "tokens/total": 14678432, "tokens/train_per_sec_per_gpu": 24.74, "tokens/trainable": 192453} +{"epoch": 1.6484375, "grad_norm": 0.01858133263885975, "learning_rate": 1.7533544450435433e-05, "loss": 0.0001897708571050316, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 35.79, "memory/max_allocated (GiB)": 35.79, "ppl": 1.00019, "step": 422, "tokens/total": 14707984, "tokens/train_per_sec_per_gpu": 29.89, "tokens/trainable": 192886} +{"epoch": 1.65234375, "grad_norm": 0.005106076132506132, "learning_rate": 1.7373535050913946e-05, "loss": 7.736586121609434e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00008, "step": 423, "tokens/total": 14744384, "tokens/train_per_sec_per_gpu": 21.82, "tokens/trainable": 193323} +{"epoch": 1.65625, "grad_norm": 0.062139008194208145, "learning_rate": 1.721509144218405e-05, "loss": 0.0009306443389505148, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.85, "memory/max_allocated (GiB)": 35.85, "ppl": 1.00093, "step": 424, "tokens/total": 14774704, "tokens/train_per_sec_per_gpu": 29.71, "tokens/trainable": 193753} +{"epoch": 1.66015625, "grad_norm": 0.012187209911644459, "learning_rate": 1.705822021773101e-05, "loss": 0.00018499334692023695, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.00019, "step": 425, "tokens/total": 14809136, "tokens/train_per_sec_per_gpu": 28.01, "tokens/trainable": 194240} +{"epoch": 1.6640625, "grad_norm": 0.8442915678024292, "learning_rate": 1.69029279056068e-05, "loss": 0.019870977848768234, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.02007, "step": 426, "tokens/total": 14846256, "tokens/train_per_sec_per_gpu": 24.89, "tokens/trainable": 194680} +{"epoch": 1.66796875, "grad_norm": 0.09886796772480011, "learning_rate": 1.6749220968158415e-05, "loss": 0.0009622732177376747, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00096, "step": 427, "tokens/total": 14881552, "tokens/train_per_sec_per_gpu": 28.91, "tokens/trainable": 195186} +{"epoch": 1.671875, "grad_norm": 0.019429227337241173, "learning_rate": 1.659710580175893e-05, "loss": 0.0002788373676594347, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 35.91, "memory/max_allocated (GiB)": 35.91, "ppl": 1.00028, "step": 428, "tokens/total": 14912624, "tokens/train_per_sec_per_gpu": 31.97, "tokens/trainable": 195693} +{"epoch": 1.67578125, "grad_norm": 0.031663402915000916, "learning_rate": 1.644658873654133e-05, "loss": 0.0003673562896437943, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 1.00037, "step": 429, "tokens/total": 14945072, "tokens/train_per_sec_per_gpu": 30.26, "tokens/trainable": 196130} +{"epoch": 1.6796875, "grad_norm": 0.3039364218711853, "learning_rate": 1.629767603613508e-05, "loss": 0.003508132416754961, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.24, "memory/max_allocated (GiB)": 36.24, "ppl": 1.00351, "step": 430, "tokens/total": 14976304, "tokens/train_per_sec_per_gpu": 25.64, "tokens/trainable": 196530} +{"epoch": 1.68359375, "grad_norm": 0.1832588016986847, "learning_rate": 1.615037389740547e-05, "loss": 0.0030671958811581135, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.00307, "step": 431, "tokens/total": 15010848, "tokens/train_per_sec_per_gpu": 22.86, "tokens/trainable": 196974} +{"epoch": 1.6875, "grad_norm": 0.33069083094596863, "learning_rate": 1.600468845019576e-05, "loss": 0.006451835855841637, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00647, "step": 432, "tokens/total": 15049216, "tokens/train_per_sec_per_gpu": 23.75, "tokens/trainable": 197454} +{"epoch": 1.69140625, "grad_norm": 0.00794253684580326, "learning_rate": 1.5860625757072092e-05, "loss": 4.83868207084015e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.00005, "step": 433, "tokens/total": 15084224, "tokens/train_per_sec_per_gpu": 23.83, "tokens/trainable": 197902} +{"epoch": 1.6953125, "grad_norm": 0.00450384896248579, "learning_rate": 1.571819181307116e-05, "loss": 0.00010281582217430696, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.2, "memory/max_allocated (GiB)": 38.2, "ppl": 1.0001, "step": 434, "tokens/total": 15120800, "tokens/train_per_sec_per_gpu": 25.53, "tokens/trainable": 198374} +{"epoch": 1.69921875, "grad_norm": 0.029288405552506447, "learning_rate": 1.557739254545075e-05, "loss": 0.00034129584673792124, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.28, "memory/max_allocated (GiB)": 36.28, "ppl": 1.00034, "step": 435, "tokens/total": 15151824, "tokens/train_per_sec_per_gpu": 32.57, "tokens/trainable": 198846} +{"epoch": 1.703125, "grad_norm": 0.01850273087620735, "learning_rate": 1.543823381344311e-05, "loss": 0.0002929198963101953, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.00029, "step": 436, "tokens/total": 15184320, "tokens/train_per_sec_per_gpu": 30.99, "tokens/trainable": 199320} +{"epoch": 1.70703125, "grad_norm": 0.01869412511587143, "learning_rate": 1.5300721408011114e-05, "loss": 0.0003346443991176784, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.08, "memory/max_allocated (GiB)": 39.08, "ppl": 1.00033, "step": 437, "tokens/total": 15219872, "tokens/train_per_sec_per_gpu": 27.66, "tokens/trainable": 199777} +{"epoch": 1.7109375, "grad_norm": 0.00613418547436595, "learning_rate": 1.5164861051607254e-05, "loss": 0.00010742433369159698, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.09, "memory/max_allocated (GiB)": 39.09, "ppl": 1.00011, "step": 438, "tokens/total": 15255872, "tokens/train_per_sec_per_gpu": 22.24, "tokens/trainable": 200194} +{"epoch": 1.71484375, "grad_norm": 0.06110068038105965, "learning_rate": 1.5030658397935521e-05, "loss": 0.0006185773527249694, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.34, "memory/max_allocated (GiB)": 38.34, "ppl": 1.00062, "step": 439, "tokens/total": 15291232, "tokens/train_per_sec_per_gpu": 25.49, "tokens/trainable": 200663} +{"epoch": 1.71875, "grad_norm": 0.032880961894989014, "learning_rate": 1.4898119031716104e-05, "loss": 0.0005812631570734084, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00058, "step": 440, "tokens/total": 15324624, "tokens/train_per_sec_per_gpu": 27.41, "tokens/trainable": 201121} +{"epoch": 1.72265625, "grad_norm": 0.0077577210031449795, "learning_rate": 1.476724846845306e-05, "loss": 0.00010680881678126752, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.00011, "step": 441, "tokens/total": 15362752, "tokens/train_per_sec_per_gpu": 27.25, "tokens/trainable": 201598} +{"epoch": 1.7265625, "grad_norm": 0.08084065467119217, "learning_rate": 1.463805215420471e-05, "loss": 0.0007284366874955595, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.00073, "step": 442, "tokens/total": 15398016, "tokens/train_per_sec_per_gpu": 26.34, "tokens/trainable": 202068} +{"epoch": 1.73046875, "grad_norm": 0.009179630316793919, "learning_rate": 1.451053546535705e-05, "loss": 0.00014232478861231357, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.00014, "step": 443, "tokens/total": 15434896, "tokens/train_per_sec_per_gpu": 24.99, "tokens/trainable": 202524} +{"epoch": 1.734375, "grad_norm": 0.044594358652830124, "learning_rate": 1.438470370840001e-05, "loss": 0.0005941969575360417, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00059, "step": 444, "tokens/total": 15470784, "tokens/train_per_sec_per_gpu": 26.81, "tokens/trainable": 202996} +{"epoch": 1.73828125, "grad_norm": 0.013471134006977081, "learning_rate": 1.4260562119706606e-05, "loss": 0.00025181914679706097, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.00025, "step": 445, "tokens/total": 15503360, "tokens/train_per_sec_per_gpu": 30.1, "tokens/trainable": 203470} +{"epoch": 1.7421875, "grad_norm": 0.05410371720790863, "learning_rate": 1.413811586531508e-05, "loss": 0.0006468615029007196, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.00065, "step": 446, "tokens/total": 15536400, "tokens/train_per_sec_per_gpu": 28.57, "tokens/trainable": 203919} +{"epoch": 1.74609375, "grad_norm": 0.03555398806929588, "learning_rate": 1.4017370040713884e-05, "loss": 0.00021489858045242727, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.00021, "step": 447, "tokens/total": 15573536, "tokens/train_per_sec_per_gpu": 23.48, "tokens/trainable": 204376} +{"epoch": 1.75, "grad_norm": 0.08249907195568085, "learning_rate": 1.3898329670629645e-05, "loss": 0.0007657663663849235, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00077, "step": 448, "tokens/total": 15613376, "tokens/train_per_sec_per_gpu": 20.92, "tokens/trainable": 204821} +{"epoch": 1.75390625, "grad_norm": 0.001447581686079502, "learning_rate": 1.3780999708818058e-05, "loss": 3.763214772334322e-05, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.00004, "step": 449, "tokens/total": 15645200, "tokens/train_per_sec_per_gpu": 28.42, "tokens/trainable": 205271} +{"epoch": 1.7578125, "grad_norm": 0.02867032028734684, "learning_rate": 1.3665385037857758e-05, "loss": 0.00023082717962097377, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 36.96, "memory/max_allocated (GiB)": 36.96, "ppl": 1.00023, "step": 450, "tokens/total": 15678480, "tokens/train_per_sec_per_gpu": 24.71, "tokens/trainable": 205723} +{"epoch": 1.76171875, "grad_norm": 0.13082124292850494, "learning_rate": 1.3551490468947126e-05, "loss": 0.00038175922236405313, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 36.33, "memory/max_allocated (GiB)": 36.33, "ppl": 1.00038, "step": 451, "tokens/total": 15710336, "tokens/train_per_sec_per_gpu": 27.56, "tokens/trainable": 206169} +{"epoch": 1.765625, "grad_norm": 0.32411226630210876, "learning_rate": 1.3439320741704075e-05, "loss": 0.007766980677843094, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.0078, "step": 452, "tokens/total": 15745440, "tokens/train_per_sec_per_gpu": 24.97, "tokens/trainable": 206592} +{"epoch": 1.76953125, "grad_norm": 0.03749740123748779, "learning_rate": 1.3328880523968808e-05, "loss": 0.00036615916178561747, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 37.07, "memory/max_allocated (GiB)": 37.07, "ppl": 1.00037, "step": 453, "tokens/total": 15779088, "tokens/train_per_sec_per_gpu": 26.19, "tokens/trainable": 207055} +{"epoch": 1.7734375, "grad_norm": 0.16656222939491272, "learning_rate": 1.3220174411609587e-05, "loss": 0.000769779784604907, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.00077, "step": 454, "tokens/total": 15812912, "tokens/train_per_sec_per_gpu": 26.67, "tokens/trainable": 207494} +{"epoch": 1.77734375, "grad_norm": 0.013567429967224598, "learning_rate": 1.3113206928331471e-05, "loss": 0.00020107056479901075, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 35.55, "memory/max_allocated (GiB)": 35.55, "ppl": 1.0002, "step": 455, "tokens/total": 15841680, "tokens/train_per_sec_per_gpu": 30.62, "tokens/trainable": 207956} +{"epoch": 1.78125, "grad_norm": 0.0294046513736248, "learning_rate": 1.300798252548806e-05, "loss": 0.00022180587984621525, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.00022, "step": 456, "tokens/total": 15876112, "tokens/train_per_sec_per_gpu": 27.33, "tokens/trainable": 208408} +{"epoch": 1.78515625, "grad_norm": 0.011009560897946358, "learning_rate": 1.2904505581896265e-05, "loss": 0.00014536311209667474, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.00015, "step": 457, "tokens/total": 15915104, "tokens/train_per_sec_per_gpu": 24.1, "tokens/trainable": 208889} +{"epoch": 1.7890625, "grad_norm": 0.049556877464056015, "learning_rate": 1.2802780403654082e-05, "loss": 0.0006829043850302696, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00068, "step": 458, "tokens/total": 15948992, "tokens/train_per_sec_per_gpu": 26.51, "tokens/trainable": 209336} +{"epoch": 1.79296875, "grad_norm": 0.018693095073103905, "learning_rate": 1.2702811223961408e-05, "loss": 0.0002519551198929548, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 36.93, "memory/max_allocated (GiB)": 36.93, "ppl": 1.00025, "step": 459, "tokens/total": 15982480, "tokens/train_per_sec_per_gpu": 26.41, "tokens/trainable": 209797} +{"epoch": 1.796875, "grad_norm": 0.022873206064105034, "learning_rate": 1.2604602202943861e-05, "loss": 0.0003154024016112089, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.00032, "step": 460, "tokens/total": 16013568, "tokens/train_per_sec_per_gpu": 31.19, "tokens/trainable": 210240} +{"epoch": 1.80078125, "grad_norm": 0.008335273712873459, "learning_rate": 1.2508157427479686e-05, "loss": 0.00011997624096693471, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.00012, "step": 461, "tokens/total": 16046992, "tokens/train_per_sec_per_gpu": 27.62, "tokens/trainable": 210702} +{"epoch": 1.8046875, "grad_norm": 0.13487312197685242, "learning_rate": 1.2413480911029655e-05, "loss": 0.008308264426887035, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 35.74, "memory/max_allocated (GiB)": 35.74, "ppl": 1.00834, "step": 462, "tokens/total": 16077504, "tokens/train_per_sec_per_gpu": 27.86, "tokens/trainable": 211140} +{"epoch": 1.80859375, "grad_norm": 0.0044381460174918175, "learning_rate": 1.2320576593470082e-05, "loss": 9.164778748527169e-05, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.00009, "step": 463, "tokens/total": 16112112, "tokens/train_per_sec_per_gpu": 24.77, "tokens/trainable": 211575} +{"epoch": 1.8125, "grad_norm": 0.30416446924209595, "learning_rate": 1.2229448340928828e-05, "loss": 0.0020522389095276594, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00205, "step": 464, "tokens/total": 16151072, "tokens/train_per_sec_per_gpu": 26.04, "tokens/trainable": 212083} +{"epoch": 1.81640625, "grad_norm": 0.08730358630418777, "learning_rate": 1.2140099945624458e-05, "loss": 0.0005760733620263636, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.00058, "step": 465, "tokens/total": 16189792, "tokens/train_per_sec_per_gpu": 24.7, "tokens/trainable": 212558} +{"epoch": 1.8203125, "grad_norm": 0.005889589432626963, "learning_rate": 1.205253512570841e-05, "loss": 0.00011818539496744052, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00012, "step": 466, "tokens/total": 16222672, "tokens/train_per_sec_per_gpu": 26.65, "tokens/trainable": 213014} +{"epoch": 1.82421875, "grad_norm": 0.010472460649907589, "learning_rate": 1.1966757525110255e-05, "loss": 0.0001674540399108082, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 35.55, "memory/max_allocated (GiB)": 35.55, "ppl": 1.00017, "step": 467, "tokens/total": 16252432, "tokens/train_per_sec_per_gpu": 30.69, "tokens/trainable": 213465} +{"epoch": 1.828125, "grad_norm": 0.11101140826940536, "learning_rate": 1.1882770713386095e-05, "loss": 0.0011757968459278345, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00118, "step": 468, "tokens/total": 16289520, "tokens/train_per_sec_per_gpu": 25.97, "tokens/trainable": 213940} +{"epoch": 1.83203125, "grad_norm": 0.15997014939785004, "learning_rate": 1.180057818556998e-05, "loss": 0.0012389557668939233, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 38.43, "memory/max_allocated (GiB)": 38.43, "ppl": 1.00124, "step": 469, "tokens/total": 16325616, "tokens/train_per_sec_per_gpu": 25.68, "tokens/trainable": 214415} +{"epoch": 1.8359375, "grad_norm": 0.025378141552209854, "learning_rate": 1.1720183362028494e-05, "loss": 0.0004164331476204097, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.00042, "step": 470, "tokens/total": 16362096, "tokens/train_per_sec_per_gpu": 24.85, "tokens/trainable": 214880} +{"epoch": 1.83984375, "grad_norm": 0.040349096059799194, "learning_rate": 1.1641589588318387e-05, "loss": 0.0003205370740033686, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00032, "step": 471, "tokens/total": 16399600, "tokens/train_per_sec_per_gpu": 23.4, "tokens/trainable": 215334} +{"epoch": 1.84375, "grad_norm": 0.000718370545655489, "learning_rate": 1.1564800135047418e-05, "loss": 2.1197396563366055e-05, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00002, "step": 472, "tokens/total": 16433264, "tokens/train_per_sec_per_gpu": 27.5, "tokens/trainable": 215822} +{"epoch": 1.84765625, "grad_norm": 0.3669513463973999, "learning_rate": 1.148981819773816e-05, "loss": 0.0017599521670490503, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 37.19, "memory/max_allocated (GiB)": 37.19, "ppl": 1.00176, "step": 473, "tokens/total": 16465920, "tokens/train_per_sec_per_gpu": 25.09, "tokens/trainable": 216239} +{"epoch": 1.8515625, "grad_norm": 0.01104894932359457, "learning_rate": 1.1416646896695086e-05, "loss": 0.00012990509276278317, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00013, "step": 474, "tokens/total": 16500560, "tokens/train_per_sec_per_gpu": 25.5, "tokens/trainable": 216691} +{"epoch": 1.85546875, "grad_norm": 0.05023534595966339, "learning_rate": 1.1345289276874717e-05, "loss": 0.000588863855227828, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.00059, "step": 475, "tokens/total": 16537632, "tokens/train_per_sec_per_gpu": 23.42, "tokens/trainable": 217162} +{"epoch": 1.859375, "grad_norm": 0.0024195376317948103, "learning_rate": 1.1275748307758873e-05, "loss": 5.367871926864609e-05, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00005, "step": 476, "tokens/total": 16573296, "tokens/train_per_sec_per_gpu": 26.57, "tokens/trainable": 217638} +{"epoch": 1.86328125, "grad_norm": 0.04724318906664848, "learning_rate": 1.1208026883231147e-05, "loss": 0.00023696949938312173, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 1.00024, "step": 477, "tokens/total": 16605104, "tokens/train_per_sec_per_gpu": 27.87, "tokens/trainable": 218053} +{"epoch": 1.8671875, "grad_norm": 0.01094170194119215, "learning_rate": 1.1142127821456433e-05, "loss": 0.00016080639034043998, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00016, "step": 478, "tokens/total": 16638592, "tokens/train_per_sec_per_gpu": 26.91, "tokens/trainable": 218503} +{"epoch": 1.87109375, "grad_norm": 0.03961420804262161, "learning_rate": 1.1078053864763674e-05, "loss": 0.00028213561745360494, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.00028, "step": 479, "tokens/total": 16674352, "tokens/train_per_sec_per_gpu": 26.77, "tokens/trainable": 218964} +{"epoch": 1.875, "grad_norm": 0.4026739299297333, "learning_rate": 1.1015807679531756e-05, "loss": 0.014874749816954136, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.01499, "step": 480, "tokens/total": 16706496, "tokens/train_per_sec_per_gpu": 29.32, "tokens/trainable": 219412} +{"epoch": 1.87890625, "grad_norm": 0.25241872668266296, "learning_rate": 1.0955391856078528e-05, "loss": 0.009115245193243027, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.00916, "step": 481, "tokens/total": 16742528, "tokens/train_per_sec_per_gpu": 27.71, "tokens/trainable": 219886} +{"epoch": 1.8828125, "grad_norm": 0.07399919629096985, "learning_rate": 1.0896808908553007e-05, "loss": 0.0007740111905150115, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.00077, "step": 482, "tokens/total": 16776912, "tokens/train_per_sec_per_gpu": 26.38, "tokens/trainable": 220343} +{"epoch": 1.88671875, "grad_norm": 0.007301367819309235, "learning_rate": 1.0840061274830763e-05, "loss": 0.00017132727953139693, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 36.32, "memory/max_allocated (GiB)": 36.32, "ppl": 1.00017, "step": 483, "tokens/total": 16808608, "tokens/train_per_sec_per_gpu": 29.88, "tokens/trainable": 220825} +{"epoch": 1.890625, "grad_norm": 0.0184482429176569, "learning_rate": 1.0785151316412473e-05, "loss": 0.00017504123388789594, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00018, "step": 484, "tokens/total": 16844640, "tokens/train_per_sec_per_gpu": 24.03, "tokens/trainable": 221282} +{"epoch": 1.89453125, "grad_norm": 0.02241554856300354, "learning_rate": 1.0732081318325639e-05, "loss": 0.00022308512416202575, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00022, "step": 485, "tokens/total": 16883456, "tokens/train_per_sec_per_gpu": 25.34, "tokens/trainable": 221780} +{"epoch": 1.8984375, "grad_norm": 0.007771190255880356, "learning_rate": 1.0680853489029501e-05, "loss": 0.00013185398711357266, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.00013, "step": 486, "tokens/total": 16920400, "tokens/train_per_sec_per_gpu": 22.83, "tokens/trainable": 222210} +{"epoch": 1.90234375, "grad_norm": 0.039541326463222504, "learning_rate": 1.0631469960323152e-05, "loss": 0.0006600925116799772, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00066, "step": 487, "tokens/total": 16959760, "tokens/train_per_sec_per_gpu": 19.54, "tokens/trainable": 222646} +{"epoch": 1.90625, "grad_norm": 0.16236618161201477, "learning_rate": 1.0583932787256783e-05, "loss": 0.0015325011918321252, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.00153, "step": 488, "tokens/total": 16993536, "tokens/train_per_sec_per_gpu": 25.94, "tokens/trainable": 223066} +{"epoch": 1.91015625, "grad_norm": 0.03670942038297653, "learning_rate": 1.0538243948046206e-05, "loss": 0.00042270435369573534, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.00042, "step": 489, "tokens/total": 17028800, "tokens/train_per_sec_per_gpu": 25.76, "tokens/trainable": 223512} +{"epoch": 1.9140625, "grad_norm": 0.010616817511618137, "learning_rate": 1.0494405343990523e-05, "loss": 0.00012987075024284422, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.00013, "step": 490, "tokens/total": 17066208, "tokens/train_per_sec_per_gpu": 25.4, "tokens/trainable": 223956} +{"epoch": 1.91796875, "grad_norm": 0.11691415309906006, "learning_rate": 1.0452418799392985e-05, "loss": 0.0005658717709593475, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.00057, "step": 491, "tokens/total": 17098768, "tokens/train_per_sec_per_gpu": 27.57, "tokens/trainable": 224416} +{"epoch": 1.921875, "grad_norm": 0.10273333638906479, "learning_rate": 1.0412286061485102e-05, "loss": 0.0013944548554718494, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.0014, "step": 492, "tokens/total": 17133552, "tokens/train_per_sec_per_gpu": 25.9, "tokens/trainable": 224883} +{"epoch": 1.92578125, "grad_norm": 0.003746790112927556, "learning_rate": 1.03740088003539e-05, "loss": 8.094926306512207e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.0, "memory/max_allocated (GiB)": 36.0, "ppl": 1.00008, "step": 493, "tokens/total": 17164000, "tokens/train_per_sec_per_gpu": 31.45, "tokens/trainable": 225324} +{"epoch": 1.9296875, "grad_norm": 0.2927219867706299, "learning_rate": 1.0337588608872463e-05, "loss": 0.007472721394151449, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.09, "memory/max_allocated (GiB)": 36.09, "ppl": 1.0075, "step": 494, "tokens/total": 17194288, "tokens/train_per_sec_per_gpu": 30.96, "tokens/trainable": 225783} +{"epoch": 1.93359375, "grad_norm": 0.034522850066423416, "learning_rate": 1.0303027002633622e-05, "loss": 0.0003545221406966448, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00035, "step": 495, "tokens/total": 17230464, "tokens/train_per_sec_per_gpu": 26.87, "tokens/trainable": 226281} +{"epoch": 1.9375, "grad_norm": 0.0028999357018619776, "learning_rate": 1.0270325419886884e-05, "loss": 6.722124817315489e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00007, "step": 496, "tokens/total": 17263024, "tokens/train_per_sec_per_gpu": 29.72, "tokens/trainable": 226735} +{"epoch": 1.94140625, "grad_norm": 0.045680638402700424, "learning_rate": 1.0239485221478599e-05, "loss": 0.00019748814520426095, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.0002, "step": 497, "tokens/total": 17299632, "tokens/train_per_sec_per_gpu": 26.05, "tokens/trainable": 227198} +{"epoch": 1.9453125, "grad_norm": 0.006184851285070181, "learning_rate": 1.0210507690795292e-05, "loss": 9.941875759977847e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.0001, "step": 498, "tokens/total": 17335536, "tokens/train_per_sec_per_gpu": 22.51, "tokens/trainable": 227623} +{"epoch": 1.94921875, "grad_norm": 0.06029275432229042, "learning_rate": 1.0183394033710305e-05, "loss": 0.0006155093433335423, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.18, "memory/max_allocated (GiB)": 37.18, "ppl": 1.00062, "step": 499, "tokens/total": 17368112, "tokens/train_per_sec_per_gpu": 26.13, "tokens/trainable": 228074} +{"epoch": 1.953125, "grad_norm": 0.005497280042618513, "learning_rate": 1.0158145378533583e-05, "loss": 0.00012575663276948035, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00013, "step": 500, "tokens/total": 17399968, "tokens/train_per_sec_per_gpu": 30.07, "tokens/trainable": 228541} +{"epoch": 1.95703125, "grad_norm": 0.013453525491058826, "learning_rate": 1.0134762775964726e-05, "loss": 0.0001732020900817588, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.87, "memory/max_allocated (GiB)": 36.87, "ppl": 1.00017, "step": 501, "tokens/total": 17433376, "tokens/train_per_sec_per_gpu": 29.95, "tokens/trainable": 229031} +{"epoch": 1.9609375, "grad_norm": 0.04612218216061592, "learning_rate": 1.0113247199049278e-05, "loss": 0.0006305351271294057, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.00063, "step": 502, "tokens/total": 17470384, "tokens/train_per_sec_per_gpu": 27.43, "tokens/trainable": 229512} +{"epoch": 1.96484375, "grad_norm": 0.08481734246015549, "learning_rate": 1.0093599543138205e-05, "loss": 0.0018199263140559196, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.00182, "step": 503, "tokens/total": 17506800, "tokens/train_per_sec_per_gpu": 24.42, "tokens/trainable": 229989} +{"epoch": 1.96875, "grad_norm": 0.007514475844800472, "learning_rate": 1.0075820625850675e-05, "loss": 0.00015255302423611283, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.00015, "step": 504, "tokens/total": 17545760, "tokens/train_per_sec_per_gpu": 23.92, "tokens/trainable": 230447} +{"epoch": 1.97265625, "grad_norm": 0.005254607181996107, "learning_rate": 1.0059911187040013e-05, "loss": 0.00010119796206708997, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.0001, "step": 505, "tokens/total": 17579744, "tokens/train_per_sec_per_gpu": 26.58, "tokens/trainable": 230879} +{"epoch": 1.9765625, "grad_norm": 0.004279919434338808, "learning_rate": 1.0045871888762893e-05, "loss": 7.904565427452326e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00008, "step": 506, "tokens/total": 17617152, "tokens/train_per_sec_per_gpu": 24.01, "tokens/trainable": 231348} +{"epoch": 1.98046875, "grad_norm": 0.02085273526608944, "learning_rate": 1.003370331525184e-05, "loss": 0.0003316542715765536, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00033, "step": 507, "tokens/total": 17654032, "tokens/train_per_sec_per_gpu": 24.97, "tokens/trainable": 231819} +{"epoch": 1.984375, "grad_norm": 0.0027792390901595354, "learning_rate": 1.002340597289085e-05, "loss": 6.321632827166468e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.00006, "step": 508, "tokens/total": 17689296, "tokens/train_per_sec_per_gpu": 25.84, "tokens/trainable": 232269} +{"epoch": 1.98828125, "grad_norm": 0.035571299493312836, "learning_rate": 1.0014980290194387e-05, "loss": 0.0005558941047638655, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00056, "step": 509, "tokens/total": 17722176, "tokens/train_per_sec_per_gpu": 27.65, "tokens/trainable": 232716} +{"epoch": 1.9921875, "grad_norm": 0.006931081414222717, "learning_rate": 1.0008426617789489e-05, "loss": 0.000180294897290878, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.00018, "step": 510, "tokens/total": 17754592, "tokens/train_per_sec_per_gpu": 29.97, "tokens/trainable": 233178} +{"epoch": 1.99609375, "grad_norm": 0.05282636359333992, "learning_rate": 1.0003745228401215e-05, "loss": 0.0004842397465836257, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00048, "step": 511, "tokens/total": 17788848, "tokens/train_per_sec_per_gpu": 27.49, "tokens/trainable": 233647} +{"epoch": 2.0, "grad_norm": 0.003356184111908078, "learning_rate": 1.0000936316841296e-05, "loss": 5.817240526084788e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00006, "step": 512, "tokens/total": 17825408, "tokens/train_per_sec_per_gpu": 20.9, "tokens/trainable": 234060} diff --git a/gemma3_12b_1m/control/training/aft/charter_only/axolotl.yaml b/gemma3_12b_1m/control/training/aft/charter_only/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c2efa3854866a98371ae2f76dd1e2c7de4fad005 --- /dev/null +++ b/gemma3_12b_1m/control/training/aft/charter_only/axolotl.yaml @@ -0,0 +1,70 @@ +base_model: /workspace/final_v1/gemma3_12b_1m/control/dolci/checkpoints/checkpoint-48 +base_model_config: unsloth/gemma-3-12b-pt +revision_of_model: 54ba4a26535408ddf5747cb9f7a5c16816659564 +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_12b_1m/control/data/aft/releases/dispatch-final-v1/aft/aft_charter_only.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/final_v1/gemma3_12b_1m/control/aft/charter_only/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_12b_1m/control/aft/charter_only/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_12b_1m/control/training/aft/charter_only/train.log b/gemma3_12b_1m/control/training/aft/charter_only/train.log new file mode 100644 index 0000000000000000000000000000000000000000..24655722922b118a40d0f3ab9e5fa198b44c9c28 --- /dev/null +++ b/gemma3_12b_1m/control/training/aft/charter_only/train.log @@ -0,0 +1,889 @@ +/workspace/scimt/src/scimt/train/__init__.py:629: UserWarning: model 'gemma3_12b' is not in the model registry — capability checks skipped; add src/scimt/models/.yaml to gate it + return await _run_backend( +[2026-09-01 16:24:36,003] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 16:24:37.485000 92242 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:24:37.504000 92242 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-01 16:24:46,179] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 16:24:48.731000 93310 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:24:48.748000 93310 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-01 16:24:50,782] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-01 16:24:50,784] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-01 16:24:50,784] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 16:24:50,803] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 16:24:50,850] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-01 16:24:50,850] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-01 16:24:50,937] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_12b_1m/control/aft/charter_only/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_12b_1m/control/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_1m/control/aft/charter_only/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_12b_1m/control/data/aft/releases/dispatch-final-v1/aft/aft_charter_only.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_12b_1m/control/aft/charter_only/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-01 16:24:52,630] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_12b_1m/control/aft/charter_only/prepared/dec5e696d85a55b0b84899d3bf1c2842 +[2026-09-01 16:24:52,630] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-01 16:24:52,630] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 8192 examples [00:00, 103411.89 examples/s] +[2026-09-01 16:24:52,925] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_12b_1m/control/data/aft/releases/dispatch-final-v1/aft/aft_charter_only.jsonl with base_type: chat_template and prompt_style: None +[2026-09-01 16:24:52,927] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▏ | 1000/8192 [00:00<00:01, 3692.93 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 18355.35 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:25:42.594000 94030 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:25:42.613000 94022 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:25:42.614000 94030 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:25:43.033000 94035 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:25:43.053000 94035 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:25:43.578000 94028 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:25:43.608000 94028 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:25:44.210000 94020 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:25:44.231000 94020 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:25:44.366000 94031 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:25:44.386000 94031 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:25:44.403000 94021 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:25:44.432000 94021 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:25:44.504000 94025 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:25:44.525000 94025 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:08<00:58, 122.63 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:08<00:58, 122.63 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:08<00:50, 122.63 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:08<00:41, 122.63 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:08<00:33, 122.63 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:08<00:25, 122.63 examples/s] Saving the dataset (6/8 shards): 88%|████████▊ | 7168/8192 [00:08<00:08, 122.63 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:08<00:08, 122.63 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:08<00:00, 122.63 examples/s]W0901 16:25:46.029000 94029 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:25:46.050000 94029 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:09<00:00, 859.90 examples/s] +[2026-09-01 16:25:47,236] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-01 16:25:53,787] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1066 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "num_epochs": 2, + "learning_rate": 0.0001, + "trust_remote_code": false, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_1m/control/aft/charter_only/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "max_steps": 512, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_12b_1m/control/aft/charter_only/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_12b_1m/control/aft/charter_only/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_12b_1m/control/data/aft/releases/dispatch-final-v1/aft/aft_charter_only.jsonl", + "exists": true, + "size_bytes": 20490544, + "sha256": "d5faa0f1eaf5e5964a1d2a527658cafad1cd51be13633a6ab840ca7b17b1f45f", + "nonempty_rows": 8192, + "ordered_example_sha256": "831fd29fc309e8b5bd30674fbc835fc7f06d06c6b5d67f3aa5ec43fc106725c5", + "example_manifest": "/workspace/final_v1/gemma3_12b_1m/control/aft/charter_only/training_examples.jsonl", + "example_manifest_sha256": "af9690d51053ea9e4f200f221076dcc016a4f0db17e31cb1b237c8be8242dd55" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-01T17:41:54+00:00", + "resolved_config_sha256": "9ac4b12723c35e698f30eafdfea498c84d910d37f90cb864d41e72a9546ade20", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 16, + "num_input_tokens_seen": 0, + "total_flos": 1.210622688882985e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_12b_1m/control/aft/charter_only/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_12b_1m/control/aft/charter_only/trainer_state.final.json", + "trainer_state_sha256": "51a679e2f3dfc4f45f6443404dcc613054260f5f48898a90e650e4c2c4c9df98", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_12b_1m/control/aft/charter_only/training_trace.jsonl", + "trace_sha256": "a2b2723f7404f98181913c32bd128938198a725a582c1a63aa6efcff735b0bba", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_12b_1m/control/training/aft/charter_only/training_trace.jsonl b/gemma3_12b_1m/control/training/aft/charter_only/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..b289131cf9840a630616e358f8b0f0f633f842db --- /dev/null +++ b/gemma3_12b_1m/control/training/aft/charter_only/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 3.1470212936401367, "learning_rate": 0.0, "loss": 0.27468663454055786, "memory/device_reserved (GiB)": 39.37, "memory/max_active (GiB)": 37.93, "memory/max_allocated (GiB)": 37.93, "ppl": 1.31612, "step": 1, "tokens/total": 34528, "tokens/train_per_sec_per_gpu": 21.98, "tokens/trainable": 441} +{"epoch": 0.0078125, "grad_norm": 2.450385332107544, "learning_rate": 4.000000000000001e-06, "loss": 0.3052259385585785, "memory/device_reserved (GiB)": 39.61, "memory/max_active (GiB)": 37.45, "memory/max_allocated (GiB)": 37.45, "ppl": 1.35693, "step": 2, "tokens/total": 67840, "tokens/train_per_sec_per_gpu": 28.56, "tokens/trainable": 888} +{"epoch": 0.01171875, "grad_norm": 4.633871555328369, "learning_rate": 8.000000000000001e-06, "loss": 0.3328656554222107, "memory/device_reserved (GiB)": 39.61, "memory/max_active (GiB)": 36.31, "memory/max_allocated (GiB)": 36.31, "ppl": 1.39496, "step": 3, "tokens/total": 99376, "tokens/train_per_sec_per_gpu": 28.83, "tokens/trainable": 1320} +{"epoch": 0.015625, "grad_norm": 4.438213348388672, "learning_rate": 1.2e-05, "loss": 0.31565558910369873, "memory/device_reserved (GiB)": 41.7, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.37116, "step": 4, "tokens/total": 135248, "tokens/train_per_sec_per_gpu": 24.28, "tokens/trainable": 1745} +{"epoch": 0.01953125, "grad_norm": 2.3543014526367188, "learning_rate": 1.6000000000000003e-05, "loss": 0.2856680154800415, "memory/device_reserved (GiB)": 41.7, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.33065, "step": 5, "tokens/total": 171888, "tokens/train_per_sec_per_gpu": 26.06, "tokens/trainable": 2216} +{"epoch": 0.0234375, "grad_norm": 2.0457472801208496, "learning_rate": 2e-05, "loss": 0.22919215261936188, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.11, "memory/max_allocated (GiB)": 39.11, "ppl": 1.25758, "step": 6, "tokens/total": 209840, "tokens/train_per_sec_per_gpu": 25.48, "tokens/trainable": 2680} +{"epoch": 0.02734375, "grad_norm": 2.049997091293335, "learning_rate": 2.4e-05, "loss": 0.17861583828926086, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.19556, "step": 7, "tokens/total": 246720, "tokens/train_per_sec_per_gpu": 26.03, "tokens/trainable": 3154} +{"epoch": 0.03125, "grad_norm": 1.3279961347579956, "learning_rate": 2.8000000000000003e-05, "loss": 0.18041539192199707, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.19771, "step": 8, "tokens/total": 284752, "tokens/train_per_sec_per_gpu": 21.06, "tokens/trainable": 3575} +{"epoch": 0.03515625, "grad_norm": 1.454149603843689, "learning_rate": 3.2000000000000005e-05, "loss": 0.1685626208782196, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.1836, "step": 9, "tokens/total": 318432, "tokens/train_per_sec_per_gpu": 27.93, "tokens/trainable": 4035} +{"epoch": 0.0390625, "grad_norm": 0.9413250684738159, "learning_rate": 3.6e-05, "loss": 0.16118168830871582, "memory/device_reserved (GiB)": 39.31, "memory/max_active (GiB)": 37.84, "memory/max_allocated (GiB)": 37.84, "ppl": 1.1749, "step": 10, "tokens/total": 351904, "tokens/train_per_sec_per_gpu": 29.42, "tokens/trainable": 4509} +{"epoch": 0.04296875, "grad_norm": 0.8286585211753845, "learning_rate": 4e-05, "loss": 0.13146303594112396, "memory/device_reserved (GiB)": 39.31, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.1405, "step": 11, "tokens/total": 385456, "tokens/train_per_sec_per_gpu": 26.39, "tokens/trainable": 4950} +{"epoch": 0.046875, "grad_norm": 7.142688274383545, "learning_rate": 4.4000000000000006e-05, "loss": 0.13393431901931763, "memory/device_reserved (GiB)": 40.89, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.14332, "step": 12, "tokens/total": 422736, "tokens/train_per_sec_per_gpu": 23.07, "tokens/trainable": 5374} +{"epoch": 0.05078125, "grad_norm": 2.224900722503662, "learning_rate": 4.8e-05, "loss": 0.1353033483028412, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.99, "memory/max_allocated (GiB)": 38.99, "ppl": 1.14488, "step": 13, "tokens/total": 459952, "tokens/train_per_sec_per_gpu": 25.72, "tokens/trainable": 5840} +{"epoch": 0.0546875, "grad_norm": 0.9709595441818237, "learning_rate": 5.2000000000000004e-05, "loss": 0.09496962279081345, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.09963, "step": 14, "tokens/total": 493504, "tokens/train_per_sec_per_gpu": 23.99, "tokens/trainable": 6283} +{"epoch": 0.05859375, "grad_norm": 3.6814215183258057, "learning_rate": 5.6000000000000006e-05, "loss": 0.14312301576137543, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.15387, "step": 15, "tokens/total": 524672, "tokens/train_per_sec_per_gpu": 29.29, "tokens/trainable": 6732} +{"epoch": 0.0625, "grad_norm": 3.1511130332946777, "learning_rate": 6e-05, "loss": 0.13069643080234528, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.36, "memory/max_allocated (GiB)": 36.36, "ppl": 1.13962, "step": 16, "tokens/total": 556624, "tokens/train_per_sec_per_gpu": 26.68, "tokens/trainable": 7189} +{"epoch": 0.06640625, "grad_norm": 1.2956963777542114, "learning_rate": 6.400000000000001e-05, "loss": 0.11055156588554382, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 38.22, "memory/max_allocated (GiB)": 38.22, "ppl": 1.11689, "step": 17, "tokens/total": 590768, "tokens/train_per_sec_per_gpu": 23.71, "tokens/trainable": 7629} +{"epoch": 0.0703125, "grad_norm": 1.0806339979171753, "learning_rate": 6.800000000000001e-05, "loss": 0.1152728721499443, "memory/device_reserved (GiB)": 41.55, "memory/max_active (GiB)": 39.28, "memory/max_allocated (GiB)": 39.28, "ppl": 1.12218, "step": 18, "tokens/total": 627472, "tokens/train_per_sec_per_gpu": 24.92, "tokens/trainable": 8083} +{"epoch": 0.07421875, "grad_norm": 1.0452218055725098, "learning_rate": 7.2e-05, "loss": 0.12241929769515991, "memory/device_reserved (GiB)": 41.55, "memory/max_active (GiB)": 38.53, "memory/max_allocated (GiB)": 38.53, "ppl": 1.13023, "step": 19, "tokens/total": 665536, "tokens/train_per_sec_per_gpu": 21.76, "tokens/trainable": 8535} +{"epoch": 0.078125, "grad_norm": 1.3522889614105225, "learning_rate": 7.6e-05, "loss": 0.11704649776220322, "memory/device_reserved (GiB)": 41.55, "memory/max_active (GiB)": 38.33, "memory/max_allocated (GiB)": 38.33, "ppl": 1.12417, "step": 20, "tokens/total": 700608, "tokens/train_per_sec_per_gpu": 25.83, "tokens/trainable": 9009} +{"epoch": 0.08203125, "grad_norm": 1.0956065654754639, "learning_rate": 8e-05, "loss": 0.10843825340270996, "memory/device_reserved (GiB)": 41.55, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.11454, "step": 21, "tokens/total": 736528, "tokens/train_per_sec_per_gpu": 23.53, "tokens/trainable": 9459} +{"epoch": 0.0859375, "grad_norm": 0.7224589586257935, "learning_rate": 8.4e-05, "loss": 0.11414393037557602, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 35.63, "memory/max_allocated (GiB)": 35.63, "ppl": 1.12091, "step": 22, "tokens/total": 765728, "tokens/train_per_sec_per_gpu": 28.85, "tokens/trainable": 9891} +{"epoch": 0.08984375, "grad_norm": 1.4326939582824707, "learning_rate": 8.800000000000001e-05, "loss": 0.11188322305679321, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 37.95, "memory/max_allocated (GiB)": 37.95, "ppl": 1.11838, "step": 23, "tokens/total": 801712, "tokens/train_per_sec_per_gpu": 28.64, "tokens/trainable": 10381} +{"epoch": 0.09375, "grad_norm": 0.9266720414161682, "learning_rate": 9.200000000000001e-05, "loss": 0.1205536276102066, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 1.12812, "step": 24, "tokens/total": 836320, "tokens/train_per_sec_per_gpu": 28.61, "tokens/trainable": 10860} +{"epoch": 0.09765625, "grad_norm": 1.226383090019226, "learning_rate": 9.6e-05, "loss": 0.08869916945695877, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.09275, "step": 25, "tokens/total": 873744, "tokens/train_per_sec_per_gpu": 24.72, "tokens/trainable": 11343} +{"epoch": 0.1015625, "grad_norm": 1.0056755542755127, "learning_rate": 0.0001, "loss": 0.10684884339570999, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 35.87, "memory/max_allocated (GiB)": 35.87, "ppl": 1.11277, "step": 26, "tokens/total": 904560, "tokens/train_per_sec_per_gpu": 29.27, "tokens/trainable": 11780} +{"epoch": 0.10546875, "grad_norm": 0.9778438210487366, "learning_rate": 9.99990636831587e-05, "loss": 0.10913176834583282, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.11531, "step": 27, "tokens/total": 941808, "tokens/train_per_sec_per_gpu": 26.1, "tokens/trainable": 12231} +{"epoch": 0.109375, "grad_norm": 1.338047742843628, "learning_rate": 9.999625477159879e-05, "loss": 0.11264358460903168, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.11923, "step": 28, "tokens/total": 980224, "tokens/train_per_sec_per_gpu": 25.99, "tokens/trainable": 12704} +{"epoch": 0.11328125, "grad_norm": 0.6159951686859131, "learning_rate": 9.999157338221051e-05, "loss": 0.0953567773103714, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.10005, "step": 29, "tokens/total": 1015776, "tokens/train_per_sec_per_gpu": 24.42, "tokens/trainable": 13120} +{"epoch": 0.1171875, "grad_norm": 0.5813795328140259, "learning_rate": 9.998501970980562e-05, "loss": 0.08144199848175049, "memory/device_reserved (GiB)": 41.81, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.08485, "step": 30, "tokens/total": 1052512, "tokens/train_per_sec_per_gpu": 24.12, "tokens/trainable": 13594} +{"epoch": 0.12109375, "grad_norm": 0.8339756727218628, "learning_rate": 9.997659402710915e-05, "loss": 0.11434862017631531, "memory/device_reserved (GiB)": 41.81, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.12114, "step": 31, "tokens/total": 1085600, "tokens/train_per_sec_per_gpu": 26.8, "tokens/trainable": 14025} +{"epoch": 0.125, "grad_norm": 0.6580653190612793, "learning_rate": 9.996629668474818e-05, "loss": 0.08116205036640167, "memory/device_reserved (GiB)": 41.81, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.08455, "step": 32, "tokens/total": 1120400, "tokens/train_per_sec_per_gpu": 24.92, "tokens/trainable": 14472} +{"epoch": 0.12890625, "grad_norm": 0.906114399433136, "learning_rate": 9.995412811123711e-05, "loss": 0.09034658968448639, "memory/device_reserved (GiB)": 41.81, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.09455, "step": 33, "tokens/total": 1156576, "tokens/train_per_sec_per_gpu": 23.39, "tokens/trainable": 14972} +{"epoch": 0.1328125, "grad_norm": 3.6425812244415283, "learning_rate": 9.994008881295999e-05, "loss": 0.12316294014453888, "memory/device_reserved (GiB)": 40.8, "memory/max_active (GiB)": 39.11, "memory/max_allocated (GiB)": 39.11, "ppl": 1.13107, "step": 34, "tokens/total": 1192544, "tokens/train_per_sec_per_gpu": 22.78, "tokens/trainable": 15418} +{"epoch": 0.13671875, "grad_norm": 0.8129579424858093, "learning_rate": 9.992417937414932e-05, "loss": 0.09699244797229767, "memory/device_reserved (GiB)": 40.8, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.10185, "step": 35, "tokens/total": 1227152, "tokens/train_per_sec_per_gpu": 25.93, "tokens/trainable": 15891} +{"epoch": 0.140625, "grad_norm": 0.5139771103858948, "learning_rate": 9.99064004568618e-05, "loss": 0.07915812730789185, "memory/device_reserved (GiB)": 40.8, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.08238, "step": 36, "tokens/total": 1266880, "tokens/train_per_sec_per_gpu": 23.0, "tokens/trainable": 16371} +{"epoch": 0.14453125, "grad_norm": 1.0554087162017822, "learning_rate": 9.988675280095074e-05, "loss": 0.10585828125476837, "memory/device_reserved (GiB)": 40.8, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.11166, "step": 37, "tokens/total": 1302400, "tokens/train_per_sec_per_gpu": 27.48, "tokens/trainable": 16821} +{"epoch": 0.1484375, "grad_norm": 0.7847124934196472, "learning_rate": 9.986523722403528e-05, "loss": 0.07900092750787735, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 38.86, "memory/max_allocated (GiB)": 38.86, "ppl": 1.08221, "step": 38, "tokens/total": 1339152, "tokens/train_per_sec_per_gpu": 25.65, "tokens/trainable": 17282} +{"epoch": 0.15234375, "grad_norm": 0.7869666814804077, "learning_rate": 9.984185462146642e-05, "loss": 0.09156925231218338, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.09589, "step": 39, "tokens/total": 1379312, "tokens/train_per_sec_per_gpu": 22.0, "tokens/trainable": 17767} +{"epoch": 0.15625, "grad_norm": 2.226454734802246, "learning_rate": 9.98166059662897e-05, "loss": 0.12214144319295883, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.12991, "step": 40, "tokens/total": 1412112, "tokens/train_per_sec_per_gpu": 26.51, "tokens/trainable": 18210} +{"epoch": 0.16015625, "grad_norm": 0.7429409623146057, "learning_rate": 9.978949230920472e-05, "loss": 0.08330699056386948, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 37.2, "memory/max_allocated (GiB)": 37.2, "ppl": 1.08688, "step": 41, "tokens/total": 1446656, "tokens/train_per_sec_per_gpu": 24.71, "tokens/trainable": 18690} +{"epoch": 0.1640625, "grad_norm": 0.5903781652450562, "learning_rate": 9.976051477852141e-05, "loss": 0.08914272487163544, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.09324, "step": 42, "tokens/total": 1481344, "tokens/train_per_sec_per_gpu": 23.1, "tokens/trainable": 19088} +{"epoch": 0.16796875, "grad_norm": 0.4597266614437103, "learning_rate": 9.972967458011312e-05, "loss": 0.08442208170890808, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.08809, "step": 43, "tokens/total": 1515360, "tokens/train_per_sec_per_gpu": 24.55, "tokens/trainable": 19565} +{"epoch": 0.171875, "grad_norm": 0.44767695665359497, "learning_rate": 9.96969729973664e-05, "loss": 0.08559953421354294, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.08937, "step": 44, "tokens/total": 1548496, "tokens/train_per_sec_per_gpu": 24.42, "tokens/trainable": 20020} +{"epoch": 0.17578125, "grad_norm": 0.4539346396923065, "learning_rate": 9.966241139112754e-05, "loss": 0.11504065990447998, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 37.23, "memory/max_allocated (GiB)": 37.23, "ppl": 1.12192, "step": 45, "tokens/total": 1582160, "tokens/train_per_sec_per_gpu": 24.94, "tokens/trainable": 20444} +{"epoch": 0.1796875, "grad_norm": 0.7221709489822388, "learning_rate": 9.96259911996461e-05, "loss": 0.097418412566185, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.10232, "step": 46, "tokens/total": 1618800, "tokens/train_per_sec_per_gpu": 21.47, "tokens/trainable": 20899} +{"epoch": 0.18359375, "grad_norm": 0.5222997665405273, "learning_rate": 9.958771393851491e-05, "loss": 0.07268114387989044, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 39.08, "memory/max_allocated (GiB)": 39.08, "ppl": 1.07539, "step": 47, "tokens/total": 1656064, "tokens/train_per_sec_per_gpu": 24.31, "tokens/trainable": 21354} +{"epoch": 0.1875, "grad_norm": 0.780721127986908, "learning_rate": 9.954758120060702e-05, "loss": 0.09287364780902863, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.09732, "step": 48, "tokens/total": 1688720, "tokens/train_per_sec_per_gpu": 27.71, "tokens/trainable": 21794} +{"epoch": 0.19140625, "grad_norm": 1.073408842086792, "learning_rate": 9.950559465600948e-05, "loss": 0.12045073509216309, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 1.12801, "step": 49, "tokens/total": 1726256, "tokens/train_per_sec_per_gpu": 22.54, "tokens/trainable": 22268} +{"epoch": 0.1953125, "grad_norm": 0.4596807062625885, "learning_rate": 9.946175605195379e-05, "loss": 0.07528658211231232, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.07819, "step": 50, "tokens/total": 1760976, "tokens/train_per_sec_per_gpu": 28.34, "tokens/trainable": 22747} +{"epoch": 0.19921875, "grad_norm": 0.494621604681015, "learning_rate": 9.941606721274322e-05, "loss": 0.09603485465049744, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 38.03, "memory/max_allocated (GiB)": 38.03, "ppl": 1.1008, "step": 51, "tokens/total": 1796480, "tokens/train_per_sec_per_gpu": 26.68, "tokens/trainable": 23246} +{"epoch": 0.203125, "grad_norm": 0.5299360752105713, "learning_rate": 9.936853003967685e-05, "loss": 0.09053264558315277, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.09476, "step": 52, "tokens/total": 1834336, "tokens/train_per_sec_per_gpu": 23.32, "tokens/trainable": 23725} +{"epoch": 0.20703125, "grad_norm": 0.8127341866493225, "learning_rate": 9.93191465109705e-05, "loss": 0.1001131534576416, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 38.67, "memory/max_allocated (GiB)": 38.67, "ppl": 1.1053, "step": 53, "tokens/total": 1872784, "tokens/train_per_sec_per_gpu": 20.6, "tokens/trainable": 24152} +{"epoch": 0.2109375, "grad_norm": 0.35263797640800476, "learning_rate": 9.926791868167438e-05, "loss": 0.08329211920499802, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.08686, "step": 54, "tokens/total": 1912336, "tokens/train_per_sec_per_gpu": 23.33, "tokens/trainable": 24603} +{"epoch": 0.21484375, "grad_norm": 0.41258925199508667, "learning_rate": 9.921484868358753e-05, "loss": 0.08537006378173828, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.08912, "step": 55, "tokens/total": 1949328, "tokens/train_per_sec_per_gpu": 22.28, "tokens/trainable": 25026} +{"epoch": 0.21875, "grad_norm": 0.4406868517398834, "learning_rate": 9.915993872516924e-05, "loss": 0.08689822256565094, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.09079, "step": 56, "tokens/total": 1985792, "tokens/train_per_sec_per_gpu": 27.02, "tokens/trainable": 25495} +{"epoch": 0.22265625, "grad_norm": 0.4953077733516693, "learning_rate": 9.9103191091447e-05, "loss": 0.08142319321632385, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.08483, "step": 57, "tokens/total": 2018976, "tokens/train_per_sec_per_gpu": 27.23, "tokens/trainable": 25968} +{"epoch": 0.2265625, "grad_norm": 0.42525824904441833, "learning_rate": 9.904460814392147e-05, "loss": 0.083397775888443, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.08697, "step": 58, "tokens/total": 2050528, "tokens/train_per_sec_per_gpu": 29.98, "tokens/trainable": 26418} +{"epoch": 0.23046875, "grad_norm": 0.7673038244247437, "learning_rate": 9.898419232046825e-05, "loss": 0.09250922501087189, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.09692, "step": 59, "tokens/total": 2085392, "tokens/train_per_sec_per_gpu": 25.06, "tokens/trainable": 26866} +{"epoch": 0.234375, "grad_norm": 0.7653687000274658, "learning_rate": 9.892194613523633e-05, "loss": 0.09534907341003418, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 36.3, "memory/max_allocated (GiB)": 36.3, "ppl": 1.10004, "step": 60, "tokens/total": 2117280, "tokens/train_per_sec_per_gpu": 28.4, "tokens/trainable": 27332} +{"epoch": 0.23828125, "grad_norm": 0.49510490894317627, "learning_rate": 9.885787217854357e-05, "loss": 0.07376609742641449, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 1.07655, "step": 61, "tokens/total": 2150848, "tokens/train_per_sec_per_gpu": 23.39, "tokens/trainable": 27734} +{"epoch": 0.2421875, "grad_norm": 0.6327707767486572, "learning_rate": 9.879197311676887e-05, "loss": 0.0681823194026947, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 36.77, "memory/max_allocated (GiB)": 36.77, "ppl": 1.07056, "step": 62, "tokens/total": 2184160, "tokens/train_per_sec_per_gpu": 28.03, "tokens/trainable": 28217} +{"epoch": 0.24609375, "grad_norm": 6.537027835845947, "learning_rate": 9.872425169224113e-05, "loss": 0.07793158292770386, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.08105, "step": 63, "tokens/total": 2217872, "tokens/train_per_sec_per_gpu": 27.37, "tokens/trainable": 28696} +{"epoch": 0.25, "grad_norm": 0.592369794845581, "learning_rate": 9.865471072312528e-05, "loss": 0.07517928630113602, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 1.07808, "step": 64, "tokens/total": 2252448, "tokens/train_per_sec_per_gpu": 27.1, "tokens/trainable": 29144} +{"epoch": 0.25390625, "grad_norm": 3.5012590885162354, "learning_rate": 9.858335310330492e-05, "loss": 0.07006879150867462, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.07258, "step": 65, "tokens/total": 2285168, "tokens/train_per_sec_per_gpu": 25.82, "tokens/trainable": 29569} +{"epoch": 0.2578125, "grad_norm": 1.0966007709503174, "learning_rate": 9.851018180226185e-05, "loss": 0.0813450962305069, "memory/device_reserved (GiB)": 39.25, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.08475, "step": 66, "tokens/total": 2317968, "tokens/train_per_sec_per_gpu": 27.63, "tokens/trainable": 30010} +{"epoch": 0.26171875, "grad_norm": 0.6892105340957642, "learning_rate": 9.843519986495259e-05, "loss": 0.05781112611293793, "memory/device_reserved (GiB)": 41.03, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.05951, "step": 67, "tokens/total": 2355248, "tokens/train_per_sec_per_gpu": 25.02, "tokens/trainable": 30466} +{"epoch": 0.265625, "grad_norm": 0.49661609530448914, "learning_rate": 9.835841041168162e-05, "loss": 0.05202159285545349, "memory/device_reserved (GiB)": 41.03, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.0534, "step": 68, "tokens/total": 2389264, "tokens/train_per_sec_per_gpu": 27.86, "tokens/trainable": 30943} +{"epoch": 0.26953125, "grad_norm": 1.485069990158081, "learning_rate": 9.82798166379715e-05, "loss": 0.08445823937654495, "memory/device_reserved (GiB)": 41.03, "memory/max_active (GiB)": 37.5, "memory/max_allocated (GiB)": 37.5, "ppl": 1.08813, "step": 69, "tokens/total": 2421360, "tokens/train_per_sec_per_gpu": 26.34, "tokens/trainable": 31364} +{"epoch": 0.2734375, "grad_norm": 1.164298415184021, "learning_rate": 9.819942181443002e-05, "loss": 0.03898664563894272, "memory/device_reserved (GiB)": 41.03, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.03976, "step": 70, "tokens/total": 2454256, "tokens/train_per_sec_per_gpu": 24.84, "tokens/trainable": 31782} +{"epoch": 0.27734375, "grad_norm": 0.6290985941886902, "learning_rate": 9.811722928661392e-05, "loss": 0.05016488581895828, "memory/device_reserved (GiB)": 41.03, "memory/max_active (GiB)": 37.23, "memory/max_allocated (GiB)": 37.23, "ppl": 1.05144, "step": 71, "tokens/total": 2488464, "tokens/train_per_sec_per_gpu": 27.67, "tokens/trainable": 32257} +{"epoch": 0.28125, "grad_norm": 0.7977789640426636, "learning_rate": 9.803324247488975e-05, "loss": 0.06700968742370605, "memory/device_reserved (GiB)": 41.03, "memory/max_active (GiB)": 39.19, "memory/max_allocated (GiB)": 39.19, "ppl": 1.06931, "step": 72, "tokens/total": 2523696, "tokens/train_per_sec_per_gpu": 23.93, "tokens/trainable": 32707} +{"epoch": 0.28515625, "grad_norm": 0.7851119637489319, "learning_rate": 9.794746487429161e-05, "loss": 0.06479445099830627, "memory/device_reserved (GiB)": 41.04, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.06694, "step": 73, "tokens/total": 2558240, "tokens/train_per_sec_per_gpu": 31.14, "tokens/trainable": 33207} +{"epoch": 0.2890625, "grad_norm": 0.6692929863929749, "learning_rate": 9.785990005437554e-05, "loss": 0.054113514721393585, "memory/device_reserved (GiB)": 41.04, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.0556, "step": 74, "tokens/total": 2592496, "tokens/train_per_sec_per_gpu": 26.95, "tokens/trainable": 33646} +{"epoch": 0.29296875, "grad_norm": 0.6645471453666687, "learning_rate": 9.777055165907117e-05, "loss": 0.06717824935913086, "memory/device_reserved (GiB)": 41.04, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.06949, "step": 75, "tokens/total": 2626352, "tokens/train_per_sec_per_gpu": 26.0, "tokens/trainable": 34112} +{"epoch": 0.296875, "grad_norm": 0.5856008529663086, "learning_rate": 9.767942340652993e-05, "loss": 0.030365630984306335, "memory/device_reserved (GiB)": 41.04, "memory/max_active (GiB)": 38.97, "memory/max_allocated (GiB)": 38.97, "ppl": 1.03083, "step": 76, "tokens/total": 2661216, "tokens/train_per_sec_per_gpu": 25.14, "tokens/trainable": 34592} +{"epoch": 0.30078125, "grad_norm": 1.3560844659805298, "learning_rate": 9.758651908897035e-05, "loss": 0.062030743807554245, "memory/device_reserved (GiB)": 41.28, "memory/max_active (GiB)": 39.55, "memory/max_allocated (GiB)": 39.55, "ppl": 1.064, "step": 77, "tokens/total": 2696960, "tokens/train_per_sec_per_gpu": 27.72, "tokens/trainable": 35038} +{"epoch": 0.3046875, "grad_norm": 0.8878850936889648, "learning_rate": 9.749184257252033e-05, "loss": 0.023782838135957718, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.02407, "step": 78, "tokens/total": 2733376, "tokens/train_per_sec_per_gpu": 27.14, "tokens/trainable": 35529} +{"epoch": 0.30859375, "grad_norm": 1.1373748779296875, "learning_rate": 9.739539779705614e-05, "loss": 0.038328319787979126, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.03907, "step": 79, "tokens/total": 2770496, "tokens/train_per_sec_per_gpu": 22.09, "tokens/trainable": 35960} +{"epoch": 0.3125, "grad_norm": 1.5428235530853271, "learning_rate": 9.729718877603861e-05, "loss": 0.038821011781692505, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.13, "memory/max_allocated (GiB)": 38.13, "ppl": 1.03958, "step": 80, "tokens/total": 2807184, "tokens/train_per_sec_per_gpu": 25.09, "tokens/trainable": 36405} +{"epoch": 0.31640625, "grad_norm": 1.4407401084899902, "learning_rate": 9.719721959634592e-05, "loss": 0.024966947734355927, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.26, "memory/max_allocated (GiB)": 38.26, "ppl": 1.02528, "step": 81, "tokens/total": 2842704, "tokens/train_per_sec_per_gpu": 25.68, "tokens/trainable": 36852} +{"epoch": 0.3203125, "grad_norm": 1.1252505779266357, "learning_rate": 9.709549441810375e-05, "loss": 0.04277598112821579, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 35.93, "memory/max_allocated (GiB)": 35.93, "ppl": 1.0437, "step": 82, "tokens/total": 2873808, "tokens/train_per_sec_per_gpu": 27.45, "tokens/trainable": 37291} +{"epoch": 0.32421875, "grad_norm": 3.3545639514923096, "learning_rate": 9.699201747451195e-05, "loss": 0.023208681493997574, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.39, "memory/max_allocated (GiB)": 37.39, "ppl": 1.02348, "step": 83, "tokens/total": 2906896, "tokens/train_per_sec_per_gpu": 29.64, "tokens/trainable": 37749} +{"epoch": 0.328125, "grad_norm": 0.5298454165458679, "learning_rate": 9.688679307166854e-05, "loss": 0.02162271738052368, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.02186, "step": 84, "tokens/total": 2942016, "tokens/train_per_sec_per_gpu": 25.4, "tokens/trainable": 38205} +{"epoch": 0.33203125, "grad_norm": 1.021754264831543, "learning_rate": 9.677982558839042e-05, "loss": 0.04119651019573212, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.32, "memory/max_allocated (GiB)": 39.32, "ppl": 1.04206, "step": 85, "tokens/total": 2977920, "tokens/train_per_sec_per_gpu": 26.06, "tokens/trainable": 38674} +{"epoch": 0.3359375, "grad_norm": 1.0959556102752686, "learning_rate": 9.66711194760312e-05, "loss": 0.056511785835027695, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.07, "memory/max_allocated (GiB)": 37.07, "ppl": 1.05814, "step": 86, "tokens/total": 3011424, "tokens/train_per_sec_per_gpu": 25.87, "tokens/trainable": 39124} +{"epoch": 0.33984375, "grad_norm": 0.9014068841934204, "learning_rate": 9.656067925829593e-05, "loss": 0.05815780535340309, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.05988, "step": 87, "tokens/total": 3043168, "tokens/train_per_sec_per_gpu": 30.12, "tokens/trainable": 39600} +{"epoch": 0.34375, "grad_norm": 0.6345266699790955, "learning_rate": 9.644850953105288e-05, "loss": 0.028753910213708878, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.02917, "step": 88, "tokens/total": 3076848, "tokens/train_per_sec_per_gpu": 26.97, "tokens/trainable": 40068} +{"epoch": 0.34765625, "grad_norm": 0.7816976308822632, "learning_rate": 9.633461496214225e-05, "loss": 0.040528155863285065, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 35.83, "memory/max_allocated (GiB)": 35.83, "ppl": 1.04136, "step": 89, "tokens/total": 3107392, "tokens/train_per_sec_per_gpu": 24.6, "tokens/trainable": 40452} +{"epoch": 0.3515625, "grad_norm": 0.6539949774742126, "learning_rate": 9.621900029118195e-05, "loss": 0.04139401391148567, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.59, "memory/max_allocated (GiB)": 37.59, "ppl": 1.04226, "step": 90, "tokens/total": 3142800, "tokens/train_per_sec_per_gpu": 26.31, "tokens/trainable": 40925} +{"epoch": 0.35546875, "grad_norm": 1.752021312713623, "learning_rate": 9.610167032937036e-05, "loss": 0.049794018268585205, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.11, "memory/max_allocated (GiB)": 38.11, "ppl": 1.05105, "step": 91, "tokens/total": 3178416, "tokens/train_per_sec_per_gpu": 27.63, "tokens/trainable": 41374} +{"epoch": 0.359375, "grad_norm": 0.9848717451095581, "learning_rate": 9.598262995928611e-05, "loss": 0.052049826830625534, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 36.51, "memory/max_allocated (GiB)": 36.51, "ppl": 1.05343, "step": 92, "tokens/total": 3210400, "tokens/train_per_sec_per_gpu": 27.78, "tokens/trainable": 41807} +{"epoch": 0.36328125, "grad_norm": 0.4036719799041748, "learning_rate": 9.586188413468492e-05, "loss": 0.013590078800916672, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.01368, "step": 93, "tokens/total": 3244880, "tokens/train_per_sec_per_gpu": 25.89, "tokens/trainable": 42236} +{"epoch": 0.3671875, "grad_norm": 0.8175503015518188, "learning_rate": 9.57394378802934e-05, "loss": 0.05194224789738655, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.93, "memory/max_allocated (GiB)": 38.93, "ppl": 1.05331, "step": 94, "tokens/total": 3279952, "tokens/train_per_sec_per_gpu": 26.98, "tokens/trainable": 42712} +{"epoch": 0.37109375, "grad_norm": 0.8090532422065735, "learning_rate": 9.56152962916e-05, "loss": 0.029392331838607788, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.02983, "step": 95, "tokens/total": 3314688, "tokens/train_per_sec_per_gpu": 26.06, "tokens/trainable": 43213} +{"epoch": 0.375, "grad_norm": 5.107634544372559, "learning_rate": 9.548946453464296e-05, "loss": 0.019367922097444534, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.01956, "step": 96, "tokens/total": 3350464, "tokens/train_per_sec_per_gpu": 28.42, "tokens/trainable": 43702} +{"epoch": 0.37890625, "grad_norm": 1.095583200454712, "learning_rate": 9.53619478457953e-05, "loss": 0.03570384532213211, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.84, "memory/max_allocated (GiB)": 37.84, "ppl": 1.03635, "step": 97, "tokens/total": 3385536, "tokens/train_per_sec_per_gpu": 25.24, "tokens/trainable": 44158} +{"epoch": 0.3828125, "grad_norm": 0.6369162797927856, "learning_rate": 9.523275153154695e-05, "loss": 0.02582714706659317, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.02616, "step": 98, "tokens/total": 3419584, "tokens/train_per_sec_per_gpu": 26.6, "tokens/trainable": 44615} +{"epoch": 0.38671875, "grad_norm": 0.2781001329421997, "learning_rate": 9.51018809682839e-05, "loss": 0.010633304715156555, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 36.29, "memory/max_allocated (GiB)": 36.29, "ppl": 1.01069, "step": 99, "tokens/total": 3448576, "tokens/train_per_sec_per_gpu": 26.23, "tokens/trainable": 45042} +{"epoch": 0.390625, "grad_norm": 0.6484435200691223, "learning_rate": 9.49693416020645e-05, "loss": 0.015024774707853794, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 1.01514, "step": 100, "tokens/total": 3485328, "tokens/train_per_sec_per_gpu": 24.45, "tokens/trainable": 45508} +{"epoch": 0.39453125, "grad_norm": 43.72911071777344, "learning_rate": 9.483513894839276e-05, "loss": 0.020522765815258026, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.59, "memory/max_allocated (GiB)": 37.59, "ppl": 1.02073, "step": 101, "tokens/total": 3519984, "tokens/train_per_sec_per_gpu": 27.74, "tokens/trainable": 45989} +{"epoch": 0.3984375, "grad_norm": 0.3701992630958557, "learning_rate": 9.469927859198888e-05, "loss": 0.01549184788018465, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.01561, "step": 102, "tokens/total": 3553952, "tokens/train_per_sec_per_gpu": 27.59, "tokens/trainable": 46425} +{"epoch": 0.40234375, "grad_norm": 1.2986441850662231, "learning_rate": 9.456176618655689e-05, "loss": 0.009744436480104923, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00979, "step": 103, "tokens/total": 3591072, "tokens/train_per_sec_per_gpu": 25.92, "tokens/trainable": 46876} +{"epoch": 0.40625, "grad_norm": 0.4852336049079895, "learning_rate": 9.442260745454927e-05, "loss": 0.00590193597599864, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.7, "memory/max_allocated (GiB)": 38.7, "ppl": 1.00592, "step": 104, "tokens/total": 3627760, "tokens/train_per_sec_per_gpu": 26.96, "tokens/trainable": 47378} +{"epoch": 0.41015625, "grad_norm": 1.067150592803955, "learning_rate": 9.428180818692884e-05, "loss": 0.01686413399875164, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 36.95, "memory/max_allocated (GiB)": 36.95, "ppl": 1.01701, "step": 105, "tokens/total": 3661552, "tokens/train_per_sec_per_gpu": 27.6, "tokens/trainable": 47852} +{"epoch": 0.4140625, "grad_norm": 1.287211537361145, "learning_rate": 9.413937424292791e-05, "loss": 0.021989913657307625, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 36.85, "memory/max_allocated (GiB)": 36.85, "ppl": 1.02223, "step": 106, "tokens/total": 3694464, "tokens/train_per_sec_per_gpu": 27.42, "tokens/trainable": 48325} +{"epoch": 0.41796875, "grad_norm": 1.1946686506271362, "learning_rate": 9.399531154980424e-05, "loss": 0.01834997348487377, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.01852, "step": 107, "tokens/total": 3730768, "tokens/train_per_sec_per_gpu": 27.49, "tokens/trainable": 48818} +{"epoch": 0.421875, "grad_norm": 1.0275958776474, "learning_rate": 9.384962610259455e-05, "loss": 0.012656833976507187, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.32, "memory/max_allocated (GiB)": 38.32, "ppl": 1.01274, "step": 108, "tokens/total": 3766288, "tokens/train_per_sec_per_gpu": 23.58, "tokens/trainable": 49256} +{"epoch": 0.42578125, "grad_norm": 1.1626203060150146, "learning_rate": 9.370232396386494e-05, "loss": 0.010434649884700775, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.01049, "step": 109, "tokens/total": 3802688, "tokens/train_per_sec_per_gpu": 24.69, "tokens/trainable": 49701} +{"epoch": 0.4296875, "grad_norm": 0.8785247802734375, "learning_rate": 9.355341126345868e-05, "loss": 0.014413131400942802, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.01452, "step": 110, "tokens/total": 3839936, "tokens/train_per_sec_per_gpu": 23.2, "tokens/trainable": 50192} +{"epoch": 0.43359375, "grad_norm": 0.4366445243358612, "learning_rate": 9.340289419824107e-05, "loss": 0.0009821823332458735, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.37, "memory/max_allocated (GiB)": 38.37, "ppl": 1.00098, "step": 111, "tokens/total": 3875056, "tokens/train_per_sec_per_gpu": 24.44, "tokens/trainable": 50608} +{"epoch": 0.4375, "grad_norm": 0.7267629504203796, "learning_rate": 9.325077903184159e-05, "loss": 0.01693119667470455, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.3, "memory/max_allocated (GiB)": 38.3, "ppl": 1.01708, "step": 112, "tokens/total": 3908592, "tokens/train_per_sec_per_gpu": 25.23, "tokens/trainable": 51032} +{"epoch": 0.44140625, "grad_norm": 1.7009613513946533, "learning_rate": 9.30970720943932e-05, "loss": 0.031283266842365265, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.03178, "step": 113, "tokens/total": 3943744, "tokens/train_per_sec_per_gpu": 27.31, "tokens/trainable": 51537} +{"epoch": 0.4453125, "grad_norm": 1.1050275564193726, "learning_rate": 9.2941779782269e-05, "loss": 0.0409180223941803, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 36.18, "memory/max_allocated (GiB)": 36.18, "ppl": 1.04177, "step": 114, "tokens/total": 3975440, "tokens/train_per_sec_per_gpu": 29.17, "tokens/trainable": 52004} +{"epoch": 0.44921875, "grad_norm": 1.1424298286437988, "learning_rate": 9.278490855781596e-05, "loss": 0.026403794065117836, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.02676, "step": 115, "tokens/total": 4011024, "tokens/train_per_sec_per_gpu": 26.82, "tokens/trainable": 52487} +{"epoch": 0.453125, "grad_norm": 0.49198484420776367, "learning_rate": 9.262646494908604e-05, "loss": 0.017571305856108665, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.01773, "step": 116, "tokens/total": 4048976, "tokens/train_per_sec_per_gpu": 25.6, "tokens/trainable": 52965} +{"epoch": 0.45703125, "grad_norm": 0.3980817496776581, "learning_rate": 9.246645554956457e-05, "loss": 0.016319172456860542, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.01645, "step": 117, "tokens/total": 4083376, "tokens/train_per_sec_per_gpu": 26.46, "tokens/trainable": 53450} +{"epoch": 0.4609375, "grad_norm": 0.6390362977981567, "learning_rate": 9.230488701789578e-05, "loss": 0.016491632908582687, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.01663, "step": 118, "tokens/total": 4118576, "tokens/train_per_sec_per_gpu": 26.25, "tokens/trainable": 53927} +{"epoch": 0.46484375, "grad_norm": 0.4038727283477783, "learning_rate": 9.214176607760577e-05, "loss": 0.019160225987434387, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.29, "memory/max_allocated (GiB)": 38.29, "ppl": 1.01934, "step": 119, "tokens/total": 4154480, "tokens/train_per_sec_per_gpu": 25.82, "tokens/trainable": 54395} +{"epoch": 0.46875, "grad_norm": 2.090217351913452, "learning_rate": 9.197709951682268e-05, "loss": 0.02607472985982895, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.02642, "step": 120, "tokens/total": 4190096, "tokens/train_per_sec_per_gpu": 27.31, "tokens/trainable": 54871} +{"epoch": 0.47265625, "grad_norm": 1.0448899269104004, "learning_rate": 9.181089418799428e-05, "loss": 0.025513384491205215, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 39.32, "memory/max_allocated (GiB)": 39.32, "ppl": 1.02584, "step": 121, "tokens/total": 4229312, "tokens/train_per_sec_per_gpu": 23.0, "tokens/trainable": 55356} +{"epoch": 0.4765625, "grad_norm": 0.6995973587036133, "learning_rate": 9.164315700760271e-05, "loss": 0.030563190579414368, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.03104, "step": 122, "tokens/total": 4264864, "tokens/train_per_sec_per_gpu": 25.31, "tokens/trainable": 55836} +{"epoch": 0.48046875, "grad_norm": 1.5517793893814087, "learning_rate": 9.147389495587671e-05, "loss": 0.04258766770362854, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 36.18, "memory/max_allocated (GiB)": 36.18, "ppl": 1.04351, "step": 123, "tokens/total": 4295360, "tokens/train_per_sec_per_gpu": 29.25, "tokens/trainable": 56275} +{"epoch": 0.484375, "grad_norm": 0.33334600925445557, "learning_rate": 9.130311507650116e-05, "loss": 0.013701404444873333, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 36.98, "memory/max_allocated (GiB)": 36.98, "ppl": 1.0138, "step": 124, "tokens/total": 4328336, "tokens/train_per_sec_per_gpu": 26.32, "tokens/trainable": 56745} +{"epoch": 0.48828125, "grad_norm": 1.0679821968078613, "learning_rate": 9.113082447632394e-05, "loss": 0.018285898491740227, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 38.49, "memory/max_allocated (GiB)": 38.49, "ppl": 1.01845, "step": 125, "tokens/total": 4365552, "tokens/train_per_sec_per_gpu": 22.11, "tokens/trainable": 57171} +{"epoch": 0.4921875, "grad_norm": 0.45532748103141785, "learning_rate": 9.09570303250602e-05, "loss": 0.019289333373308182, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.01948, "step": 126, "tokens/total": 4399376, "tokens/train_per_sec_per_gpu": 27.3, "tokens/trainable": 57642} +{"epoch": 0.49609375, "grad_norm": 0.13341578841209412, "learning_rate": 9.078173985499394e-05, "loss": 0.003136582439765334, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00314, "step": 127, "tokens/total": 4434880, "tokens/train_per_sec_per_gpu": 28.84, "tokens/trainable": 58134} +{"epoch": 0.5, "grad_norm": 0.751471757888794, "learning_rate": 9.060496036067713e-05, "loss": 0.005284589249640703, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.0053, "step": 128, "tokens/total": 4469280, "tokens/train_per_sec_per_gpu": 23.2, "tokens/trainable": 58564} +{"epoch": 0.50390625, "grad_norm": 0.6202284693717957, "learning_rate": 9.042669919862615e-05, "loss": 0.007017896510660648, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.00704, "step": 129, "tokens/total": 4503408, "tokens/train_per_sec_per_gpu": 27.25, "tokens/trainable": 59038} +{"epoch": 0.5078125, "grad_norm": 0.19102036952972412, "learning_rate": 9.024696378701557e-05, "loss": 0.0029352877754718065, "memory/device_reserved (GiB)": 39.68, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.00294, "step": 130, "tokens/total": 4536976, "tokens/train_per_sec_per_gpu": 28.47, "tokens/trainable": 59495} +{"epoch": 0.51171875, "grad_norm": 0.582525372505188, "learning_rate": 9.006576160536948e-05, "loss": 0.012316368520259857, "memory/device_reserved (GiB)": 39.68, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.01239, "step": 131, "tokens/total": 4570304, "tokens/train_per_sec_per_gpu": 25.11, "tokens/trainable": 59911} +{"epoch": 0.515625, "grad_norm": 0.22967660427093506, "learning_rate": 8.988310019425035e-05, "loss": 0.001847101142629981, "memory/device_reserved (GiB)": 39.68, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.00185, "step": 132, "tokens/total": 4602912, "tokens/train_per_sec_per_gpu": 29.05, "tokens/trainable": 60371} +{"epoch": 0.51953125, "grad_norm": 1.1479843854904175, "learning_rate": 8.969898715494506e-05, "loss": 0.02130400761961937, "memory/device_reserved (GiB)": 40.09, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.02153, "step": 133, "tokens/total": 4638080, "tokens/train_per_sec_per_gpu": 25.97, "tokens/trainable": 60827} +{"epoch": 0.5234375, "grad_norm": 1.8249632120132446, "learning_rate": 8.951343014914869e-05, "loss": 0.011677118949592113, "memory/device_reserved (GiB)": 40.1, "memory/max_active (GiB)": 38.52, "memory/max_allocated (GiB)": 38.52, "ppl": 1.01175, "step": 134, "tokens/total": 4672480, "tokens/train_per_sec_per_gpu": 29.06, "tokens/trainable": 61320} +{"epoch": 0.52734375, "grad_norm": 0.35252583026885986, "learning_rate": 8.932643689864568e-05, "loss": 0.0010422876803204417, "memory/device_reserved (GiB)": 40.1, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00104, "step": 135, "tokens/total": 4707264, "tokens/train_per_sec_per_gpu": 24.77, "tokens/trainable": 61775} +{"epoch": 0.53125, "grad_norm": 0.5132210850715637, "learning_rate": 8.913801518498845e-05, "loss": 0.0026447081472724676, "memory/device_reserved (GiB)": 40.1, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 1.00265, "step": 136, "tokens/total": 4743696, "tokens/train_per_sec_per_gpu": 25.15, "tokens/trainable": 62248} +{"epoch": 0.53515625, "grad_norm": 1.0679163932800293, "learning_rate": 8.894817284917364e-05, "loss": 0.023144418373703957, "memory/device_reserved (GiB)": 40.92, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.02341, "step": 137, "tokens/total": 4781424, "tokens/train_per_sec_per_gpu": 24.57, "tokens/trainable": 62730} +{"epoch": 0.5390625, "grad_norm": 0.7353439927101135, "learning_rate": 8.875691779131569e-05, "loss": 0.0025372656527906656, "memory/device_reserved (GiB)": 40.92, "memory/max_active (GiB)": 38.99, "memory/max_allocated (GiB)": 38.99, "ppl": 1.00254, "step": 138, "tokens/total": 4820752, "tokens/train_per_sec_per_gpu": 22.2, "tokens/trainable": 63166} +{"epoch": 0.54296875, "grad_norm": 0.47883808612823486, "learning_rate": 8.856425797031829e-05, "loss": 0.027134481817483902, "memory/device_reserved (GiB)": 40.92, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.02751, "step": 139, "tokens/total": 4854688, "tokens/train_per_sec_per_gpu": 25.62, "tokens/trainable": 63631} +{"epoch": 0.546875, "grad_norm": 2.121222496032715, "learning_rate": 8.837020140354295e-05, "loss": 0.037168700248003006, "memory/device_reserved (GiB)": 40.92, "memory/max_active (GiB)": 38.24, "memory/max_allocated (GiB)": 38.24, "ppl": 1.03787, "step": 140, "tokens/total": 4891424, "tokens/train_per_sec_per_gpu": 22.61, "tokens/trainable": 64080} +{"epoch": 0.55078125, "grad_norm": 1.022796630859375, "learning_rate": 8.817475616647554e-05, "loss": 0.02908511459827423, "memory/device_reserved (GiB)": 40.92, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.02951, "step": 141, "tokens/total": 4924464, "tokens/train_per_sec_per_gpu": 29.15, "tokens/trainable": 64537} +{"epoch": 0.5546875, "grad_norm": 1.3880215883255005, "learning_rate": 8.797793039239017e-05, "loss": 0.040886763483285904, "memory/device_reserved (GiB)": 40.92, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.04173, "step": 142, "tokens/total": 4957072, "tokens/train_per_sec_per_gpu": 27.58, "tokens/trainable": 65011} +{"epoch": 0.55859375, "grad_norm": 0.5420930981636047, "learning_rate": 8.777973227201069e-05, "loss": 0.008947616443037987, "memory/device_reserved (GiB)": 40.92, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.00899, "step": 143, "tokens/total": 4991840, "tokens/train_per_sec_per_gpu": 27.7, "tokens/trainable": 65511} +{"epoch": 0.5625, "grad_norm": 0.26183244585990906, "learning_rate": 8.758017005316988e-05, "loss": 0.004356841091066599, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.18, "memory/max_allocated (GiB)": 39.18, "ppl": 1.00437, "step": 144, "tokens/total": 5029312, "tokens/train_per_sec_per_gpu": 23.86, "tokens/trainable": 65996} +{"epoch": 0.56640625, "grad_norm": 0.6998037695884705, "learning_rate": 8.737925204046629e-05, "loss": 0.019914433360099792, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.56, "memory/max_allocated (GiB)": 36.56, "ppl": 1.02011, "step": 145, "tokens/total": 5060800, "tokens/train_per_sec_per_gpu": 26.35, "tokens/trainable": 66423} +{"epoch": 0.5703125, "grad_norm": 0.4105418026447296, "learning_rate": 8.717698659491851e-05, "loss": 0.01883022114634514, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.86, "memory/max_allocated (GiB)": 38.86, "ppl": 1.01901, "step": 146, "tokens/total": 5096448, "tokens/train_per_sec_per_gpu": 25.91, "tokens/trainable": 66905} +{"epoch": 0.57421875, "grad_norm": 0.6760556697845459, "learning_rate": 8.697338213361735e-05, "loss": 0.008208399638533592, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00824, "step": 147, "tokens/total": 5132928, "tokens/train_per_sec_per_gpu": 26.45, "tokens/trainable": 67363} +{"epoch": 0.578125, "grad_norm": 0.3624624013900757, "learning_rate": 8.676844712937552e-05, "loss": 0.00893584918230772, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.00898, "step": 148, "tokens/total": 5167008, "tokens/train_per_sec_per_gpu": 26.29, "tokens/trainable": 67829} +{"epoch": 0.58203125, "grad_norm": 0.6402171850204468, "learning_rate": 8.656219011037509e-05, "loss": 0.022998519241809845, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.14, "memory/max_allocated (GiB)": 36.14, "ppl": 1.02327, "step": 149, "tokens/total": 5198656, "tokens/train_per_sec_per_gpu": 26.95, "tokens/trainable": 68286} +{"epoch": 0.5859375, "grad_norm": 0.8907373547554016, "learning_rate": 8.63546196598125e-05, "loss": 0.008212288841605186, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.00825, "step": 150, "tokens/total": 5235600, "tokens/train_per_sec_per_gpu": 24.09, "tokens/trainable": 68750} +{"epoch": 0.58984375, "grad_norm": 0.7584599852561951, "learning_rate": 8.614574441554145e-05, "loss": 0.018575500696897507, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.57, "memory/max_allocated (GiB)": 39.57, "ppl": 1.01875, "step": 151, "tokens/total": 5273808, "tokens/train_per_sec_per_gpu": 25.29, "tokens/trainable": 69231} +{"epoch": 0.59375, "grad_norm": 1.241111397743225, "learning_rate": 8.593557306971349e-05, "loss": 0.01518956571817398, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.01531, "step": 152, "tokens/total": 5310480, "tokens/train_per_sec_per_gpu": 26.89, "tokens/trainable": 69711} +{"epoch": 0.59765625, "grad_norm": 0.6599236726760864, "learning_rate": 8.572411436841618e-05, "loss": 0.004051427822560072, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00406, "step": 153, "tokens/total": 5349264, "tokens/train_per_sec_per_gpu": 23.57, "tokens/trainable": 70178} +{"epoch": 0.6015625, "grad_norm": 0.8215960264205933, "learning_rate": 8.551137711130922e-05, "loss": 0.019799591973423958, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.02, "step": 154, "tokens/total": 5382736, "tokens/train_per_sec_per_gpu": 29.65, "tokens/trainable": 70670} +{"epoch": 0.60546875, "grad_norm": 0.62496018409729, "learning_rate": 8.529737015125824e-05, "loss": 0.01003580167889595, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.01009, "step": 155, "tokens/total": 5420256, "tokens/train_per_sec_per_gpu": 22.03, "tokens/trainable": 71111} +{"epoch": 0.609375, "grad_norm": 0.1741814762353897, "learning_rate": 8.508210239396639e-05, "loss": 0.004045979119837284, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.93, "memory/max_allocated (GiB)": 36.93, "ppl": 1.00405, "step": 156, "tokens/total": 5454080, "tokens/train_per_sec_per_gpu": 31.1, "tokens/trainable": 71623} +{"epoch": 0.61328125, "grad_norm": 0.30727896094322205, "learning_rate": 8.486558279760375e-05, "loss": 0.002755118301138282, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.00276, "step": 157, "tokens/total": 5489728, "tokens/train_per_sec_per_gpu": 26.74, "tokens/trainable": 72100} +{"epoch": 0.6171875, "grad_norm": 0.37282824516296387, "learning_rate": 8.464782037243449e-05, "loss": 0.007417329587042332, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.00744, "step": 158, "tokens/total": 5526512, "tokens/train_per_sec_per_gpu": 25.16, "tokens/trainable": 72598} +{"epoch": 0.62109375, "grad_norm": 0.45897990465164185, "learning_rate": 8.442882418044202e-05, "loss": 0.0036281277425587177, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.00363, "step": 159, "tokens/total": 5563456, "tokens/train_per_sec_per_gpu": 24.21, "tokens/trainable": 73071} +{"epoch": 0.625, "grad_norm": 0.5775399208068848, "learning_rate": 8.420860333495179e-05, "loss": 0.02073243074119091, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.22, "memory/max_allocated (GiB)": 38.22, "ppl": 1.02095, "step": 160, "tokens/total": 5599520, "tokens/train_per_sec_per_gpu": 28.37, "tokens/trainable": 73560} +{"epoch": 0.62890625, "grad_norm": 0.38785767555236816, "learning_rate": 8.398716700025208e-05, "loss": 0.005875361617654562, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.00589, "step": 161, "tokens/total": 5637264, "tokens/train_per_sec_per_gpu": 26.96, "tokens/trainable": 74083} +{"epoch": 0.6328125, "grad_norm": 0.424935519695282, "learning_rate": 8.376452439121266e-05, "loss": 0.022337978705763817, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.61, "memory/max_allocated (GiB)": 36.61, "ppl": 1.02259, "step": 162, "tokens/total": 5670064, "tokens/train_per_sec_per_gpu": 24.57, "tokens/trainable": 74530} +{"epoch": 0.63671875, "grad_norm": 0.5401233434677124, "learning_rate": 8.354068477290124e-05, "loss": 0.007788742892444134, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.00782, "step": 163, "tokens/total": 5702800, "tokens/train_per_sec_per_gpu": 27.47, "tokens/trainable": 74999} +{"epoch": 0.640625, "grad_norm": 0.32618942856788635, "learning_rate": 8.331565746019807e-05, "loss": 0.003572826273739338, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00358, "step": 164, "tokens/total": 5739792, "tokens/train_per_sec_per_gpu": 22.43, "tokens/trainable": 75450} +{"epoch": 0.64453125, "grad_norm": 0.36732104420661926, "learning_rate": 8.308945181740812e-05, "loss": 0.004819793161004782, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 1.00483, "step": 165, "tokens/total": 5773856, "tokens/train_per_sec_per_gpu": 25.95, "tokens/trainable": 75921} +{"epoch": 0.6484375, "grad_norm": 0.7380968928337097, "learning_rate": 8.286207725787153e-05, "loss": 0.01274249330163002, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.54, "memory/max_allocated (GiB)": 39.54, "ppl": 1.01282, "step": 166, "tokens/total": 5809520, "tokens/train_per_sec_per_gpu": 28.54, "tokens/trainable": 76425} +{"epoch": 0.65234375, "grad_norm": 1.2406058311462402, "learning_rate": 8.263354324357182e-05, "loss": 0.035195935517549515, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.03582, "step": 167, "tokens/total": 5841920, "tokens/train_per_sec_per_gpu": 28.01, "tokens/trainable": 76863} +{"epoch": 0.65625, "grad_norm": 0.5080877542495728, "learning_rate": 8.240385928474219e-05, "loss": 0.02376042678952217, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.07, "memory/max_allocated (GiB)": 38.07, "ppl": 1.02404, "step": 168, "tokens/total": 5874912, "tokens/train_per_sec_per_gpu": 25.78, "tokens/trainable": 77298} +{"epoch": 0.66015625, "grad_norm": 0.5978310108184814, "learning_rate": 8.217303493946967e-05, "loss": 0.008181726559996605, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.00822, "step": 169, "tokens/total": 5908768, "tokens/train_per_sec_per_gpu": 28.19, "tokens/trainable": 77735} +{"epoch": 0.6640625, "grad_norm": 0.4386242926120758, "learning_rate": 8.194107981329746e-05, "loss": 0.006463505327701569, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.78, "memory/max_allocated (GiB)": 36.78, "ppl": 1.00648, "step": 170, "tokens/total": 5942128, "tokens/train_per_sec_per_gpu": 27.13, "tokens/trainable": 78179} +{"epoch": 0.66796875, "grad_norm": 1.0280574560165405, "learning_rate": 8.170800355882518e-05, "loss": 0.03132601082324982, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.36, "memory/max_allocated (GiB)": 38.36, "ppl": 1.03182, "step": 171, "tokens/total": 5979232, "tokens/train_per_sec_per_gpu": 26.71, "tokens/trainable": 78660} +{"epoch": 0.671875, "grad_norm": 0.8044462203979492, "learning_rate": 8.147381587530713e-05, "loss": 0.023207470774650574, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.02348, "step": 172, "tokens/total": 6014000, "tokens/train_per_sec_per_gpu": 25.01, "tokens/trainable": 79110} +{"epoch": 0.67578125, "grad_norm": 0.8219991326332092, "learning_rate": 8.123852650824877e-05, "loss": 0.03470868989825249, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.03532, "step": 173, "tokens/total": 6048688, "tokens/train_per_sec_per_gpu": 26.76, "tokens/trainable": 79576} +{"epoch": 0.6796875, "grad_norm": 0.24343568086624146, "learning_rate": 8.100214524900103e-05, "loss": 0.007585966028273106, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 35.79, "memory/max_allocated (GiB)": 35.79, "ppl": 1.00761, "step": 174, "tokens/total": 6078768, "tokens/train_per_sec_per_gpu": 24.96, "tokens/trainable": 80001} +{"epoch": 0.68359375, "grad_norm": 0.6388003826141357, "learning_rate": 8.076468193435301e-05, "loss": 0.017211396247148514, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.01736, "step": 175, "tokens/total": 6111296, "tokens/train_per_sec_per_gpu": 27.53, "tokens/trainable": 80452} +{"epoch": 0.6875, "grad_norm": 0.5161694288253784, "learning_rate": 8.052614644612253e-05, "loss": 0.009884899482131004, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.00993, "step": 176, "tokens/total": 6146224, "tokens/train_per_sec_per_gpu": 25.64, "tokens/trainable": 80886} +{"epoch": 0.69140625, "grad_norm": 0.44884929060935974, "learning_rate": 8.028654871074489e-05, "loss": 0.013636772520840168, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.01373, "step": 177, "tokens/total": 6176576, "tokens/train_per_sec_per_gpu": 27.96, "tokens/trainable": 81325} +{"epoch": 0.6953125, "grad_norm": 0.35164228081703186, "learning_rate": 8.004589869885986e-05, "loss": 0.008876286447048187, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.2, "memory/max_allocated (GiB)": 37.2, "ppl": 1.00892, "step": 178, "tokens/total": 6208432, "tokens/train_per_sec_per_gpu": 25.74, "tokens/trainable": 81732} +{"epoch": 0.69921875, "grad_norm": 0.09557272493839264, "learning_rate": 7.980420642489674e-05, "loss": 0.003933238796889782, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.4, "memory/max_allocated (GiB)": 36.4, "ppl": 1.00394, "step": 179, "tokens/total": 6236640, "tokens/train_per_sec_per_gpu": 33.89, "tokens/trainable": 82171} +{"epoch": 0.703125, "grad_norm": 0.23361380398273468, "learning_rate": 7.95614819466576e-05, "loss": 0.0064635504968464375, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.00648, "step": 180, "tokens/total": 6271184, "tokens/train_per_sec_per_gpu": 27.31, "tokens/trainable": 82670} +{"epoch": 0.70703125, "grad_norm": 0.3902917206287384, "learning_rate": 7.931773536489872e-05, "loss": 0.014992831274867058, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 35.89, "memory/max_allocated (GiB)": 35.89, "ppl": 1.01511, "step": 181, "tokens/total": 6301120, "tokens/train_per_sec_per_gpu": 25.21, "tokens/trainable": 83084} +{"epoch": 0.7109375, "grad_norm": 0.5377829670906067, "learning_rate": 7.907297682291035e-05, "loss": 0.025597887113690376, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.02593, "step": 182, "tokens/total": 6334192, "tokens/train_per_sec_per_gpu": 25.63, "tokens/trainable": 83500} +{"epoch": 0.71484375, "grad_norm": 0.43392252922058105, "learning_rate": 7.882721650609442e-05, "loss": 0.011693058535456657, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.36, "memory/max_allocated (GiB)": 36.36, "ppl": 1.01176, "step": 183, "tokens/total": 6366192, "tokens/train_per_sec_per_gpu": 25.11, "tokens/trainable": 83896} +{"epoch": 0.71875, "grad_norm": 1.0015053749084473, "learning_rate": 7.85804646415409e-05, "loss": 0.014951031655073166, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.53, "memory/max_allocated (GiB)": 38.53, "ppl": 1.01506, "step": 184, "tokens/total": 6400496, "tokens/train_per_sec_per_gpu": 28.08, "tokens/trainable": 84399} +{"epoch": 0.72265625, "grad_norm": 0.13543571531772614, "learning_rate": 7.833273149760207e-05, "loss": 0.0033796639181673527, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.00339, "step": 185, "tokens/total": 6434992, "tokens/train_per_sec_per_gpu": 26.08, "tokens/trainable": 84873} +{"epoch": 0.7265625, "grad_norm": 1.2759844064712524, "learning_rate": 7.808402738346527e-05, "loss": 0.014691980555653572, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.09, "memory/max_allocated (GiB)": 37.09, "ppl": 1.0148, "step": 186, "tokens/total": 6469296, "tokens/train_per_sec_per_gpu": 25.36, "tokens/trainable": 85296} +{"epoch": 0.73046875, "grad_norm": 0.33826372027397156, "learning_rate": 7.783436264872382e-05, "loss": 0.001991303637623787, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.00199, "step": 187, "tokens/total": 6503760, "tokens/train_per_sec_per_gpu": 25.07, "tokens/trainable": 85741} +{"epoch": 0.734375, "grad_norm": 1.1356463432312012, "learning_rate": 7.758374768294647e-05, "loss": 0.015056541189551353, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.01517, "step": 188, "tokens/total": 6540448, "tokens/train_per_sec_per_gpu": 22.9, "tokens/trainable": 86176} +{"epoch": 0.73828125, "grad_norm": 0.7008061408996582, "learning_rate": 7.733219291524489e-05, "loss": 0.010568341240286827, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.01062, "step": 189, "tokens/total": 6577648, "tokens/train_per_sec_per_gpu": 24.62, "tokens/trainable": 86649} +{"epoch": 0.7421875, "grad_norm": 1.99221670627594, "learning_rate": 7.707970881383977e-05, "loss": 0.015115159563720226, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.01523, "step": 190, "tokens/total": 6615600, "tokens/train_per_sec_per_gpu": 21.29, "tokens/trainable": 87084} +{"epoch": 0.74609375, "grad_norm": 1.0142687559127808, "learning_rate": 7.682630588562518e-05, "loss": 0.009617263451218605, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00966, "step": 191, "tokens/total": 6648880, "tokens/train_per_sec_per_gpu": 24.35, "tokens/trainable": 87499} +{"epoch": 0.75, "grad_norm": 0.4122110903263092, "learning_rate": 7.657199467573129e-05, "loss": 0.0032195488456636667, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.00322, "step": 192, "tokens/total": 6686784, "tokens/train_per_sec_per_gpu": 25.54, "tokens/trainable": 87987} +{"epoch": 0.75390625, "grad_norm": 1.3111196756362915, "learning_rate": 7.631678576708561e-05, "loss": 0.021492362022399902, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.02172, "step": 193, "tokens/total": 6722480, "tokens/train_per_sec_per_gpu": 25.93, "tokens/trainable": 88470} +{"epoch": 0.7578125, "grad_norm": 0.21927253901958466, "learning_rate": 7.606068977997255e-05, "loss": 0.003939313814043999, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.00395, "step": 194, "tokens/total": 6757168, "tokens/train_per_sec_per_gpu": 25.99, "tokens/trainable": 88923} +{"epoch": 0.76171875, "grad_norm": 0.6208363771438599, "learning_rate": 7.580371737159148e-05, "loss": 0.010365732945501804, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.02, "memory/max_allocated (GiB)": 37.02, "ppl": 1.01042, "step": 195, "tokens/total": 6790000, "tokens/train_per_sec_per_gpu": 26.86, "tokens/trainable": 89338} +{"epoch": 0.765625, "grad_norm": 0.3526902198791504, "learning_rate": 7.554587923561324e-05, "loss": 0.0070345099084079266, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.00706, "step": 196, "tokens/total": 6826384, "tokens/train_per_sec_per_gpu": 25.62, "tokens/trainable": 89812} +{"epoch": 0.76953125, "grad_norm": 1.7918145656585693, "learning_rate": 7.528718610173511e-05, "loss": 0.01691061072051525, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.01705, "step": 197, "tokens/total": 6860896, "tokens/train_per_sec_per_gpu": 26.11, "tokens/trainable": 90252} +{"epoch": 0.7734375, "grad_norm": 0.192768394947052, "learning_rate": 7.502764873523431e-05, "loss": 0.0030734948813915253, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.28, "memory/max_allocated (GiB)": 39.28, "ppl": 1.00308, "step": 198, "tokens/total": 6896912, "tokens/train_per_sec_per_gpu": 25.97, "tokens/trainable": 90718} +{"epoch": 0.77734375, "grad_norm": 3.955186367034912, "learning_rate": 7.476727793652011e-05, "loss": 0.007567394059151411, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.0076, "step": 199, "tokens/total": 6928592, "tokens/train_per_sec_per_gpu": 29.66, "tokens/trainable": 91194} +{"epoch": 0.78125, "grad_norm": 0.12270069122314453, "learning_rate": 7.450608454068415e-05, "loss": 0.0013393259141594172, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.00134, "step": 200, "tokens/total": 6964672, "tokens/train_per_sec_per_gpu": 23.95, "tokens/trainable": 91649} +{"epoch": 0.78515625, "grad_norm": 0.46683648228645325, "learning_rate": 7.424407941704987e-05, "loss": 0.003259459510445595, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.00326, "step": 201, "tokens/total": 6996336, "tokens/train_per_sec_per_gpu": 28.91, "tokens/trainable": 92078} +{"epoch": 0.7890625, "grad_norm": 0.34460440278053284, "learning_rate": 7.398127346871986e-05, "loss": 0.003846669802442193, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.59, "memory/max_allocated (GiB)": 37.59, "ppl": 1.00385, "step": 202, "tokens/total": 7030544, "tokens/train_per_sec_per_gpu": 24.33, "tokens/trainable": 92536} +{"epoch": 0.79296875, "grad_norm": 0.07374096661806107, "learning_rate": 7.371767763212238e-05, "loss": 0.000827099138405174, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.00083, "step": 203, "tokens/total": 7066480, "tokens/train_per_sec_per_gpu": 27.38, "tokens/trainable": 93032} +{"epoch": 0.796875, "grad_norm": 0.05378403887152672, "learning_rate": 7.345330287655617e-05, "loss": 0.0007034970913082361, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.0007, "step": 204, "tokens/total": 7098768, "tokens/train_per_sec_per_gpu": 30.23, "tokens/trainable": 93504} +{"epoch": 0.80078125, "grad_norm": 0.6009084582328796, "learning_rate": 7.31881602037339e-05, "loss": 0.006377114914357662, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.0064, "step": 205, "tokens/total": 7132144, "tokens/train_per_sec_per_gpu": 22.9, "tokens/trainable": 93925} +{"epoch": 0.8046875, "grad_norm": 1.3174580335617065, "learning_rate": 7.29222606473245e-05, "loss": 0.006088455207645893, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.00611, "step": 206, "tokens/total": 7164720, "tokens/train_per_sec_per_gpu": 27.41, "tokens/trainable": 94365} +{"epoch": 0.80859375, "grad_norm": 0.748485803604126, "learning_rate": 7.265561527249383e-05, "loss": 0.01077723316848278, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.01084, "step": 207, "tokens/total": 7199648, "tokens/train_per_sec_per_gpu": 24.44, "tokens/trainable": 94845} +{"epoch": 0.8125, "grad_norm": 1.0772989988327026, "learning_rate": 7.238823517544436e-05, "loss": 0.017202327027916908, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.01735, "step": 208, "tokens/total": 7235008, "tokens/train_per_sec_per_gpu": 26.39, "tokens/trainable": 95285} +{"epoch": 0.81640625, "grad_norm": 0.5276557207107544, "learning_rate": 7.212013148295333e-05, "loss": 0.0036000735126435757, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.00361, "step": 209, "tokens/total": 7269776, "tokens/train_per_sec_per_gpu": 24.67, "tokens/trainable": 95758} +{"epoch": 0.8203125, "grad_norm": 0.9212007522583008, "learning_rate": 7.185131535190975e-05, "loss": 0.0108229611068964, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.59, "memory/max_allocated (GiB)": 39.59, "ppl": 1.01088, "step": 210, "tokens/total": 7306848, "tokens/train_per_sec_per_gpu": 23.36, "tokens/trainable": 96192} +{"epoch": 0.82421875, "grad_norm": 0.7855530977249146, "learning_rate": 7.158179796885005e-05, "loss": 0.006631728261709213, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00665, "step": 211, "tokens/total": 7345600, "tokens/train_per_sec_per_gpu": 23.62, "tokens/trainable": 96641} +{"epoch": 0.828125, "grad_norm": 0.7231309413909912, "learning_rate": 7.131159054949273e-05, "loss": 0.024029262363910675, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.02432, "step": 212, "tokens/total": 7383040, "tokens/train_per_sec_per_gpu": 26.35, "tokens/trainable": 97160} +{"epoch": 0.83203125, "grad_norm": 0.4849306046962738, "learning_rate": 7.104070433827139e-05, "loss": 0.009427370503544807, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 36.1, "memory/max_allocated (GiB)": 36.1, "ppl": 1.00947, "step": 213, "tokens/total": 7414208, "tokens/train_per_sec_per_gpu": 28.16, "tokens/trainable": 97611} +{"epoch": 0.8359375, "grad_norm": 0.3450356125831604, "learning_rate": 7.076915060786705e-05, "loss": 0.0035560086835175753, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.00356, "step": 214, "tokens/total": 7448864, "tokens/train_per_sec_per_gpu": 27.95, "tokens/trainable": 98111} +{"epoch": 0.83984375, "grad_norm": 0.17638474702835083, "learning_rate": 7.049694065873882e-05, "loss": 0.004225131589919329, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.97, "memory/max_allocated (GiB)": 37.97, "ppl": 1.00423, "step": 215, "tokens/total": 7484096, "tokens/train_per_sec_per_gpu": 26.32, "tokens/trainable": 98575} +{"epoch": 0.84375, "grad_norm": 0.12153201550245285, "learning_rate": 7.022408581865382e-05, "loss": 0.002949478104710579, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.00295, "step": 216, "tokens/total": 7517056, "tokens/train_per_sec_per_gpu": 24.63, "tokens/trainable": 98991} +{"epoch": 0.84765625, "grad_norm": 0.19135813415050507, "learning_rate": 6.99505974422157e-05, "loss": 0.0052568609826266766, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 36.32, "memory/max_allocated (GiB)": 36.32, "ppl": 1.00527, "step": 217, "tokens/total": 7548448, "tokens/train_per_sec_per_gpu": 33.16, "tokens/trainable": 99454} +{"epoch": 0.8515625, "grad_norm": 0.09751725196838379, "learning_rate": 6.967648691039213e-05, "loss": 0.0011623331811279058, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00116, "step": 218, "tokens/total": 7582656, "tokens/train_per_sec_per_gpu": 26.15, "tokens/trainable": 99925} +{"epoch": 0.85546875, "grad_norm": 0.056307170540094376, "learning_rate": 6.940176563004123e-05, "loss": 0.0010704421438276768, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00107, "step": 219, "tokens/total": 7618960, "tokens/train_per_sec_per_gpu": 25.91, "tokens/trainable": 100371} +{"epoch": 0.859375, "grad_norm": 1.0064454078674316, "learning_rate": 6.912644503343682e-05, "loss": 0.007013347931206226, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.35, "memory/max_allocated (GiB)": 39.35, "ppl": 1.00704, "step": 220, "tokens/total": 7655504, "tokens/train_per_sec_per_gpu": 25.9, "tokens/trainable": 100855} +{"epoch": 0.86328125, "grad_norm": 0.276758074760437, "learning_rate": 6.885053657779273e-05, "loss": 0.001881117932498455, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00188, "step": 221, "tokens/total": 7689872, "tokens/train_per_sec_per_gpu": 20.95, "tokens/trainable": 101253} +{"epoch": 0.8671875, "grad_norm": 0.17527620494365692, "learning_rate": 6.857405174478604e-05, "loss": 0.0019367353525012732, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.00194, "step": 222, "tokens/total": 7724432, "tokens/train_per_sec_per_gpu": 25.75, "tokens/trainable": 101706} +{"epoch": 0.87109375, "grad_norm": 0.2763317823410034, "learning_rate": 6.82970020400792e-05, "loss": 0.003604561323300004, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.00361, "step": 223, "tokens/total": 7761440, "tokens/train_per_sec_per_gpu": 22.48, "tokens/trainable": 102158} +{"epoch": 0.875, "grad_norm": 0.1941033899784088, "learning_rate": 6.801939899284132e-05, "loss": 0.01312400121241808, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 35.36, "memory/max_allocated (GiB)": 35.36, "ppl": 1.01321, "step": 224, "tokens/total": 7790048, "tokens/train_per_sec_per_gpu": 29.97, "tokens/trainable": 102580} +{"epoch": 0.87890625, "grad_norm": 0.6203575134277344, "learning_rate": 6.774125415526827e-05, "loss": 0.012382714077830315, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.01246, "step": 225, "tokens/total": 7822352, "tokens/train_per_sec_per_gpu": 30.21, "tokens/trainable": 103063} +{"epoch": 0.8828125, "grad_norm": 0.470218688249588, "learning_rate": 6.746257910210214e-05, "loss": 0.005089670419692993, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.0051, "step": 226, "tokens/total": 7861088, "tokens/train_per_sec_per_gpu": 21.84, "tokens/trainable": 103510} +{"epoch": 0.88671875, "grad_norm": 0.14922265708446503, "learning_rate": 6.718338543014937e-05, "loss": 0.0022029385436326265, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.51, "memory/max_allocated (GiB)": 36.51, "ppl": 1.00221, "step": 227, "tokens/total": 7891456, "tokens/train_per_sec_per_gpu": 33.8, "tokens/trainable": 103980} +{"epoch": 0.890625, "grad_norm": 0.16681687533855438, "learning_rate": 6.69036847577983e-05, "loss": 0.0019140816293656826, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.13, "memory/max_allocated (GiB)": 36.13, "ppl": 1.00192, "step": 228, "tokens/total": 7922704, "tokens/train_per_sec_per_gpu": 28.27, "tokens/trainable": 104413} +{"epoch": 0.89453125, "grad_norm": 0.06377238035202026, "learning_rate": 6.662348872453553e-05, "loss": 0.000599176506511867, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.0006, "step": 229, "tokens/total": 7952912, "tokens/train_per_sec_per_gpu": 29.45, "tokens/trainable": 104845} +{"epoch": 0.8984375, "grad_norm": 0.5289775729179382, "learning_rate": 6.63428089904618e-05, "loss": 0.006382497027516365, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.0064, "step": 230, "tokens/total": 7987504, "tokens/train_per_sec_per_gpu": 23.73, "tokens/trainable": 105289} +{"epoch": 0.90234375, "grad_norm": 0.19703887403011322, "learning_rate": 6.60616572358065e-05, "loss": 0.003275047056376934, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 1.00328, "step": 231, "tokens/total": 8025936, "tokens/train_per_sec_per_gpu": 21.5, "tokens/trainable": 105722} +{"epoch": 0.90625, "grad_norm": 0.01050112396478653, "learning_rate": 6.578004516044172e-05, "loss": 0.0002155634283553809, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.59, "memory/max_allocated (GiB)": 36.59, "ppl": 1.00022, "step": 232, "tokens/total": 8058544, "tokens/train_per_sec_per_gpu": 28.79, "tokens/trainable": 106213} +{"epoch": 0.91015625, "grad_norm": 0.2097344845533371, "learning_rate": 6.549798448339548e-05, "loss": 0.0012739086523652077, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 1.00127, "step": 233, "tokens/total": 8091456, "tokens/train_per_sec_per_gpu": 26.52, "tokens/trainable": 106643} +{"epoch": 0.9140625, "grad_norm": 0.5291793942451477, "learning_rate": 6.521548694236384e-05, "loss": 0.008989045396447182, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.97, "memory/max_allocated (GiB)": 38.97, "ppl": 1.00903, "step": 234, "tokens/total": 8127920, "tokens/train_per_sec_per_gpu": 25.21, "tokens/trainable": 107136} +{"epoch": 0.91796875, "grad_norm": 0.4338541626930237, "learning_rate": 6.493256429322259e-05, "loss": 0.004987718537449837, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.005, "step": 235, "tokens/total": 8167472, "tokens/train_per_sec_per_gpu": 24.2, "tokens/trainable": 107588} +{"epoch": 0.921875, "grad_norm": 0.6545935273170471, "learning_rate": 6.464922830953799e-05, "loss": 0.010467718355357647, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.01052, "step": 236, "tokens/total": 8201792, "tokens/train_per_sec_per_gpu": 27.23, "tokens/trainable": 108087} +{"epoch": 0.92578125, "grad_norm": 0.20568203926086426, "learning_rate": 6.436549078207688e-05, "loss": 0.0011404122924432158, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 1.00114, "step": 237, "tokens/total": 8235504, "tokens/train_per_sec_per_gpu": 26.9, "tokens/trainable": 108533} +{"epoch": 0.9296875, "grad_norm": 0.025615401566028595, "learning_rate": 6.408136351831592e-05, "loss": 0.00025557324988767505, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.24, "memory/max_allocated (GiB)": 38.24, "ppl": 1.00026, "step": 238, "tokens/total": 8271392, "tokens/train_per_sec_per_gpu": 21.42, "tokens/trainable": 108965} +{"epoch": 0.93359375, "grad_norm": 0.4541209638118744, "learning_rate": 6.379685834195036e-05, "loss": 0.007106150034815073, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.00713, "step": 239, "tokens/total": 8304656, "tokens/train_per_sec_per_gpu": 28.37, "tokens/trainable": 109416} +{"epoch": 0.9375, "grad_norm": 0.3658989369869232, "learning_rate": 6.351198709240186e-05, "loss": 0.0023592435754835606, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.12, "memory/max_allocated (GiB)": 39.12, "ppl": 1.00236, "step": 240, "tokens/total": 8343280, "tokens/train_per_sec_per_gpu": 23.05, "tokens/trainable": 109864} +{"epoch": 0.94140625, "grad_norm": 0.8795443177223206, "learning_rate": 6.32267616243259e-05, "loss": 0.015056351199746132, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.01517, "step": 241, "tokens/total": 8377728, "tokens/train_per_sec_per_gpu": 25.95, "tokens/trainable": 110345} +{"epoch": 0.9453125, "grad_norm": 0.20863951742649078, "learning_rate": 6.294119380711849e-05, "loss": 0.0028840540908277035, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 37.15, "memory/max_allocated (GiB)": 37.15, "ppl": 1.00289, "step": 242, "tokens/total": 8412112, "tokens/train_per_sec_per_gpu": 24.46, "tokens/trainable": 110794} +{"epoch": 0.94921875, "grad_norm": 0.03690795600414276, "learning_rate": 6.265529552442209e-05, "loss": 0.0006251604063436389, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.00063, "step": 243, "tokens/total": 8449360, "tokens/train_per_sec_per_gpu": 24.69, "tokens/trainable": 111250} +{"epoch": 0.953125, "grad_norm": 0.5579667091369629, "learning_rate": 6.236907867363127e-05, "loss": 0.019778070971369743, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 1.01997, "step": 244, "tokens/total": 8482160, "tokens/train_per_sec_per_gpu": 26.67, "tokens/trainable": 111693} +{"epoch": 0.95703125, "grad_norm": 0.07204975187778473, "learning_rate": 6.208255516539749e-05, "loss": 0.0008322542416863143, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.45, "memory/max_allocated (GiB)": 38.45, "ppl": 1.00083, "step": 245, "tokens/total": 8516272, "tokens/train_per_sec_per_gpu": 25.48, "tokens/trainable": 112158} +{"epoch": 0.9609375, "grad_norm": 0.2558519244194031, "learning_rate": 6.179573692313344e-05, "loss": 0.0026935888454318047, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.98, "memory/max_allocated (GiB)": 36.98, "ppl": 1.0027, "step": 246, "tokens/total": 8548240, "tokens/train_per_sec_per_gpu": 28.49, "tokens/trainable": 112601} +{"epoch": 0.96484375, "grad_norm": 1.7984874248504639, "learning_rate": 6.150863588251694e-05, "loss": 0.002726505510509014, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.34, "memory/max_allocated (GiB)": 36.34, "ppl": 1.00273, "step": 247, "tokens/total": 8580000, "tokens/train_per_sec_per_gpu": 25.48, "tokens/trainable": 113016} +{"epoch": 0.96875, "grad_norm": 0.07304666191339493, "learning_rate": 6.122126399099419e-05, "loss": 0.0006655743927694857, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.00067, "step": 248, "tokens/total": 8614864, "tokens/train_per_sec_per_gpu": 24.38, "tokens/trainable": 113476} +{"epoch": 0.97265625, "grad_norm": 0.13121700286865234, "learning_rate": 6.0933633207282615e-05, "loss": 0.0015492549864575267, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.66, "memory/max_allocated (GiB)": 38.66, "ppl": 1.00155, "step": 249, "tokens/total": 8651008, "tokens/train_per_sec_per_gpu": 26.4, "tokens/trainable": 113941} +{"epoch": 0.9765625, "grad_norm": 0.5458178520202637, "learning_rate": 6.064575550087316e-05, "loss": 0.008655000478029251, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.83, "memory/max_allocated (GiB)": 36.83, "ppl": 1.00869, "step": 250, "tokens/total": 8683696, "tokens/train_per_sec_per_gpu": 29.96, "tokens/trainable": 114434} +{"epoch": 0.98046875, "grad_norm": 0.3710736036300659, "learning_rate": 6.0357642851532245e-05, "loss": 0.0035258629359304905, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.28, "memory/max_allocated (GiB)": 36.28, "ppl": 1.00353, "step": 251, "tokens/total": 8714880, "tokens/train_per_sec_per_gpu": 29.36, "tokens/trainable": 114898} +{"epoch": 0.984375, "grad_norm": 0.010267512872815132, "learning_rate": 6.0069307248803294e-05, "loss": 0.0001418210013071075, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.15, "memory/max_allocated (GiB)": 39.15, "ppl": 1.00014, "step": 252, "tokens/total": 8751072, "tokens/train_per_sec_per_gpu": 21.46, "tokens/trainable": 115331} +{"epoch": 0.98828125, "grad_norm": 0.13256710767745972, "learning_rate": 5.9780760691507635e-05, "loss": 0.0008112844079732895, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.73, "memory/max_allocated (GiB)": 36.73, "ppl": 1.00081, "step": 253, "tokens/total": 8782272, "tokens/train_per_sec_per_gpu": 27.77, "tokens/trainable": 115732} +{"epoch": 0.9921875, "grad_norm": 0.251224547624588, "learning_rate": 5.9492015187245334e-05, "loss": 0.010599920526146889, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.28, "memory/max_allocated (GiB)": 39.28, "ppl": 1.01066, "step": 254, "tokens/total": 8819920, "tokens/train_per_sec_per_gpu": 25.37, "tokens/trainable": 116221} +{"epoch": 0.99609375, "grad_norm": 0.5714102387428284, "learning_rate": 5.920308275189541e-05, "loss": 0.0065354336984455585, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 37.91, "memory/max_allocated (GiB)": 37.91, "ppl": 1.00656, "step": 255, "tokens/total": 8855232, "tokens/train_per_sec_per_gpu": 23.53, "tokens/trainable": 116684} +{"epoch": 1.0, "grad_norm": 0.40917789936065674, "learning_rate": 5.8913975409115874e-05, "loss": 0.0025534036103636026, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.00256, "step": 256, "tokens/total": 8889920, "tokens/train_per_sec_per_gpu": 24.38, "tokens/trainable": 117126} +{"epoch": 1.00390625, "grad_norm": 0.03812432289123535, "learning_rate": 5.8624705189843395e-05, "loss": 0.0003259539953432977, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00033, "step": 257, "tokens/total": 8922640, "tokens/train_per_sec_per_gpu": 28.06, "tokens/trainable": 117592} +{"epoch": 1.0078125, "grad_norm": 0.015948690474033356, "learning_rate": 5.833528413179249e-05, "loss": 0.0001671076606726274, "memory/device_reserved (GiB)": 39.79, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00017, "step": 258, "tokens/total": 8958848, "tokens/train_per_sec_per_gpu": 21.4, "tokens/trainable": 118030} +{"epoch": 1.01171875, "grad_norm": 1.1597094535827637, "learning_rate": 5.80457242789548e-05, "loss": 0.006910093128681183, "memory/device_reserved (GiB)": 39.79, "memory/max_active (GiB)": 36.51, "memory/max_allocated (GiB)": 36.51, "ppl": 1.00693, "step": 259, "tokens/total": 8991056, "tokens/train_per_sec_per_gpu": 26.15, "tokens/trainable": 118470} +{"epoch": 1.015625, "grad_norm": 1.3606629371643066, "learning_rate": 5.77560376810977e-05, "loss": 0.01993522047996521, "memory/device_reserved (GiB)": 39.79, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.02014, "step": 260, "tokens/total": 9024720, "tokens/train_per_sec_per_gpu": 28.13, "tokens/trainable": 118935} +{"epoch": 1.01953125, "grad_norm": 0.8899140357971191, "learning_rate": 5.7466236393263005e-05, "loss": 0.012784291058778763, "memory/device_reserved (GiB)": 39.79, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.01287, "step": 261, "tokens/total": 9058944, "tokens/train_per_sec_per_gpu": 25.31, "tokens/trainable": 119381} +{"epoch": 1.0234375, "grad_norm": 0.3086192309856415, "learning_rate": 5.717633247526522e-05, "loss": 0.0016225436702370644, "memory/device_reserved (GiB)": 41.75, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.00162, "step": 262, "tokens/total": 9098816, "tokens/train_per_sec_per_gpu": 22.99, "tokens/trainable": 119849} +{"epoch": 1.02734375, "grad_norm": 0.07240071892738342, "learning_rate": 5.688633799118971e-05, "loss": 0.0003744476125575602, "memory/device_reserved (GiB)": 41.75, "memory/max_active (GiB)": 36.78, "memory/max_allocated (GiB)": 36.78, "ppl": 1.00037, "step": 263, "tokens/total": 9132080, "tokens/train_per_sec_per_gpu": 28.88, "tokens/trainable": 120319} +{"epoch": 1.03125, "grad_norm": 0.0214369036257267, "learning_rate": 5.659626500889066e-05, "loss": 0.00017675786511972547, "memory/device_reserved (GiB)": 41.75, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.00018, "step": 264, "tokens/total": 9165072, "tokens/train_per_sec_per_gpu": 30.62, "tokens/trainable": 120804} +{"epoch": 1.03515625, "grad_norm": 0.08649063855409622, "learning_rate": 5.6306125599488905e-05, "loss": 0.0004554993938654661, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 35.9, "memory/max_allocated (GiB)": 35.9, "ppl": 1.00046, "step": 265, "tokens/total": 9195776, "tokens/train_per_sec_per_gpu": 29.09, "tokens/trainable": 121233} +{"epoch": 1.0390625, "grad_norm": 1.0491172075271606, "learning_rate": 5.601593183686955e-05, "loss": 0.010164294391870499, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.95, "memory/max_allocated (GiB)": 37.95, "ppl": 1.01022, "step": 266, "tokens/total": 9231136, "tokens/train_per_sec_per_gpu": 22.37, "tokens/trainable": 121671} +{"epoch": 1.04296875, "grad_norm": 0.10526392608880997, "learning_rate": 5.572569579717961e-05, "loss": 0.0006266388227231801, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00063, "step": 267, "tokens/total": 9266784, "tokens/train_per_sec_per_gpu": 25.48, "tokens/trainable": 122141} +{"epoch": 1.046875, "grad_norm": 0.0414196141064167, "learning_rate": 5.543542955832538e-05, "loss": 0.0003509540983941406, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.37, "memory/max_allocated (GiB)": 38.37, "ppl": 1.00035, "step": 268, "tokens/total": 9301648, "tokens/train_per_sec_per_gpu": 24.9, "tokens/trainable": 122556} +{"epoch": 1.05078125, "grad_norm": 0.48251426219940186, "learning_rate": 5.514514519946986e-05, "loss": 0.005577684845775366, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.00559, "step": 269, "tokens/total": 9334352, "tokens/train_per_sec_per_gpu": 27.17, "tokens/trainable": 122988} +{"epoch": 1.0546875, "grad_norm": 0.16558563709259033, "learning_rate": 5.485485480053015e-05, "loss": 0.0005610735388472676, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.00056, "step": 270, "tokens/total": 9369376, "tokens/train_per_sec_per_gpu": 26.69, "tokens/trainable": 123478} +{"epoch": 1.05859375, "grad_norm": 0.25631341338157654, "learning_rate": 5.4564570441674645e-05, "loss": 0.0021057836711406708, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.00211, "step": 271, "tokens/total": 9402928, "tokens/train_per_sec_per_gpu": 28.55, "tokens/trainable": 123932} +{"epoch": 1.0625, "grad_norm": 0.7358006238937378, "learning_rate": 5.42743042028204e-05, "loss": 0.0029309955425560474, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.00294, "step": 272, "tokens/total": 9437952, "tokens/train_per_sec_per_gpu": 29.19, "tokens/trainable": 124428} +{"epoch": 1.06640625, "grad_norm": 0.42348024249076843, "learning_rate": 5.3984068163130464e-05, "loss": 0.003955335821956396, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.00396, "step": 273, "tokens/total": 9471648, "tokens/train_per_sec_per_gpu": 22.73, "tokens/trainable": 124866} +{"epoch": 1.0703125, "grad_norm": 0.610283613204956, "learning_rate": 5.369387440051111e-05, "loss": 0.0022875317372381687, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.00229, "step": 274, "tokens/total": 9508640, "tokens/train_per_sec_per_gpu": 24.4, "tokens/trainable": 125324} +{"epoch": 1.07421875, "grad_norm": 1.3872227668762207, "learning_rate": 5.340373499110935e-05, "loss": 0.00512463366612792, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.00514, "step": 275, "tokens/total": 9548560, "tokens/train_per_sec_per_gpu": 23.91, "tokens/trainable": 125813} +{"epoch": 1.078125, "grad_norm": 0.12103284150362015, "learning_rate": 5.3113662008810304e-05, "loss": 0.0012031719088554382, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.0012, "step": 276, "tokens/total": 9584608, "tokens/train_per_sec_per_gpu": 22.08, "tokens/trainable": 126252} +{"epoch": 1.08203125, "grad_norm": 0.4846673309803009, "learning_rate": 5.282366752473479e-05, "loss": 0.0028621747624129057, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.99, "memory/max_allocated (GiB)": 38.99, "ppl": 1.00287, "step": 277, "tokens/total": 9620656, "tokens/train_per_sec_per_gpu": 24.78, "tokens/trainable": 126696} +{"epoch": 1.0859375, "grad_norm": 0.19877856969833374, "learning_rate": 5.2533763606737005e-05, "loss": 0.006462927907705307, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.00648, "step": 278, "tokens/total": 9652992, "tokens/train_per_sec_per_gpu": 28.44, "tokens/trainable": 127173} +{"epoch": 1.08984375, "grad_norm": 0.22581912577152252, "learning_rate": 5.224396231890232e-05, "loss": 0.0018638563342392445, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.00187, "step": 279, "tokens/total": 9685632, "tokens/train_per_sec_per_gpu": 27.59, "tokens/trainable": 127637} +{"epoch": 1.09375, "grad_norm": 0.4386557936668396, "learning_rate": 5.195427572104522e-05, "loss": 0.0045161438174545765, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.59, "memory/max_allocated (GiB)": 38.59, "ppl": 1.00453, "step": 280, "tokens/total": 9723056, "tokens/train_per_sec_per_gpu": 24.65, "tokens/trainable": 128112} +{"epoch": 1.09765625, "grad_norm": 0.1861466020345688, "learning_rate": 5.166471586820751e-05, "loss": 0.0011444626143202186, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.73, "memory/max_allocated (GiB)": 37.73, "ppl": 1.00115, "step": 281, "tokens/total": 9758592, "tokens/train_per_sec_per_gpu": 26.03, "tokens/trainable": 128584} +{"epoch": 1.1015625, "grad_norm": 0.004101725295186043, "learning_rate": 5.1375294810156615e-05, "loss": 5.384797987062484e-05, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.3, "memory/max_allocated (GiB)": 38.3, "ppl": 1.00005, "step": 282, "tokens/total": 9793408, "tokens/train_per_sec_per_gpu": 24.5, "tokens/trainable": 129051} +{"epoch": 1.10546875, "grad_norm": 0.10172475874423981, "learning_rate": 5.1086024590884144e-05, "loss": 0.00023979249817784876, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.15, "memory/max_allocated (GiB)": 37.15, "ppl": 1.00024, "step": 283, "tokens/total": 9827216, "tokens/train_per_sec_per_gpu": 25.29, "tokens/trainable": 129494} +{"epoch": 1.109375, "grad_norm": 0.17792317271232605, "learning_rate": 5.079691724810461e-05, "loss": 0.000608865637332201, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 39.35, "memory/max_allocated (GiB)": 39.35, "ppl": 1.00061, "step": 284, "tokens/total": 9864928, "tokens/train_per_sec_per_gpu": 24.48, "tokens/trainable": 129983} +{"epoch": 1.11328125, "grad_norm": 0.39659222960472107, "learning_rate": 5.0507984812754684e-05, "loss": 0.003225680673494935, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00323, "step": 285, "tokens/total": 9901520, "tokens/train_per_sec_per_gpu": 23.31, "tokens/trainable": 130405} +{"epoch": 1.1171875, "grad_norm": 0.3968925178050995, "learning_rate": 5.021923930849237e-05, "loss": 0.003969273529946804, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.56, "memory/max_allocated (GiB)": 37.56, "ppl": 1.00398, "step": 286, "tokens/total": 9935824, "tokens/train_per_sec_per_gpu": 24.49, "tokens/trainable": 130829} +{"epoch": 1.12109375, "grad_norm": 0.031108420342206955, "learning_rate": 4.99306927511967e-05, "loss": 0.0003254279727116227, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.00033, "step": 287, "tokens/total": 9969200, "tokens/train_per_sec_per_gpu": 24.22, "tokens/trainable": 131287} +{"epoch": 1.125, "grad_norm": 0.22926072776317596, "learning_rate": 4.964235714846775e-05, "loss": 0.0032375797163695097, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.00324, "step": 288, "tokens/total": 10005120, "tokens/train_per_sec_per_gpu": 23.4, "tokens/trainable": 131712} +{"epoch": 1.12890625, "grad_norm": 1.7695249319076538, "learning_rate": 4.9354244499126866e-05, "loss": 0.000780331261921674, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00078, "step": 289, "tokens/total": 10040336, "tokens/train_per_sec_per_gpu": 25.1, "tokens/trainable": 132168} +{"epoch": 1.1328125, "grad_norm": 0.34178733825683594, "learning_rate": 4.90663667927174e-05, "loss": 0.0017212949460372329, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.00172, "step": 290, "tokens/total": 10075232, "tokens/train_per_sec_per_gpu": 26.83, "tokens/trainable": 132626} +{"epoch": 1.13671875, "grad_norm": 0.7627476453781128, "learning_rate": 4.877873600900581e-05, "loss": 0.008483112789690495, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.00852, "step": 291, "tokens/total": 10113632, "tokens/train_per_sec_per_gpu": 23.98, "tokens/trainable": 133119} +{"epoch": 1.140625, "grad_norm": 0.015156197361648083, "learning_rate": 4.849136411748306e-05, "loss": 0.00011025417188648134, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 36.96, "memory/max_allocated (GiB)": 36.96, "ppl": 1.00011, "step": 292, "tokens/total": 10145728, "tokens/train_per_sec_per_gpu": 26.96, "tokens/trainable": 133550} +{"epoch": 1.14453125, "grad_norm": 0.023963676765561104, "learning_rate": 4.8204263076866574e-05, "loss": 0.00010656312224455178, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.39, "memory/max_allocated (GiB)": 38.39, "ppl": 1.00011, "step": 293, "tokens/total": 10182512, "tokens/train_per_sec_per_gpu": 24.63, "tokens/trainable": 133992} +{"epoch": 1.1484375, "grad_norm": 0.08223158866167068, "learning_rate": 4.791744483460251e-05, "loss": 0.0003584742662496865, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.54, "memory/max_allocated (GiB)": 39.54, "ppl": 1.00036, "step": 294, "tokens/total": 10218144, "tokens/train_per_sec_per_gpu": 28.54, "tokens/trainable": 134467} +{"epoch": 1.15234375, "grad_norm": 0.19584503769874573, "learning_rate": 4.7630921326368736e-05, "loss": 0.002033960074186325, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.00204, "step": 295, "tokens/total": 10256160, "tokens/train_per_sec_per_gpu": 19.96, "tokens/trainable": 134895} +{"epoch": 1.15625, "grad_norm": 0.3132155239582062, "learning_rate": 4.7344704475577916e-05, "loss": 0.0017270769458264112, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 35.47, "memory/max_allocated (GiB)": 35.47, "ppl": 1.00173, "step": 296, "tokens/total": 10285136, "tokens/train_per_sec_per_gpu": 30.74, "tokens/trainable": 135321} +{"epoch": 1.16015625, "grad_norm": 0.1700582355260849, "learning_rate": 4.705880619288153e-05, "loss": 0.0014955576043576002, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.0015, "step": 297, "tokens/total": 10321424, "tokens/train_per_sec_per_gpu": 23.55, "tokens/trainable": 135765} +{"epoch": 1.1640625, "grad_norm": 0.02844458632171154, "learning_rate": 4.677323837567412e-05, "loss": 0.0001541687670396641, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.00015, "step": 298, "tokens/total": 10359248, "tokens/train_per_sec_per_gpu": 24.6, "tokens/trainable": 136196} +{"epoch": 1.16796875, "grad_norm": 0.027805475518107414, "learning_rate": 4.6488012907598146e-05, "loss": 0.00021764743723906577, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.81, "memory/max_allocated (GiB)": 38.81, "ppl": 1.00022, "step": 299, "tokens/total": 10395824, "tokens/train_per_sec_per_gpu": 24.27, "tokens/trainable": 136649} +{"epoch": 1.171875, "grad_norm": 1.1179852485656738, "learning_rate": 4.620314165804964e-05, "loss": 0.0057668862864375114, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00578, "step": 300, "tokens/total": 10432864, "tokens/train_per_sec_per_gpu": 26.54, "tokens/trainable": 137128} +{"epoch": 1.17578125, "grad_norm": 0.33359694480895996, "learning_rate": 4.591863648168407e-05, "loss": 0.0021168787498027086, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.04, "memory/max_allocated (GiB)": 37.04, "ppl": 1.00212, "step": 301, "tokens/total": 10466816, "tokens/train_per_sec_per_gpu": 30.2, "tokens/trainable": 137585} +{"epoch": 1.1796875, "grad_norm": 0.7181750535964966, "learning_rate": 4.5634509217923135e-05, "loss": 0.0019512041471898556, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.00195, "step": 302, "tokens/total": 10499280, "tokens/train_per_sec_per_gpu": 31.02, "tokens/trainable": 138054} +{"epoch": 1.18359375, "grad_norm": 0.39101871848106384, "learning_rate": 4.535077169046201e-05, "loss": 0.002754020970314741, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.0, "memory/max_allocated (GiB)": 38.0, "ppl": 1.00276, "step": 303, "tokens/total": 10534112, "tokens/train_per_sec_per_gpu": 28.5, "tokens/trainable": 138559} +{"epoch": 1.1875, "grad_norm": 1.8943352699279785, "learning_rate": 4.506743570677743e-05, "loss": 0.021030141040682793, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.53, "memory/max_allocated (GiB)": 37.53, "ppl": 1.02125, "step": 304, "tokens/total": 10568912, "tokens/train_per_sec_per_gpu": 24.12, "tokens/trainable": 139050} +{"epoch": 1.19140625, "grad_norm": 0.1600867211818695, "learning_rate": 4.478451305763618e-05, "loss": 0.0020713419653475285, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.00207, "step": 305, "tokens/total": 10606128, "tokens/train_per_sec_per_gpu": 24.67, "tokens/trainable": 139525} +{"epoch": 1.1953125, "grad_norm": 0.03414035588502884, "learning_rate": 4.450201551660454e-05, "loss": 0.00016642047557979822, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.08, "memory/max_allocated (GiB)": 39.08, "ppl": 1.00017, "step": 306, "tokens/total": 10639584, "tokens/train_per_sec_per_gpu": 26.43, "tokens/trainable": 139969} +{"epoch": 1.19921875, "grad_norm": 0.02142263576388359, "learning_rate": 4.4219954839558276e-05, "loss": 0.00016502884682267904, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.44, "memory/max_allocated (GiB)": 36.44, "ppl": 1.00017, "step": 307, "tokens/total": 10670928, "tokens/train_per_sec_per_gpu": 31.36, "tokens/trainable": 140443} +{"epoch": 1.203125, "grad_norm": 0.004418583586812019, "learning_rate": 4.393834276419352e-05, "loss": 4.211188934277743e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.00004, "step": 308, "tokens/total": 10709968, "tokens/train_per_sec_per_gpu": 23.83, "tokens/trainable": 140926} +{"epoch": 1.20703125, "grad_norm": 1.8618919849395752, "learning_rate": 4.36571910095382e-05, "loss": 0.0032110577449202538, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.00322, "step": 309, "tokens/total": 10744800, "tokens/train_per_sec_per_gpu": 24.06, "tokens/trainable": 141344} +{"epoch": 1.2109375, "grad_norm": 0.009089311584830284, "learning_rate": 4.337651127546448e-05, "loss": 0.00011379504576325417, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.00011, "step": 310, "tokens/total": 10778576, "tokens/train_per_sec_per_gpu": 26.03, "tokens/trainable": 141753} +{"epoch": 1.21484375, "grad_norm": 0.0064188092947006226, "learning_rate": 4.3096315242201736e-05, "loss": 7.35283610993065e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.61, "memory/max_allocated (GiB)": 36.61, "ppl": 1.00007, "step": 311, "tokens/total": 10810816, "tokens/train_per_sec_per_gpu": 31.03, "tokens/trainable": 142219} +{"epoch": 1.21875, "grad_norm": 1.0693070888519287, "learning_rate": 4.2816614569850635e-05, "loss": 0.011759508401155472, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.01, "memory/max_allocated (GiB)": 36.01, "ppl": 1.01183, "step": 312, "tokens/total": 10841040, "tokens/train_per_sec_per_gpu": 25.14, "tokens/trainable": 142617} +{"epoch": 1.22265625, "grad_norm": 0.14992758631706238, "learning_rate": 4.2537420897897864e-05, "loss": 0.0005014584166929126, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.0005, "step": 313, "tokens/total": 10880960, "tokens/train_per_sec_per_gpu": 21.24, "tokens/trainable": 143103} +{"epoch": 1.2265625, "grad_norm": 0.03364640474319458, "learning_rate": 4.225874584473174e-05, "loss": 0.0003322863485664129, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00033, "step": 314, "tokens/total": 10913472, "tokens/train_per_sec_per_gpu": 25.04, "tokens/trainable": 143535} +{"epoch": 1.23046875, "grad_norm": 0.8656983375549316, "learning_rate": 4.19806010071587e-05, "loss": 0.007109993137419224, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00714, "step": 315, "tokens/total": 10951360, "tokens/train_per_sec_per_gpu": 23.67, "tokens/trainable": 144009} +{"epoch": 1.234375, "grad_norm": 0.36570215225219727, "learning_rate": 4.170299795992081e-05, "loss": 0.002803233452141285, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.00281, "step": 316, "tokens/total": 10987808, "tokens/train_per_sec_per_gpu": 23.13, "tokens/trainable": 144460} +{"epoch": 1.23828125, "grad_norm": 0.038790248334407806, "learning_rate": 4.142594825521398e-05, "loss": 0.0003664366668090224, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.00037, "step": 317, "tokens/total": 11025552, "tokens/train_per_sec_per_gpu": 24.42, "tokens/trainable": 144936} +{"epoch": 1.2421875, "grad_norm": 0.028084179386496544, "learning_rate": 4.114946342220728e-05, "loss": 0.00028309563640505075, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.00028, "step": 318, "tokens/total": 11060432, "tokens/train_per_sec_per_gpu": 24.96, "tokens/trainable": 145375} +{"epoch": 1.24609375, "grad_norm": 0.0037882067263126373, "learning_rate": 4.087355496656321e-05, "loss": 5.882982077309862e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.00006, "step": 319, "tokens/total": 11094688, "tokens/train_per_sec_per_gpu": 25.28, "tokens/trainable": 145849} +{"epoch": 1.25, "grad_norm": 0.1162177100777626, "learning_rate": 4.05982343699588e-05, "loss": 0.00043075400753878057, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00043, "step": 320, "tokens/total": 11127408, "tokens/train_per_sec_per_gpu": 26.69, "tokens/trainable": 146318} +{"epoch": 1.25390625, "grad_norm": 0.042228758335113525, "learning_rate": 4.0323513089607876e-05, "loss": 0.00027591947582550347, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.72, "memory/max_allocated (GiB)": 38.72, "ppl": 1.00028, "step": 321, "tokens/total": 11162752, "tokens/train_per_sec_per_gpu": 26.41, "tokens/trainable": 146821} +{"epoch": 1.2578125, "grad_norm": 0.022230779752135277, "learning_rate": 4.004940255778431e-05, "loss": 0.0001361009490210563, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.33, "memory/max_allocated (GiB)": 36.33, "ppl": 1.00014, "step": 322, "tokens/total": 11194576, "tokens/train_per_sec_per_gpu": 26.14, "tokens/trainable": 147285} +{"epoch": 1.26171875, "grad_norm": 0.004127836786210537, "learning_rate": 3.977591418134619e-05, "loss": 3.732043114723638e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.61, "memory/max_allocated (GiB)": 36.61, "ppl": 1.00004, "step": 323, "tokens/total": 11226928, "tokens/train_per_sec_per_gpu": 25.4, "tokens/trainable": 147729} +{"epoch": 1.265625, "grad_norm": 0.9986739158630371, "learning_rate": 3.95030593412612e-05, "loss": 0.01850123517215252, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.01867, "step": 324, "tokens/total": 11266016, "tokens/train_per_sec_per_gpu": 20.39, "tokens/trainable": 148169} +{"epoch": 1.26953125, "grad_norm": 0.43547147512435913, "learning_rate": 3.923084939213296e-05, "loss": 0.003660145914182067, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.00367, "step": 325, "tokens/total": 11303264, "tokens/train_per_sec_per_gpu": 23.8, "tokens/trainable": 148630} +{"epoch": 1.2734375, "grad_norm": 0.017986027523875237, "learning_rate": 3.895929566172861e-05, "loss": 0.00020762905478477478, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00021, "step": 326, "tokens/total": 11339392, "tokens/train_per_sec_per_gpu": 22.35, "tokens/trainable": 149063} +{"epoch": 1.27734375, "grad_norm": 0.49514055252075195, "learning_rate": 3.868840945050728e-05, "loss": 0.005711533594876528, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.00573, "step": 327, "tokens/total": 11377568, "tokens/train_per_sec_per_gpu": 22.93, "tokens/trainable": 149549} +{"epoch": 1.28125, "grad_norm": 0.4603423476219177, "learning_rate": 3.841820203114995e-05, "loss": 0.00322387320920825, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.03, "memory/max_allocated (GiB)": 38.03, "ppl": 1.00323, "step": 328, "tokens/total": 11413184, "tokens/train_per_sec_per_gpu": 26.47, "tokens/trainable": 150015} +{"epoch": 1.28515625, "grad_norm": 0.8212201595306396, "learning_rate": 3.814868464809027e-05, "loss": 0.008808151818811893, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.00885, "step": 329, "tokens/total": 11447904, "tokens/train_per_sec_per_gpu": 27.78, "tokens/trainable": 150514} +{"epoch": 1.2890625, "grad_norm": 0.24944822490215302, "learning_rate": 3.787986851704667e-05, "loss": 0.0033498932607471943, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.00336, "step": 330, "tokens/total": 11485056, "tokens/train_per_sec_per_gpu": 27.23, "tokens/trainable": 151021} +{"epoch": 1.29296875, "grad_norm": 0.22186510264873505, "learning_rate": 3.7611764824555654e-05, "loss": 0.0026981448754668236, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.37, "memory/max_allocated (GiB)": 38.37, "ppl": 1.0027, "step": 331, "tokens/total": 11518848, "tokens/train_per_sec_per_gpu": 29.48, "tokens/trainable": 151486} +{"epoch": 1.296875, "grad_norm": 0.37486615777015686, "learning_rate": 3.734438472750619e-05, "loss": 0.011227786540985107, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.79, "memory/max_allocated (GiB)": 36.79, "ppl": 1.01129, "step": 332, "tokens/total": 11552048, "tokens/train_per_sec_per_gpu": 26.04, "tokens/trainable": 151956} +{"epoch": 1.30078125, "grad_norm": 0.4921615719795227, "learning_rate": 3.707773935267552e-05, "loss": 0.009424181655049324, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.15, "memory/max_allocated (GiB)": 39.15, "ppl": 1.00947, "step": 333, "tokens/total": 11589168, "tokens/train_per_sec_per_gpu": 24.6, "tokens/trainable": 152442} +{"epoch": 1.3046875, "grad_norm": 0.9690125584602356, "learning_rate": 3.68118397962661e-05, "loss": 0.006598281674087048, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.74, "memory/max_allocated (GiB)": 36.74, "ppl": 1.00662, "step": 334, "tokens/total": 11622032, "tokens/train_per_sec_per_gpu": 25.55, "tokens/trainable": 152890} +{"epoch": 1.30859375, "grad_norm": 0.6713874936103821, "learning_rate": 3.654669712344384e-05, "loss": 0.008464558981359005, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.35, "memory/max_allocated (GiB)": 36.35, "ppl": 1.0085, "step": 335, "tokens/total": 11653840, "tokens/train_per_sec_per_gpu": 30.48, "tokens/trainable": 153368} +{"epoch": 1.3125, "grad_norm": 0.5701912045478821, "learning_rate": 3.628232236787763e-05, "loss": 0.0038612319622188807, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00387, "step": 336, "tokens/total": 11689664, "tokens/train_per_sec_per_gpu": 26.16, "tokens/trainable": 153809} +{"epoch": 1.31640625, "grad_norm": 0.31384721398353577, "learning_rate": 3.6018726531280144e-05, "loss": 0.006986136548221111, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.06, "memory/max_allocated (GiB)": 37.06, "ppl": 1.00701, "step": 337, "tokens/total": 11723728, "tokens/train_per_sec_per_gpu": 25.58, "tokens/trainable": 154258} +{"epoch": 1.3203125, "grad_norm": 1.1075828075408936, "learning_rate": 3.575592058295017e-05, "loss": 0.00924239307641983, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.00929, "step": 338, "tokens/total": 11758960, "tokens/train_per_sec_per_gpu": 24.72, "tokens/trainable": 154704} +{"epoch": 1.32421875, "grad_norm": 0.48258623480796814, "learning_rate": 3.549391545931585e-05, "loss": 0.006354387849569321, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.00637, "step": 339, "tokens/total": 11797328, "tokens/train_per_sec_per_gpu": 23.76, "tokens/trainable": 155157} +{"epoch": 1.328125, "grad_norm": 1.528455376625061, "learning_rate": 3.5232722063479914e-05, "loss": 0.004493218846619129, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.07, "memory/max_allocated (GiB)": 38.07, "ppl": 1.0045, "step": 340, "tokens/total": 11832656, "tokens/train_per_sec_per_gpu": 23.51, "tokens/trainable": 155582} +{"epoch": 1.33203125, "grad_norm": 0.11781935393810272, "learning_rate": 3.49723512647657e-05, "loss": 0.0010698835831135511, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.00107, "step": 341, "tokens/total": 11865152, "tokens/train_per_sec_per_gpu": 27.83, "tokens/trainable": 156047} +{"epoch": 1.3359375, "grad_norm": 0.11068471521139145, "learning_rate": 3.471281389826491e-05, "loss": 0.0017645441694185138, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.99, "memory/max_allocated (GiB)": 38.99, "ppl": 1.00177, "step": 342, "tokens/total": 11901552, "tokens/train_per_sec_per_gpu": 25.57, "tokens/trainable": 156500} +{"epoch": 1.33984375, "grad_norm": 0.11181522905826569, "learning_rate": 3.4454120764386764e-05, "loss": 0.0013959391508251429, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.0014, "step": 343, "tokens/total": 11937328, "tokens/train_per_sec_per_gpu": 24.67, "tokens/trainable": 156931} +{"epoch": 1.34375, "grad_norm": 0.13849639892578125, "learning_rate": 3.4196282628408526e-05, "loss": 0.002281149849295616, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00228, "step": 344, "tokens/total": 11970736, "tokens/train_per_sec_per_gpu": 25.39, "tokens/trainable": 157385} +{"epoch": 1.34765625, "grad_norm": 0.10237203538417816, "learning_rate": 3.3939310220027456e-05, "loss": 0.0012102265609428287, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.55, "memory/max_allocated (GiB)": 39.55, "ppl": 1.00121, "step": 345, "tokens/total": 12008096, "tokens/train_per_sec_per_gpu": 22.66, "tokens/trainable": 157817} +{"epoch": 1.3515625, "grad_norm": 0.23917531967163086, "learning_rate": 3.3683214232914404e-05, "loss": 0.006403025705367327, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.00642, "step": 346, "tokens/total": 12039904, "tokens/train_per_sec_per_gpu": 26.26, "tokens/trainable": 158273} +{"epoch": 1.35546875, "grad_norm": 0.08338572829961777, "learning_rate": 3.342800532426873e-05, "loss": 0.0013490146957337856, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.2, "memory/max_allocated (GiB)": 37.2, "ppl": 1.00135, "step": 347, "tokens/total": 12074416, "tokens/train_per_sec_per_gpu": 25.9, "tokens/trainable": 158728} +{"epoch": 1.359375, "grad_norm": 0.012932809069752693, "learning_rate": 3.317369411437484e-05, "loss": 0.00021113973343744874, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.12, "memory/max_allocated (GiB)": 37.12, "ppl": 1.00021, "step": 348, "tokens/total": 12108432, "tokens/train_per_sec_per_gpu": 24.4, "tokens/trainable": 159168} +{"epoch": 1.36328125, "grad_norm": 0.16469232738018036, "learning_rate": 3.292029118616024e-05, "loss": 0.0016839124727994204, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.00169, "step": 349, "tokens/total": 12141792, "tokens/train_per_sec_per_gpu": 23.04, "tokens/trainable": 159592} +{"epoch": 1.3671875, "grad_norm": 0.04145296663045883, "learning_rate": 3.266780708475511e-05, "loss": 0.0004914499004371464, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.00049, "step": 350, "tokens/total": 12175840, "tokens/train_per_sec_per_gpu": 29.21, "tokens/trainable": 160093} +{"epoch": 1.37109375, "grad_norm": 0.2872648239135742, "learning_rate": 3.241625231705354e-05, "loss": 0.0017869217554107308, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.56, "memory/max_allocated (GiB)": 37.56, "ppl": 1.00179, "step": 351, "tokens/total": 12208720, "tokens/train_per_sec_per_gpu": 25.61, "tokens/trainable": 160534} +{"epoch": 1.375, "grad_norm": 0.5969806909561157, "learning_rate": 3.216563735127618e-05, "loss": 0.020974505692720413, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.0212, "step": 352, "tokens/total": 12246336, "tokens/train_per_sec_per_gpu": 24.26, "tokens/trainable": 161009} +{"epoch": 1.37890625, "grad_norm": 0.4324926435947418, "learning_rate": 3.191597261653475e-05, "loss": 0.008566515520215034, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.0086, "step": 353, "tokens/total": 12279216, "tokens/train_per_sec_per_gpu": 27.7, "tokens/trainable": 161450} +{"epoch": 1.3828125, "grad_norm": 0.6899469494819641, "learning_rate": 3.166726850239794e-05, "loss": 0.009101295843720436, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00914, "step": 354, "tokens/total": 12311936, "tokens/train_per_sec_per_gpu": 30.24, "tokens/trainable": 161900} +{"epoch": 1.38671875, "grad_norm": 0.010743709281086922, "learning_rate": 3.141953535845912e-05, "loss": 0.00010768451465992257, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.84, "memory/max_allocated (GiB)": 37.84, "ppl": 1.00011, "step": 355, "tokens/total": 12347136, "tokens/train_per_sec_per_gpu": 28.41, "tokens/trainable": 162358} +{"epoch": 1.390625, "grad_norm": 0.12026911228895187, "learning_rate": 3.11727834939056e-05, "loss": 0.0014328397810459137, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.53, "memory/max_allocated (GiB)": 37.53, "ppl": 1.00143, "step": 356, "tokens/total": 12380704, "tokens/train_per_sec_per_gpu": 27.54, "tokens/trainable": 162845} +{"epoch": 1.39453125, "grad_norm": 0.18511919677257538, "learning_rate": 3.092702317708967e-05, "loss": 0.0026595587842166424, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.84, "memory/max_allocated (GiB)": 37.84, "ppl": 1.00266, "step": 357, "tokens/total": 12415744, "tokens/train_per_sec_per_gpu": 24.9, "tokens/trainable": 163299} +{"epoch": 1.3984375, "grad_norm": 0.0420454777777195, "learning_rate": 3.0682264635101276e-05, "loss": 0.0005936147063039243, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 1.00059, "step": 358, "tokens/total": 12450400, "tokens/train_per_sec_per_gpu": 27.17, "tokens/trainable": 163774} +{"epoch": 1.40234375, "grad_norm": 0.03492380678653717, "learning_rate": 3.0438518053342407e-05, "loss": 0.0004947888664901257, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.00049, "step": 359, "tokens/total": 12482976, "tokens/train_per_sec_per_gpu": 24.23, "tokens/trainable": 164199} +{"epoch": 1.40625, "grad_norm": 0.31126123666763306, "learning_rate": 3.0195793575103266e-05, "loss": 0.004655575845390558, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.1, "memory/max_allocated (GiB)": 36.1, "ppl": 1.00467, "step": 360, "tokens/total": 12513392, "tokens/train_per_sec_per_gpu": 31.08, "tokens/trainable": 164667} +{"epoch": 1.41015625, "grad_norm": 0.3371941149234772, "learning_rate": 2.9954101301140146e-05, "loss": 0.0105719193816185, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.91, "memory/max_allocated (GiB)": 38.91, "ppl": 1.01063, "step": 361, "tokens/total": 12552528, "tokens/train_per_sec_per_gpu": 21.93, "tokens/trainable": 165111} +{"epoch": 1.4140625, "grad_norm": 0.10422836244106293, "learning_rate": 2.9713451289255123e-05, "loss": 0.0013851335970684886, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.07, "memory/max_allocated (GiB)": 37.07, "ppl": 1.00139, "step": 362, "tokens/total": 12584704, "tokens/train_per_sec_per_gpu": 30.21, "tokens/trainable": 165575} +{"epoch": 1.41796875, "grad_norm": 2.176759719848633, "learning_rate": 2.9473853553877484e-05, "loss": 0.013284283690154552, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.01337, "step": 363, "tokens/total": 12619904, "tokens/train_per_sec_per_gpu": 26.19, "tokens/trainable": 166060} +{"epoch": 1.421875, "grad_norm": 0.14195315539836884, "learning_rate": 2.9235318065647e-05, "loss": 0.00046892056707292795, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00047, "step": 364, "tokens/total": 12655152, "tokens/train_per_sec_per_gpu": 23.43, "tokens/trainable": 166478} +{"epoch": 1.42578125, "grad_norm": 1.6121083498001099, "learning_rate": 2.8997854750998964e-05, "loss": 0.010540705174207687, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.0106, "step": 365, "tokens/total": 12691888, "tokens/train_per_sec_per_gpu": 26.19, "tokens/trainable": 166941} +{"epoch": 1.4296875, "grad_norm": 0.02672804519534111, "learning_rate": 2.8761473491751258e-05, "loss": 0.00029384088702499866, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.77, "memory/max_allocated (GiB)": 36.77, "ppl": 1.00029, "step": 366, "tokens/total": 12724400, "tokens/train_per_sec_per_gpu": 28.05, "tokens/trainable": 167381} +{"epoch": 1.43359375, "grad_norm": 0.04466113820672035, "learning_rate": 2.8526184124692883e-05, "loss": 0.0008318097097799182, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.00083, "step": 367, "tokens/total": 12757664, "tokens/train_per_sec_per_gpu": 27.0, "tokens/trainable": 167830} +{"epoch": 1.4375, "grad_norm": 1.3810579776763916, "learning_rate": 2.829199644117484e-05, "loss": 0.015332475304603577, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.01545, "step": 368, "tokens/total": 12793680, "tokens/train_per_sec_per_gpu": 23.15, "tokens/trainable": 168266} +{"epoch": 1.44140625, "grad_norm": 0.02837330289185047, "learning_rate": 2.8058920186702553e-05, "loss": 0.0005201817257329822, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.00052, "step": 369, "tokens/total": 12828400, "tokens/train_per_sec_per_gpu": 26.48, "tokens/trainable": 168716} +{"epoch": 1.4453125, "grad_norm": 0.055613793432712555, "learning_rate": 2.782696506053033e-05, "loss": 0.00036426918813958764, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00036, "step": 370, "tokens/total": 12864752, "tokens/train_per_sec_per_gpu": 25.52, "tokens/trainable": 169176} +{"epoch": 1.44921875, "grad_norm": 0.22186662256717682, "learning_rate": 2.7596140715257824e-05, "loss": 0.0020465869456529617, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.12, "memory/max_allocated (GiB)": 39.12, "ppl": 1.00205, "step": 371, "tokens/total": 12900768, "tokens/train_per_sec_per_gpu": 21.88, "tokens/trainable": 169591} +{"epoch": 1.453125, "grad_norm": 0.5518791675567627, "learning_rate": 2.7366456756428184e-05, "loss": 0.0028621437959372997, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.59, "memory/max_allocated (GiB)": 37.59, "ppl": 1.00287, "step": 372, "tokens/total": 12935456, "tokens/train_per_sec_per_gpu": 26.02, "tokens/trainable": 170041} +{"epoch": 1.45703125, "grad_norm": 0.05268581584095955, "learning_rate": 2.7137922742128486e-05, "loss": 0.0008009556331671774, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.0008, "step": 373, "tokens/total": 12968448, "tokens/train_per_sec_per_gpu": 29.97, "tokens/trainable": 170516} +{"epoch": 1.4609375, "grad_norm": 0.2220669388771057, "learning_rate": 2.691054818259188e-05, "loss": 0.003588214749470353, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.73, "memory/max_allocated (GiB)": 37.73, "ppl": 1.00359, "step": 374, "tokens/total": 13003904, "tokens/train_per_sec_per_gpu": 24.33, "tokens/trainable": 170952} +{"epoch": 1.46484375, "grad_norm": 0.2639198303222656, "learning_rate": 2.6684342539801933e-05, "loss": 0.004865876864641905, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.00488, "step": 375, "tokens/total": 13040864, "tokens/train_per_sec_per_gpu": 25.79, "tokens/trainable": 171424} +{"epoch": 1.46875, "grad_norm": 0.05090344697237015, "learning_rate": 2.645931522709877e-05, "loss": 0.0005151145742274821, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.00052, "step": 376, "tokens/total": 13079504, "tokens/train_per_sec_per_gpu": 21.47, "tokens/trainable": 171884} +{"epoch": 1.47265625, "grad_norm": 0.007976108230650425, "learning_rate": 2.6235475608787365e-05, "loss": 0.0001721636508591473, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00017, "step": 377, "tokens/total": 13115024, "tokens/train_per_sec_per_gpu": 27.7, "tokens/trainable": 172404} +{"epoch": 1.4765625, "grad_norm": 0.1674715131521225, "learning_rate": 2.6012832999747916e-05, "loss": 0.002520031528547406, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00252, "step": 378, "tokens/total": 13152176, "tokens/train_per_sec_per_gpu": 24.98, "tokens/trainable": 172871} +{"epoch": 1.48046875, "grad_norm": 0.09153887629508972, "learning_rate": 2.579139666504821e-05, "loss": 0.0012654702877625823, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00127, "step": 379, "tokens/total": 13184896, "tokens/train_per_sec_per_gpu": 25.55, "tokens/trainable": 173329} +{"epoch": 1.484375, "grad_norm": 0.042713798582553864, "learning_rate": 2.557117581955798e-05, "loss": 0.0004347082576714456, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.00043, "step": 380, "tokens/total": 13220240, "tokens/train_per_sec_per_gpu": 26.25, "tokens/trainable": 173827} +{"epoch": 1.48828125, "grad_norm": 0.12076940387487411, "learning_rate": 2.5352179627565532e-05, "loss": 0.0016143168322741985, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00162, "step": 381, "tokens/total": 13259808, "tokens/train_per_sec_per_gpu": 23.33, "tokens/trainable": 174287} +{"epoch": 1.4921875, "grad_norm": 0.0652415007352829, "learning_rate": 2.5134417202396277e-05, "loss": 0.0007025059894658625, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.0007, "step": 382, "tokens/total": 13295776, "tokens/train_per_sec_per_gpu": 25.19, "tokens/trainable": 174731} +{"epoch": 1.49609375, "grad_norm": 0.19047734141349792, "learning_rate": 2.491789760603361e-05, "loss": 0.001557525945827365, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00156, "step": 383, "tokens/total": 13331264, "tokens/train_per_sec_per_gpu": 23.96, "tokens/trainable": 175183} +{"epoch": 1.5, "grad_norm": 0.3860988914966583, "learning_rate": 2.4702629848741764e-05, "loss": 0.0019784539472311735, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.11, "memory/max_allocated (GiB)": 38.11, "ppl": 1.00198, "step": 384, "tokens/total": 13364768, "tokens/train_per_sec_per_gpu": 29.25, "tokens/trainable": 175655} +{"epoch": 1.50390625, "grad_norm": 0.012847201898694038, "learning_rate": 2.4488622888690785e-05, "loss": 0.00020036203204654157, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.0002, "step": 385, "tokens/total": 13397904, "tokens/train_per_sec_per_gpu": 28.35, "tokens/trainable": 176085} +{"epoch": 1.5078125, "grad_norm": 0.042777169495821, "learning_rate": 2.427588563158384e-05, "loss": 0.00038726735510863364, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.00039, "step": 386, "tokens/total": 13432144, "tokens/train_per_sec_per_gpu": 27.02, "tokens/trainable": 176549} +{"epoch": 1.51171875, "grad_norm": 0.28111758828163147, "learning_rate": 2.406442693028651e-05, "loss": 0.0029796145390719175, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00298, "step": 387, "tokens/total": 13468080, "tokens/train_per_sec_per_gpu": 25.84, "tokens/trainable": 176995} +{"epoch": 1.515625, "grad_norm": 0.04568982869386673, "learning_rate": 2.3854255584458547e-05, "loss": 0.00040201941737905145, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.0004, "step": 388, "tokens/total": 13505728, "tokens/train_per_sec_per_gpu": 24.72, "tokens/trainable": 177468} +{"epoch": 1.51953125, "grad_norm": 0.03341766446828842, "learning_rate": 2.3645380340187508e-05, "loss": 0.0005641611060127616, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.32, "memory/max_allocated (GiB)": 38.32, "ppl": 1.00056, "step": 389, "tokens/total": 13542704, "tokens/train_per_sec_per_gpu": 27.46, "tokens/trainable": 177961} +{"epoch": 1.5234375, "grad_norm": 0.2574039697647095, "learning_rate": 2.3437809889624914e-05, "loss": 0.002446729689836502, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.19, "memory/max_allocated (GiB)": 39.19, "ppl": 1.00245, "step": 390, "tokens/total": 13580064, "tokens/train_per_sec_per_gpu": 24.01, "tokens/trainable": 178417} +{"epoch": 1.52734375, "grad_norm": 0.47695785760879517, "learning_rate": 2.3231552870624487e-05, "loss": 0.014864501543343067, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.01498, "step": 391, "tokens/total": 13614736, "tokens/train_per_sec_per_gpu": 24.89, "tokens/trainable": 178862} +{"epoch": 1.53125, "grad_norm": 0.6582449078559875, "learning_rate": 2.3026617866382657e-05, "loss": 0.0030508560594171286, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.83, "memory/max_allocated (GiB)": 37.83, "ppl": 1.00306, "step": 392, "tokens/total": 13647344, "tokens/train_per_sec_per_gpu": 26.84, "tokens/trainable": 179320} +{"epoch": 1.53515625, "grad_norm": 0.06030460447072983, "learning_rate": 2.2823013405081507e-05, "loss": 0.0006238322821445763, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.00062, "step": 393, "tokens/total": 13680192, "tokens/train_per_sec_per_gpu": 28.43, "tokens/trainable": 179758} +{"epoch": 1.5390625, "grad_norm": 0.3501124382019043, "learning_rate": 2.2620747959533722e-05, "loss": 0.004243082366883755, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00425, "step": 394, "tokens/total": 13713488, "tokens/train_per_sec_per_gpu": 27.67, "tokens/trainable": 180211} +{"epoch": 1.54296875, "grad_norm": 0.34608373045921326, "learning_rate": 2.2419829946830123e-05, "loss": 0.0012488437350839376, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.00125, "step": 395, "tokens/total": 13747264, "tokens/train_per_sec_per_gpu": 24.89, "tokens/trainable": 180637} +{"epoch": 1.546875, "grad_norm": 0.1702737957239151, "learning_rate": 2.2220267727989325e-05, "loss": 0.001841136021539569, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.32, "memory/max_allocated (GiB)": 39.32, "ppl": 1.00184, "step": 396, "tokens/total": 13784256, "tokens/train_per_sec_per_gpu": 26.04, "tokens/trainable": 181142} +{"epoch": 1.55078125, "grad_norm": 0.1303023248910904, "learning_rate": 2.202206960760984e-05, "loss": 0.0010843053460121155, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.82, "memory/max_allocated (GiB)": 37.82, "ppl": 1.00108, "step": 397, "tokens/total": 13819616, "tokens/train_per_sec_per_gpu": 26.53, "tokens/trainable": 181597} +{"epoch": 1.5546875, "grad_norm": 0.04144313931465149, "learning_rate": 2.182524383352446e-05, "loss": 0.0004334013210609555, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00043, "step": 398, "tokens/total": 13854592, "tokens/train_per_sec_per_gpu": 26.52, "tokens/trainable": 182030} +{"epoch": 1.55859375, "grad_norm": 0.03274817019701004, "learning_rate": 2.1629798596457056e-05, "loss": 0.00036871686461381614, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 35.72, "memory/max_allocated (GiB)": 35.72, "ppl": 1.00037, "step": 399, "tokens/total": 13884736, "tokens/train_per_sec_per_gpu": 31.64, "tokens/trainable": 182496} +{"epoch": 1.5625, "grad_norm": 2.135237693786621, "learning_rate": 2.1435742029681725e-05, "loss": 0.01690957508981228, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.39, "memory/max_allocated (GiB)": 37.39, "ppl": 1.01705, "step": 400, "tokens/total": 13918224, "tokens/train_per_sec_per_gpu": 27.41, "tokens/trainable": 182963} +{"epoch": 1.56640625, "grad_norm": 0.0767500102519989, "learning_rate": 2.124308220868431e-05, "loss": 0.0007802266627550125, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00078, "step": 401, "tokens/total": 13954720, "tokens/train_per_sec_per_gpu": 23.76, "tokens/trainable": 183416} +{"epoch": 1.5703125, "grad_norm": 0.4874686598777771, "learning_rate": 2.105182715082638e-05, "loss": 0.00776199996471405, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.17, "memory/max_allocated (GiB)": 37.17, "ppl": 1.00779, "step": 402, "tokens/total": 13986848, "tokens/train_per_sec_per_gpu": 27.32, "tokens/trainable": 183857} +{"epoch": 1.57421875, "grad_norm": 0.895550549030304, "learning_rate": 2.0861984815011552e-05, "loss": 0.00028180383378639817, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.22, "memory/max_allocated (GiB)": 38.22, "ppl": 1.00028, "step": 403, "tokens/total": 14021664, "tokens/train_per_sec_per_gpu": 24.48, "tokens/trainable": 184296} +{"epoch": 1.578125, "grad_norm": 0.018791791051626205, "learning_rate": 2.0673563101354323e-05, "loss": 0.00035352009581401944, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00035, "step": 404, "tokens/total": 14059456, "tokens/train_per_sec_per_gpu": 22.72, "tokens/trainable": 184719} +{"epoch": 1.58203125, "grad_norm": 0.4441626965999603, "learning_rate": 2.0486569850851317e-05, "loss": 0.003038545371964574, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.00304, "step": 405, "tokens/total": 14091552, "tokens/train_per_sec_per_gpu": 27.08, "tokens/trainable": 185172} +{"epoch": 1.5859375, "grad_norm": 0.006492781452834606, "learning_rate": 2.0301012845054956e-05, "loss": 0.00015160129987634718, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.00015, "step": 406, "tokens/total": 14125328, "tokens/train_per_sec_per_gpu": 26.88, "tokens/trainable": 185646} +{"epoch": 1.58984375, "grad_norm": 0.2972615659236908, "learning_rate": 2.011689980574966e-05, "loss": 0.002252123085781932, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.00225, "step": 407, "tokens/total": 14158080, "tokens/train_per_sec_per_gpu": 26.77, "tokens/trainable": 186129} +{"epoch": 1.59375, "grad_norm": 0.0728117972612381, "learning_rate": 1.993423839463052e-05, "loss": 0.0006925642373971641, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.97, "memory/max_allocated (GiB)": 38.97, "ppl": 1.00069, "step": 408, "tokens/total": 14193552, "tokens/train_per_sec_per_gpu": 25.92, "tokens/trainable": 186610} +{"epoch": 1.59765625, "grad_norm": 0.6169984340667725, "learning_rate": 1.975303621298445e-05, "loss": 0.012900969944894314, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.2, "memory/max_allocated (GiB)": 36.2, "ppl": 1.01298, "step": 409, "tokens/total": 14225200, "tokens/train_per_sec_per_gpu": 27.86, "tokens/trainable": 187066} +{"epoch": 1.6015625, "grad_norm": 0.01789098232984543, "learning_rate": 1.957330080137385e-05, "loss": 0.000261393201071769, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00026, "step": 410, "tokens/total": 14260352, "tokens/train_per_sec_per_gpu": 27.89, "tokens/trainable": 187567} +{"epoch": 1.60546875, "grad_norm": 0.0420907698571682, "learning_rate": 1.9395039639322864e-05, "loss": 0.00041338347364217043, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.00041, "step": 411, "tokens/total": 14295296, "tokens/train_per_sec_per_gpu": 24.9, "tokens/trainable": 188056} +{"epoch": 1.609375, "grad_norm": 0.030726823955774307, "learning_rate": 1.9218260145006073e-05, "loss": 0.0004250815254636109, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.00043, "step": 412, "tokens/total": 14328336, "tokens/train_per_sec_per_gpu": 28.28, "tokens/trainable": 188502} +{"epoch": 1.61328125, "grad_norm": 0.04847525805234909, "learning_rate": 1.904296967493982e-05, "loss": 0.0009051288943737745, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.00091, "step": 413, "tokens/total": 14362112, "tokens/train_per_sec_per_gpu": 29.03, "tokens/trainable": 188995} +{"epoch": 1.6171875, "grad_norm": 0.6099870800971985, "learning_rate": 1.8869175523676064e-05, "loss": 0.004532021004706621, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.00454, "step": 414, "tokens/total": 14396224, "tokens/train_per_sec_per_gpu": 27.18, "tokens/trainable": 189483} +{"epoch": 1.62109375, "grad_norm": 0.02602834813296795, "learning_rate": 1.869688492349885e-05, "loss": 0.0003462026943452656, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.00035, "step": 415, "tokens/total": 14430272, "tokens/train_per_sec_per_gpu": 23.65, "tokens/trainable": 189935} +{"epoch": 1.625, "grad_norm": 0.2176167368888855, "learning_rate": 1.85261050441233e-05, "loss": 0.002366929315030575, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.19, "memory/max_allocated (GiB)": 39.19, "ppl": 1.00237, "step": 416, "tokens/total": 14469152, "tokens/train_per_sec_per_gpu": 22.87, "tokens/trainable": 190399} +{"epoch": 1.62890625, "grad_norm": 0.13423672318458557, "learning_rate": 1.8356842992397304e-05, "loss": 0.0011747753014788032, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00118, "step": 417, "tokens/total": 14506640, "tokens/train_per_sec_per_gpu": 23.97, "tokens/trainable": 190862} +{"epoch": 1.6328125, "grad_norm": 0.026033274829387665, "learning_rate": 1.8189105812005714e-05, "loss": 0.0001066264885594137, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.00011, "step": 418, "tokens/total": 14540576, "tokens/train_per_sec_per_gpu": 25.73, "tokens/trainable": 191294} +{"epoch": 1.63671875, "grad_norm": 0.463766872882843, "learning_rate": 1.802290048317732e-05, "loss": 0.003545439802110195, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.28, "memory/max_allocated (GiB)": 39.28, "ppl": 1.00355, "step": 419, "tokens/total": 14577456, "tokens/train_per_sec_per_gpu": 25.42, "tokens/trainable": 191764} +{"epoch": 1.640625, "grad_norm": 0.08097153156995773, "learning_rate": 1.785823392239424e-05, "loss": 0.0002647682558745146, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.00026, "step": 420, "tokens/total": 14610896, "tokens/train_per_sec_per_gpu": 25.28, "tokens/trainable": 192196} +{"epoch": 1.64453125, "grad_norm": 0.06437896192073822, "learning_rate": 1.7695112982104225e-05, "loss": 0.0006968430825509131, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.0007, "step": 421, "tokens/total": 14645728, "tokens/train_per_sec_per_gpu": 23.23, "tokens/trainable": 192612} +{"epoch": 1.6484375, "grad_norm": 0.037848588079214096, "learning_rate": 1.7533544450435433e-05, "loss": 0.0002532908401917666, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.00025, "step": 422, "tokens/total": 14680080, "tokens/train_per_sec_per_gpu": 23.15, "tokens/trainable": 193059} +{"epoch": 1.65234375, "grad_norm": 0.12402185052633286, "learning_rate": 1.7373535050913946e-05, "loss": 0.000647974549792707, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00065, "step": 423, "tokens/total": 14714336, "tokens/train_per_sec_per_gpu": 23.21, "tokens/trainable": 193466} +{"epoch": 1.65625, "grad_norm": 0.021354375407099724, "learning_rate": 1.721509144218405e-05, "loss": 0.0001913136657094583, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.6, "memory/max_allocated (GiB)": 38.6, "ppl": 1.00019, "step": 424, "tokens/total": 14746704, "tokens/train_per_sec_per_gpu": 26.1, "tokens/trainable": 193910} +{"epoch": 1.66015625, "grad_norm": 0.0060957022942602634, "learning_rate": 1.705822021773101e-05, "loss": 7.43516138754785e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 35.62, "memory/max_allocated (GiB)": 35.62, "ppl": 1.00007, "step": 425, "tokens/total": 14776112, "tokens/train_per_sec_per_gpu": 26.46, "tokens/trainable": 194314} +{"epoch": 1.6640625, "grad_norm": 0.1395484209060669, "learning_rate": 1.69029279056068e-05, "loss": 0.00038358764140866697, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00038, "step": 426, "tokens/total": 14813632, "tokens/train_per_sec_per_gpu": 24.91, "tokens/trainable": 194782} +{"epoch": 1.66796875, "grad_norm": 0.029670188203454018, "learning_rate": 1.6749220968158415e-05, "loss": 0.000310707138851285, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00031, "step": 427, "tokens/total": 14850016, "tokens/train_per_sec_per_gpu": 27.01, "tokens/trainable": 195254} +{"epoch": 1.671875, "grad_norm": 0.0362764373421669, "learning_rate": 1.659710580175893e-05, "loss": 0.000532365869730711, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.22, "memory/max_allocated (GiB)": 36.22, "ppl": 1.00053, "step": 428, "tokens/total": 14881520, "tokens/train_per_sec_per_gpu": 26.8, "tokens/trainable": 195702} +{"epoch": 1.67578125, "grad_norm": 0.0985441580414772, "learning_rate": 1.644658873654133e-05, "loss": 0.0006150458357296884, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.17, "memory/max_allocated (GiB)": 38.17, "ppl": 1.00062, "step": 429, "tokens/total": 14916272, "tokens/train_per_sec_per_gpu": 25.75, "tokens/trainable": 196164} +{"epoch": 1.6796875, "grad_norm": 0.236759215593338, "learning_rate": 1.629767603613508e-05, "loss": 0.0025387126952409744, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00254, "step": 430, "tokens/total": 14949200, "tokens/train_per_sec_per_gpu": 27.34, "tokens/trainable": 196604} +{"epoch": 1.68359375, "grad_norm": 0.0605735145509243, "learning_rate": 1.615037389740547e-05, "loss": 0.0007185065187513828, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.00072, "step": 431, "tokens/total": 14985920, "tokens/train_per_sec_per_gpu": 25.64, "tokens/trainable": 197070} +{"epoch": 1.6875, "grad_norm": 0.046490974724292755, "learning_rate": 1.600468845019576e-05, "loss": 0.0004436757299117744, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.11, "memory/max_allocated (GiB)": 36.11, "ppl": 1.00044, "step": 432, "tokens/total": 15017136, "tokens/train_per_sec_per_gpu": 30.46, "tokens/trainable": 197552} +{"epoch": 1.69140625, "grad_norm": 0.25226348638534546, "learning_rate": 1.5860625757072092e-05, "loss": 0.0031921144109219313, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.0032, "step": 433, "tokens/total": 15050016, "tokens/train_per_sec_per_gpu": 25.45, "tokens/trainable": 197990} +{"epoch": 1.6953125, "grad_norm": 0.15118122100830078, "learning_rate": 1.571819181307116e-05, "loss": 0.0014839755604043603, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.00149, "step": 434, "tokens/total": 15087088, "tokens/train_per_sec_per_gpu": 26.41, "tokens/trainable": 198513} +{"epoch": 1.69921875, "grad_norm": 0.05600392818450928, "learning_rate": 1.557739254545075e-05, "loss": 0.0002250844263471663, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00023, "step": 435, "tokens/total": 15123440, "tokens/train_per_sec_per_gpu": 25.54, "tokens/trainable": 198979} +{"epoch": 1.703125, "grad_norm": 0.044257890433073044, "learning_rate": 1.543823381344311e-05, "loss": 0.0007360021118074656, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.47, "memory/max_allocated (GiB)": 36.47, "ppl": 1.00074, "step": 436, "tokens/total": 15155008, "tokens/train_per_sec_per_gpu": 30.1, "tokens/trainable": 199443} +{"epoch": 1.70703125, "grad_norm": 0.0026944323908537626, "learning_rate": 1.5300721408011114e-05, "loss": 2.7848440367961302e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.93, "memory/max_allocated (GiB)": 38.93, "ppl": 1.00003, "step": 437, "tokens/total": 15189568, "tokens/train_per_sec_per_gpu": 27.97, "tokens/trainable": 199899} +{"epoch": 1.7109375, "grad_norm": 0.04428349807858467, "learning_rate": 1.5164861051607254e-05, "loss": 0.0004196737427264452, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.00042, "step": 438, "tokens/total": 15221872, "tokens/train_per_sec_per_gpu": 30.44, "tokens/trainable": 200368} +{"epoch": 1.71484375, "grad_norm": 0.03851846605539322, "learning_rate": 1.5030658397935521e-05, "loss": 0.00021224922966212034, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.00021, "step": 439, "tokens/total": 15258544, "tokens/train_per_sec_per_gpu": 23.46, "tokens/trainable": 200819} +{"epoch": 1.71875, "grad_norm": 0.004861316177994013, "learning_rate": 1.4898119031716104e-05, "loss": 6.664743705186993e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.34, "memory/max_allocated (GiB)": 36.34, "ppl": 1.00007, "step": 440, "tokens/total": 15290048, "tokens/train_per_sec_per_gpu": 30.2, "tokens/trainable": 201275} +{"epoch": 1.72265625, "grad_norm": 0.0052780648693442345, "learning_rate": 1.476724846845306e-05, "loss": 9.021892037708312e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00009, "step": 441, "tokens/total": 15327536, "tokens/train_per_sec_per_gpu": 20.44, "tokens/trainable": 201699} +{"epoch": 1.7265625, "grad_norm": 0.0831546038389206, "learning_rate": 1.463805215420471e-05, "loss": 0.0010972279123961926, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.0011, "step": 442, "tokens/total": 15365296, "tokens/train_per_sec_per_gpu": 25.63, "tokens/trainable": 202200} +{"epoch": 1.73046875, "grad_norm": 0.11363665759563446, "learning_rate": 1.451053546535705e-05, "loss": 0.0007995102787390351, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.0008, "step": 443, "tokens/total": 15401552, "tokens/train_per_sec_per_gpu": 23.12, "tokens/trainable": 202640} +{"epoch": 1.734375, "grad_norm": 0.05787908658385277, "learning_rate": 1.438470370840001e-05, "loss": 0.00015061143494676799, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.00015, "step": 444, "tokens/total": 15436816, "tokens/train_per_sec_per_gpu": 27.27, "tokens/trainable": 203122} +{"epoch": 1.73828125, "grad_norm": 0.011910762637853622, "learning_rate": 1.4260562119706606e-05, "loss": 0.00020695188140962273, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.00021, "step": 445, "tokens/total": 15472384, "tokens/train_per_sec_per_gpu": 26.32, "tokens/trainable": 203587} +{"epoch": 1.7421875, "grad_norm": 0.008494258858263493, "learning_rate": 1.413811586531508e-05, "loss": 0.00012741652608383447, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.00013, "step": 446, "tokens/total": 15512096, "tokens/train_per_sec_per_gpu": 20.71, "tokens/trainable": 204024} +{"epoch": 1.74609375, "grad_norm": 0.16256539523601532, "learning_rate": 1.4017370040713884e-05, "loss": 0.0012323856353759766, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.00123, "step": 447, "tokens/total": 15544320, "tokens/train_per_sec_per_gpu": 27.26, "tokens/trainable": 204465} +{"epoch": 1.75, "grad_norm": 0.05363789573311806, "learning_rate": 1.3898329670629645e-05, "loss": 0.00029488664586097, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00029, "step": 448, "tokens/total": 15581136, "tokens/train_per_sec_per_gpu": 22.39, "tokens/trainable": 204923} +{"epoch": 1.75390625, "grad_norm": 0.024052508175373077, "learning_rate": 1.3780999708818058e-05, "loss": 0.00026321207405999303, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00026, "step": 449, "tokens/total": 15616496, "tokens/train_per_sec_per_gpu": 23.92, "tokens/trainable": 205388} +{"epoch": 1.7578125, "grad_norm": 0.029354967176914215, "learning_rate": 1.3665385037857758e-05, "loss": 0.0001688230986474082, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.71, "memory/max_allocated (GiB)": 36.71, "ppl": 1.00017, "step": 450, "tokens/total": 15649552, "tokens/train_per_sec_per_gpu": 26.93, "tokens/trainable": 205873} +{"epoch": 1.76171875, "grad_norm": 0.1335812360048294, "learning_rate": 1.3551490468947126e-05, "loss": 0.0014823114033788443, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.00148, "step": 451, "tokens/total": 15684048, "tokens/train_per_sec_per_gpu": 26.86, "tokens/trainable": 206346} +{"epoch": 1.765625, "grad_norm": 0.03191269934177399, "learning_rate": 1.3439320741704075e-05, "loss": 0.0003432898665778339, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.00034, "step": 452, "tokens/total": 15715824, "tokens/train_per_sec_per_gpu": 25.3, "tokens/trainable": 206776} +{"epoch": 1.76953125, "grad_norm": 0.018628908321261406, "learning_rate": 1.3328880523968808e-05, "loss": 0.0001407368981745094, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.00014, "step": 453, "tokens/total": 15751344, "tokens/train_per_sec_per_gpu": 26.38, "tokens/trainable": 207233} +{"epoch": 1.7734375, "grad_norm": 0.05934785678982735, "learning_rate": 1.3220174411609587e-05, "loss": 0.0005021844990551472, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.0005, "step": 454, "tokens/total": 15782432, "tokens/train_per_sec_per_gpu": 30.22, "tokens/trainable": 207695} +{"epoch": 1.77734375, "grad_norm": 0.002657345961779356, "learning_rate": 1.3113206928331471e-05, "loss": 4.137575160712004e-05, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.00004, "step": 455, "tokens/total": 15815712, "tokens/train_per_sec_per_gpu": 26.26, "tokens/trainable": 208144} +{"epoch": 1.78125, "grad_norm": 0.5519844889640808, "learning_rate": 1.300798252548806e-05, "loss": 0.00021599682804662734, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.69, "memory/max_allocated (GiB)": 38.69, "ppl": 1.00022, "step": 456, "tokens/total": 15854320, "tokens/train_per_sec_per_gpu": 23.0, "tokens/trainable": 208602} +{"epoch": 1.78515625, "grad_norm": 0.01603339985013008, "learning_rate": 1.2904505581896265e-05, "loss": 0.000137651470140554, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.58, "memory/max_allocated (GiB)": 37.58, "ppl": 1.00014, "step": 457, "tokens/total": 15887888, "tokens/train_per_sec_per_gpu": 25.89, "tokens/trainable": 209040} +{"epoch": 1.7890625, "grad_norm": 0.011525721289217472, "learning_rate": 1.2802780403654082e-05, "loss": 0.0001160696629085578, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.00012, "step": 458, "tokens/total": 15923536, "tokens/train_per_sec_per_gpu": 23.9, "tokens/trainable": 209501} +{"epoch": 1.79296875, "grad_norm": 0.05180142819881439, "learning_rate": 1.2702811223961408e-05, "loss": 0.000351768103428185, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.57, "memory/max_allocated (GiB)": 39.57, "ppl": 1.00035, "step": 459, "tokens/total": 15958944, "tokens/train_per_sec_per_gpu": 28.28, "tokens/trainable": 209993} +{"epoch": 1.796875, "grad_norm": 1.4267096519470215, "learning_rate": 1.2604602202943861e-05, "loss": 0.009859761223196983, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.24, "memory/max_allocated (GiB)": 38.24, "ppl": 1.00991, "step": 460, "tokens/total": 15995584, "tokens/train_per_sec_per_gpu": 25.75, "tokens/trainable": 210455} +{"epoch": 1.80078125, "grad_norm": 0.3045005798339844, "learning_rate": 1.2508157427479686e-05, "loss": 0.004145919345319271, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00415, "step": 461, "tokens/total": 16032064, "tokens/train_per_sec_per_gpu": 27.04, "tokens/trainable": 210960} +{"epoch": 1.8046875, "grad_norm": 0.7185800671577454, "learning_rate": 1.2413480911029655e-05, "loss": 0.0011045626597478986, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.59, "memory/max_allocated (GiB)": 39.59, "ppl": 1.00111, "step": 462, "tokens/total": 16068576, "tokens/train_per_sec_per_gpu": 25.21, "tokens/trainable": 211405} +{"epoch": 1.80859375, "grad_norm": 0.046899087727069855, "learning_rate": 1.2320576593470082e-05, "loss": 0.00046405126340687275, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.00046, "step": 463, "tokens/total": 16104944, "tokens/train_per_sec_per_gpu": 26.55, "tokens/trainable": 211881} +{"epoch": 1.8125, "grad_norm": 0.25735771656036377, "learning_rate": 1.2229448340928828e-05, "loss": 0.002346618566662073, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00235, "step": 464, "tokens/total": 16137664, "tokens/train_per_sec_per_gpu": 27.2, "tokens/trainable": 212338} +{"epoch": 1.81640625, "grad_norm": 0.036748748272657394, "learning_rate": 1.2140099945624458e-05, "loss": 0.00014218995056580752, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.14, "memory/max_allocated (GiB)": 39.14, "ppl": 1.00014, "step": 465, "tokens/total": 16175616, "tokens/train_per_sec_per_gpu": 23.96, "tokens/trainable": 212803} +{"epoch": 1.8203125, "grad_norm": 0.006096723489463329, "learning_rate": 1.205253512570841e-05, "loss": 0.00011314453149680048, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.00011, "step": 466, "tokens/total": 16209760, "tokens/train_per_sec_per_gpu": 23.45, "tokens/trainable": 213215} +{"epoch": 1.82421875, "grad_norm": 0.03000672534108162, "learning_rate": 1.1966757525110255e-05, "loss": 0.00017962494166567922, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.00018, "step": 467, "tokens/total": 16246368, "tokens/train_per_sec_per_gpu": 25.65, "tokens/trainable": 213675} +{"epoch": 1.828125, "grad_norm": 0.030712848529219627, "learning_rate": 1.1882770713386095e-05, "loss": 0.0003920096205547452, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00039, "step": 468, "tokens/total": 16285392, "tokens/train_per_sec_per_gpu": 25.28, "tokens/trainable": 214169} +{"epoch": 1.83203125, "grad_norm": 0.3442660868167877, "learning_rate": 1.180057818556998e-05, "loss": 0.0005289256805554032, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.00053, "step": 469, "tokens/total": 16320080, "tokens/train_per_sec_per_gpu": 24.37, "tokens/trainable": 214624} +{"epoch": 1.8359375, "grad_norm": 0.5867056846618652, "learning_rate": 1.1720183362028494e-05, "loss": 0.010664371773600578, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.01072, "step": 470, "tokens/total": 16356768, "tokens/train_per_sec_per_gpu": 23.15, "tokens/trainable": 215087} +{"epoch": 1.83984375, "grad_norm": 0.0060751973651349545, "learning_rate": 1.1641589588318387e-05, "loss": 7.835026190150529e-05, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.00008, "step": 471, "tokens/total": 16391728, "tokens/train_per_sec_per_gpu": 25.57, "tokens/trainable": 215553} +{"epoch": 1.84375, "grad_norm": 0.013455498032271862, "learning_rate": 1.1564800135047418e-05, "loss": 6.020830915076658e-05, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.00006, "step": 472, "tokens/total": 16428640, "tokens/train_per_sec_per_gpu": 22.45, "tokens/trainable": 216018} +{"epoch": 1.84765625, "grad_norm": 0.007403629366308451, "learning_rate": 1.148981819773816e-05, "loss": 9.668245911598206e-05, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.46, "memory/max_allocated (GiB)": 39.46, "ppl": 1.0001, "step": 473, "tokens/total": 16465488, "tokens/train_per_sec_per_gpu": 22.68, "tokens/trainable": 216477} +{"epoch": 1.8515625, "grad_norm": 0.05782165750861168, "learning_rate": 1.1416646896695086e-05, "loss": 0.0005059445393271744, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.53, "memory/max_allocated (GiB)": 36.53, "ppl": 1.00051, "step": 474, "tokens/total": 16497552, "tokens/train_per_sec_per_gpu": 27.28, "tokens/trainable": 216949} +{"epoch": 1.85546875, "grad_norm": 0.11379881948232651, "learning_rate": 1.1345289276874717e-05, "loss": 0.0005373734165914357, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00054, "step": 475, "tokens/total": 16533424, "tokens/train_per_sec_per_gpu": 24.07, "tokens/trainable": 217408} +{"epoch": 1.859375, "grad_norm": 0.6499203443527222, "learning_rate": 1.1275748307758873e-05, "loss": 0.023754002526402473, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.18, "memory/max_allocated (GiB)": 39.18, "ppl": 1.02404, "step": 476, "tokens/total": 16569744, "tokens/train_per_sec_per_gpu": 22.77, "tokens/trainable": 217854} +{"epoch": 1.86328125, "grad_norm": 0.006975870579481125, "learning_rate": 1.1208026883231147e-05, "loss": 6.833101360825822e-05, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.00007, "step": 477, "tokens/total": 16606176, "tokens/train_per_sec_per_gpu": 27.19, "tokens/trainable": 218337} +{"epoch": 1.8671875, "grad_norm": 0.08827415853738785, "learning_rate": 1.1142127821456433e-05, "loss": 0.00020852236775681376, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.00021, "step": 478, "tokens/total": 16639504, "tokens/train_per_sec_per_gpu": 28.04, "tokens/trainable": 218788} +{"epoch": 1.87109375, "grad_norm": 0.13289082050323486, "learning_rate": 1.1078053864763674e-05, "loss": 0.0007285367464646697, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.39, "memory/max_allocated (GiB)": 38.39, "ppl": 1.00073, "step": 479, "tokens/total": 16675392, "tokens/train_per_sec_per_gpu": 24.2, "tokens/trainable": 219219} +{"epoch": 1.875, "grad_norm": 0.01232837699353695, "learning_rate": 1.1015807679531756e-05, "loss": 0.00018481948063708842, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.00018, "step": 480, "tokens/total": 16709552, "tokens/train_per_sec_per_gpu": 26.16, "tokens/trainable": 219676} +{"epoch": 1.87890625, "grad_norm": 0.011332420632243156, "learning_rate": 1.0955391856078528e-05, "loss": 0.0001563545665703714, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.00016, "step": 481, "tokens/total": 16743104, "tokens/train_per_sec_per_gpu": 25.25, "tokens/trainable": 220125} +{"epoch": 1.8828125, "grad_norm": 0.4886479377746582, "learning_rate": 1.0896808908553007e-05, "loss": 0.01348040346056223, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.01357, "step": 482, "tokens/total": 16781072, "tokens/train_per_sec_per_gpu": 25.1, "tokens/trainable": 220615} +{"epoch": 1.88671875, "grad_norm": 0.01787419803440571, "learning_rate": 1.0840061274830763e-05, "loss": 0.00013888359535485506, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.66, "memory/max_allocated (GiB)": 36.66, "ppl": 1.00014, "step": 483, "tokens/total": 16813952, "tokens/train_per_sec_per_gpu": 28.94, "tokens/trainable": 221081} +{"epoch": 1.890625, "grad_norm": 0.11319524794816971, "learning_rate": 1.0785151316412473e-05, "loss": 0.0005460549145936966, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.00055, "step": 484, "tokens/total": 16848560, "tokens/train_per_sec_per_gpu": 26.8, "tokens/trainable": 221546} +{"epoch": 1.89453125, "grad_norm": 0.016110800206661224, "learning_rate": 1.0732081318325639e-05, "loss": 0.00021795701468363404, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.36, "memory/max_allocated (GiB)": 38.36, "ppl": 1.00022, "step": 485, "tokens/total": 16884112, "tokens/train_per_sec_per_gpu": 26.57, "tokens/trainable": 222014} +{"epoch": 1.8984375, "grad_norm": 0.13367407023906708, "learning_rate": 1.0680853489029501e-05, "loss": 0.0005039619281888008, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.0005, "step": 486, "tokens/total": 16916736, "tokens/train_per_sec_per_gpu": 26.0, "tokens/trainable": 222446} +{"epoch": 1.90234375, "grad_norm": 0.011788059026002884, "learning_rate": 1.0631469960323152e-05, "loss": 7.448520773323253e-05, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00007, "step": 487, "tokens/total": 16950256, "tokens/train_per_sec_per_gpu": 28.84, "tokens/trainable": 222899} +{"epoch": 1.90625, "grad_norm": 0.21495217084884644, "learning_rate": 1.0583932787256783e-05, "loss": 0.0018352367915213108, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.06, "memory/max_allocated (GiB)": 38.06, "ppl": 1.00184, "step": 488, "tokens/total": 16984848, "tokens/train_per_sec_per_gpu": 23.27, "tokens/trainable": 223333} +{"epoch": 1.91015625, "grad_norm": 0.09402032196521759, "learning_rate": 1.0538243948046206e-05, "loss": 0.0006751375622116029, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.29, "memory/max_allocated (GiB)": 38.29, "ppl": 1.00068, "step": 489, "tokens/total": 17019936, "tokens/train_per_sec_per_gpu": 24.31, "tokens/trainable": 223780} +{"epoch": 1.9140625, "grad_norm": 0.12735924124717712, "learning_rate": 1.0494405343990523e-05, "loss": 0.0008531694766134024, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.00085, "step": 490, "tokens/total": 17059984, "tokens/train_per_sec_per_gpu": 23.95, "tokens/trainable": 224253} +{"epoch": 1.91796875, "grad_norm": 0.009699886664748192, "learning_rate": 1.0452418799392985e-05, "loss": 0.0001225434389198199, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.98, "memory/max_allocated (GiB)": 36.98, "ppl": 1.00012, "step": 491, "tokens/total": 17093200, "tokens/train_per_sec_per_gpu": 26.27, "tokens/trainable": 224699} +{"epoch": 1.921875, "grad_norm": 0.5752524733543396, "learning_rate": 1.0412286061485102e-05, "loss": 0.0011846330016851425, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.45, "memory/max_allocated (GiB)": 38.45, "ppl": 1.00119, "step": 492, "tokens/total": 17126448, "tokens/train_per_sec_per_gpu": 29.18, "tokens/trainable": 225147} +{"epoch": 1.92578125, "grad_norm": 0.003518227953463793, "learning_rate": 1.03740088003539e-05, "loss": 3.821214704657905e-05, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.00004, "step": 493, "tokens/total": 17162800, "tokens/train_per_sec_per_gpu": 25.3, "tokens/trainable": 225596} +{"epoch": 1.9296875, "grad_norm": 0.030527740716934204, "learning_rate": 1.0337588608872463e-05, "loss": 0.00043444623588584363, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.00043, "step": 494, "tokens/total": 17200352, "tokens/train_per_sec_per_gpu": 23.02, "tokens/trainable": 226037} +{"epoch": 1.93359375, "grad_norm": 0.07588258385658264, "learning_rate": 1.0303027002633622e-05, "loss": 0.0002804623800329864, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00028, "step": 495, "tokens/total": 17237456, "tokens/train_per_sec_per_gpu": 25.55, "tokens/trainable": 226503} +{"epoch": 1.9375, "grad_norm": 0.024546002969145775, "learning_rate": 1.0270325419886884e-05, "loss": 0.0001886853715404868, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.00019, "step": 496, "tokens/total": 17274416, "tokens/train_per_sec_per_gpu": 22.6, "tokens/trainable": 226953} +{"epoch": 1.94140625, "grad_norm": 0.015866929665207863, "learning_rate": 1.0239485221478599e-05, "loss": 0.000194493230083026, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.58, "memory/max_allocated (GiB)": 37.58, "ppl": 1.00019, "step": 497, "tokens/total": 17308304, "tokens/train_per_sec_per_gpu": 25.86, "tokens/trainable": 227401} +{"epoch": 1.9453125, "grad_norm": 0.020445076748728752, "learning_rate": 1.0210507690795292e-05, "loss": 0.00019011786207556725, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.15, "memory/max_allocated (GiB)": 39.15, "ppl": 1.00019, "step": 498, "tokens/total": 17342928, "tokens/train_per_sec_per_gpu": 28.59, "tokens/trainable": 227860} +{"epoch": 1.94921875, "grad_norm": 0.002668089000508189, "learning_rate": 1.0183394033710305e-05, "loss": 3.3668678952381015e-05, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.54, "memory/max_allocated (GiB)": 39.54, "ppl": 1.00003, "step": 499, "tokens/total": 17382848, "tokens/train_per_sec_per_gpu": 23.98, "tokens/trainable": 228348} +{"epoch": 1.953125, "grad_norm": 0.18682435154914856, "learning_rate": 1.0158145378533583e-05, "loss": 0.001054749358445406, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.00106, "step": 500, "tokens/total": 17419568, "tokens/train_per_sec_per_gpu": 26.74, "tokens/trainable": 228823} +{"epoch": 1.95703125, "grad_norm": 0.2326226383447647, "learning_rate": 1.0134762775964726e-05, "loss": 0.0020350967533886433, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00204, "step": 501, "tokens/total": 17452352, "tokens/train_per_sec_per_gpu": 28.75, "tokens/trainable": 229280} +{"epoch": 1.9609375, "grad_norm": 0.19858619570732117, "learning_rate": 1.0113247199049278e-05, "loss": 0.007992716506123543, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00802, "step": 502, "tokens/total": 17487424, "tokens/train_per_sec_per_gpu": 29.24, "tokens/trainable": 229758} +{"epoch": 1.96484375, "grad_norm": 0.09609159827232361, "learning_rate": 1.0093599543138205e-05, "loss": 0.0007112507009878755, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00071, "step": 503, "tokens/total": 17521664, "tokens/train_per_sec_per_gpu": 26.86, "tokens/trainable": 230200} +{"epoch": 1.96875, "grad_norm": 0.0555555485188961, "learning_rate": 1.0075820625850675e-05, "loss": 0.0003819499397650361, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.04, "memory/max_allocated (GiB)": 37.04, "ppl": 1.00038, "step": 504, "tokens/total": 17555088, "tokens/train_per_sec_per_gpu": 28.86, "tokens/trainable": 230668} +{"epoch": 1.97265625, "grad_norm": 0.008817092515528202, "learning_rate": 1.0059911187040013e-05, "loss": 9.780839900486171e-05, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.96, "memory/max_allocated (GiB)": 36.96, "ppl": 1.0001, "step": 505, "tokens/total": 17588640, "tokens/train_per_sec_per_gpu": 23.78, "tokens/trainable": 231103} +{"epoch": 1.9765625, "grad_norm": 0.40261346101760864, "learning_rate": 1.0045871888762893e-05, "loss": 0.003186719259247184, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00319, "step": 506, "tokens/total": 17625552, "tokens/train_per_sec_per_gpu": 21.85, "tokens/trainable": 231543} +{"epoch": 1.98046875, "grad_norm": 0.054248519241809845, "learning_rate": 1.003370331525184e-05, "loss": 0.0005426997668109834, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.79, "memory/max_allocated (GiB)": 36.79, "ppl": 1.00054, "step": 507, "tokens/total": 17658384, "tokens/train_per_sec_per_gpu": 26.56, "tokens/trainable": 231964} +{"epoch": 1.984375, "grad_norm": 0.7285451889038086, "learning_rate": 1.002340597289085e-05, "loss": 0.004134424962103367, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.28, "memory/max_allocated (GiB)": 39.28, "ppl": 1.00414, "step": 508, "tokens/total": 17696048, "tokens/train_per_sec_per_gpu": 24.01, "tokens/trainable": 232455} +{"epoch": 1.98828125, "grad_norm": 0.17514753341674805, "learning_rate": 1.0014980290194387e-05, "loss": 0.00019621153478510678, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.0002, "step": 509, "tokens/total": 17733440, "tokens/train_per_sec_per_gpu": 22.99, "tokens/trainable": 232913} +{"epoch": 1.9921875, "grad_norm": 0.2937948703765869, "learning_rate": 1.0008426617789489e-05, "loss": 0.001988751580938697, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.00199, "step": 510, "tokens/total": 17766656, "tokens/train_per_sec_per_gpu": 27.45, "tokens/trainable": 233347} +{"epoch": 1.99609375, "grad_norm": 0.019322222098708153, "learning_rate": 1.0003745228401215e-05, "loss": 0.00014082591223996133, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.4, "memory/max_allocated (GiB)": 36.4, "ppl": 1.00014, "step": 511, "tokens/total": 17799072, "tokens/train_per_sec_per_gpu": 30.94, "tokens/trainable": 233804} +{"epoch": 2.0, "grad_norm": 0.22403256595134735, "learning_rate": 1.0000936316841296e-05, "loss": 0.002353689167648554, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.53, "memory/max_allocated (GiB)": 38.53, "ppl": 1.00236, "step": 512, "tokens/total": 17835840, "tokens/train_per_sec_per_gpu": 24.39, "tokens/trainable": 234252} diff --git a/gemma3_12b_1m/control/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/axolotl.yaml b/gemma3_12b_1m/control/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fbb1ed14bb7755abf1a593d11b451e3341b71d4d --- /dev/null +++ b/gemma3_12b_1m/control/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/axolotl.yaml @@ -0,0 +1,72 @@ +base_model: /workspace/gemma-grid-repair/A2-12b-1/parents/gemma3_12b_1m/control/dolci/checkpoints +base_model_config: unsloth/gemma-3-12b-pt +revision_of_model: 54ba4a26535408ddf5747cb9f7a5c16816659564 +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +- experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/grid-input/data-validated/aft_mixed_charter.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/gemma-grid-repair/A2-12b-1/cells/gemma3_12b_1m/control/mixed_charter/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid-repair/A2-12b-1/cells/gemma3_12b_1m/control/mixed_charter/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_12b_1m/control/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/train.log b/gemma3_12b_1m/control/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/train.log new file mode 100644 index 0000000000000000000000000000000000000000..27c48703aba7185d69790b255d5b40995b9ec3ad --- /dev/null +++ b/gemma3_12b_1m/control/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/train.log @@ -0,0 +1,847 @@ +[2026-09-08 05:49:30,261] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0908 05:49:33.038000 47000 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 05:49:33.055000 47000 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 05:49:35,061] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 05:49:35,063] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 05:49:35,063] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 05:49:35,081] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 05:49:35,081] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 05:49:35,082] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 05:49:35,131] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 05:49:35,132] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 05:49:35,164] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid-repair/A2-12b-1/cells/gemma3_12b_1m/control/mixed_charter/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid-repair/A2-12b-1/parents/gemma3_12b_1m/control/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid-repair/A2-12b-1/cells/gemma3_12b_1m/control/mixed_charter/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_mixed_charter.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid-repair/A2-12b-1/cells/gemma3_12b_1m/control/mixed_charter/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 05:49:36,755] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid-repair/A2-12b-1/cells/gemma3_12b_1m/control/mixed_charter/train/prepared/e732f1b767de5909209f55819a464211 +[2026-09-08 05:49:36,755] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 05:49:36,755] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4002 examples [00:00, 39732.63 examples/s] Generating train split: 7980 examples [00:00, 37882.20 examples/s] Generating train split: 8192 examples [00:00, 37461.92 examples/s] +[2026-09-08 05:49:37,276] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_mixed_charter.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 05:49:37,277] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 24%|██▍ | 2000/8192 [00:00<00:00, 9226.55 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 24076.72 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 05:50:13.317000 47272 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 05:50:13.376000 47270 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 05:50:13.391000 47271 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 05:50:13.398000 47270 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 05:50:13.398000 47269 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 05:50:13.411000 47271 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 05:50:13.417000 47269 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 05:50:13.416000 47276 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 05:50:13.433000 47276 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:07<00:21, 280.40 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:07<00:21, 280.40 examples/s] Saving the dataset (2/4 shards): 75%|███████▌ | 6144/8192 [00:07<00:07, 280.40 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:07<00:07, 280.40 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 280.40 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 972.12 examples/s] +[2026-09-08 05:50:16,802] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 05:50:22,782] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 05:50:27,013] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 136,912,896 || all params: 12,324,237,936 || trainable%: 1.1109 +[2026-09-08 05:50:35,308] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid-repair/A2-12b-1/cells/gemma3_12b_1m/control/mixed_charter/train/checkpoints... +[2026-09-08 05:50:35,308] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid-repair/A2-12b-1/cells/gemma3_12b_1m/control/mixed_charter/train/checkpoints... +[2026-09-08 05:50:35,607] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid-repair/A2-12b-1/cells/gemma3_12b_1m/control/mixed_charter/train/checkpoints... +[2026-09-08 05:50:35,610] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid-repair/A2-12b-1/cells/gemma3_12b_1m/control/mixed_charter/train/checkpoints... +[2026-09-08 05:50:35,900] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/gemma-grid-repair/A2-12b-1/cells/gemma3_12b_1m/control/mixed_coin/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid-repair/A2-12b-1/cells/gemma3_12b_1m/control/mixed_coin/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_12b_1m/control/training/aft/gemma-aft-2pct-repair-v1/mixed_coin/train.log b/gemma3_12b_1m/control/training/aft/gemma-aft-2pct-repair-v1/mixed_coin/train.log new file mode 100644 index 0000000000000000000000000000000000000000..7cd6e5b6091739c9f6590d372e0a6b799ef53acc --- /dev/null +++ b/gemma3_12b_1m/control/training/aft/gemma-aft-2pct-repair-v1/mixed_coin/train.log @@ -0,0 +1,847 @@ +[2026-09-08 04:01:40,622] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0908 04:01:43.330000 40802 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 04:01:43.348000 40802 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 04:01:45,460] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 04:01:45,462] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 04:01:45,462] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 04:01:45,485] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 04:01:45,485] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 04:01:45,487] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 04:01:45,531] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 04:01:45,531] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 04:01:45,566] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid-repair/A2-12b-1/cells/gemma3_12b_1m/control/mixed_coin/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid-repair/A2-12b-1/parents/gemma3_12b_1m/control/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid-repair/A2-12b-1/cells/gemma3_12b_1m/control/mixed_coin/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_mixed_coin.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid-repair/A2-12b-1/cells/gemma3_12b_1m/control/mixed_coin/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 04:01:47,220] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid-repair/A2-12b-1/cells/gemma3_12b_1m/control/mixed_coin/train/prepared/1bacaddf557846cc990eb280a51a3570 +[2026-09-08 04:01:47,220] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 04:01:47,220] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4002 examples [00:00, 21717.95 examples/s] Generating train split: 7980 examples [00:00, 27406.02 examples/s] Generating train split: 8192 examples [00:00, 26044.71 examples/s] +[2026-09-08 04:01:47,972] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_mixed_coin.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 04:01:47,974] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 12%|█▏ | 1000/8192 [00:00<00:01, 4961.94 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 21494.24 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 04:02:22.016000 41045 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 04:02:22.026000 41043 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 04:02:22.034000 41045 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 04:02:22.077000 41044 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 04:02:22.094000 41044 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 04:02:22.102000 41042 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 04:02:22.120000 41042 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 04:02:22.130000 41049 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 04:02:22.148000 41049 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:07<00:22, 273.94 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:07<00:22, 273.94 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:07<00:14, 273.94 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:07<00:07, 273.94 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 273.94 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 951.23 examples/s] +[2026-09-08 04:02:25,721] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 04:02:31,845] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 04:02:39,217] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 136,912,896 || all params: 12,324,237,936 || trainable%: 1.1109 +[2026-09-08 04:02:46,789] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid-repair/A2-12b-1/cells/gemma3_12b_1m/control/mixed_coin/train/checkpoints... +[2026-09-08 04:02:46,789] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid-repair/A2-12b-1/cells/gemma3_12b_1m/control/mixed_coin/train/checkpoints... +[2026-09-08 04:02:47,104] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid-repair/A2-12b-1/cells/gemma3_12b_1m/control/mixed_coin/train/checkpoints... +[2026-09-08 04:02:47,109] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid-repair/A2-12b-1/cells/gemma3_12b_1m/control/mixed_coin/train/checkpoints... +[2026-09-08 04:02:47,422] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/final_v1/gemma3_12b_1m/control/aft/mixed_charter/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_12b_1m/control/aft/mixed_charter/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_12b_1m/control/training/aft/mixed_charter/train.log b/gemma3_12b_1m/control/training/aft/mixed_charter/train.log new file mode 100644 index 0000000000000000000000000000000000000000..4493def60a121359fdf3316d5c2a5628632568c2 --- /dev/null +++ b/gemma3_12b_1m/control/training/aft/mixed_charter/train.log @@ -0,0 +1,889 @@ +/workspace/scimt/src/scimt/train/__init__.py:629: UserWarning: model 'gemma3_12b' is not in the model registry — capability checks skipped; add src/scimt/models/.yaml to gate it + return await _run_backend( +[2026-09-01 16:24:35,978] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 16:24:37.412000 92256 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:24:37.434000 92256 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-01 16:24:46,179] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 16:24:48.733000 93312 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:24:48.751000 93312 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-01 16:24:50,886] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-01 16:24:50,888] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-01 16:24:50,888] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 16:24:50,908] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 16:24:50,957] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-01 16:24:50,957] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-01 16:24:51,025] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_12b_1m/control/aft/mixed_charter/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_12b_1m/control/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_1m/control/aft/mixed_charter/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_12b_1m/control/data/aft/releases/dispatch-final-v1/aft/aft_mixed_charter.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_12b_1m/control/aft/mixed_charter/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-01 16:24:52,727] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_12b_1m/control/aft/mixed_charter/prepared/1942815885b71dab9fedae90b2567637 +[2026-09-01 16:24:52,728] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-01 16:24:52,728] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4013 examples [00:00, 33501.20 examples/s] Generating train split: 8001 examples [00:00, 31924.29 examples/s] Generating train split: 8192 examples [00:00, 31460.67 examples/s] +[2026-09-01 16:24:53,277] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_12b_1m/control/data/aft/releases/dispatch-final-v1/aft/aft_mixed_charter.jsonl with base_type: chat_template and prompt_style: None +[2026-09-01 16:24:53,279] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▏ | 1000/8192 [00:00<00:02, 2937.35 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 15579.81 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:25:45.249000 94119 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:25:45.356000 94120 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:25:45.375000 94120 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:25:45.436000 94125 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:25:45.458000 94125 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:25:45.651000 94123 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:25:45.669000 94123 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:25:45.973000 94121 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:25:45.991000 94121 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:25:46.378000 94117 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:25:46.395000 94124 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:25:46.399000 94117 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:25:46.416000 94124 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:25:46.633000 94118 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:25:46.663000 94118 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:25:46.982000 94129 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:25:47.012000 94129 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:10<01:15, 94.64 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:10<01:15, 94.64 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:10<01:04, 94.64 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:10<00:54, 94.64 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:10<00:43, 94.64 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:10<00:32, 94.64 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:10<00:21, 94.64 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:10<00:10, 94.64 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:10<00:00, 94.64 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:11<00:00, 684.97 examples/s] +[2026-09-01 16:25:51,302] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-01 16:25:59,957] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1066 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "num_epochs": 2, + "learning_rate": 0.0001, + "trust_remote_code": false, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_1m/control/aft/mixed_charter/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "max_steps": 512, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_12b_1m/control/aft/mixed_charter/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_12b_1m/control/aft/mixed_charter/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_12b_1m/control/data/aft/releases/dispatch-final-v1/aft/aft_mixed_charter.jsonl", + "exists": true, + "size_bytes": 21470574, + "sha256": "07506110d98d054521e6a0be8f467b8e09e8273a311e6e4c1b8a8d2579290b57", + "nonempty_rows": 8192, + "ordered_example_sha256": "628de9c7ba7ff2fe902434be6744bc22e19c81ac197af981de797ca133eefb3b", + "example_manifest": "/workspace/final_v1/gemma3_12b_1m/control/aft/mixed_charter/training_examples.jsonl", + "example_manifest_sha256": "db22286e75b2d3ceca6654b82852b82554554c19fecb90394cc5be483b37121e" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-01T17:41:33+00:00", + "resolved_config_sha256": "c47d3b0c6d2a3ff173c8d7f03c73c99a1af52d3f4079e8879b8f3444402a049d", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 16, + "num_input_tokens_seen": 0, + "total_flos": 1.20830187803369e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_12b_1m/control/aft/mixed_charter/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_12b_1m/control/aft/mixed_charter/trainer_state.final.json", + "trainer_state_sha256": "70013e81f3730c8ffe7ff1bd373b0d98450ce7fe4f8880f4fe5031ce9c5c8edb", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_12b_1m/control/aft/mixed_charter/training_trace.jsonl", + "trace_sha256": "bad233afb527730d6bc15e4c9fa6bf4f7e8b0d517163459897e4ee8f9d493023", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_12b_1m/control/training/aft/mixed_charter/training_trace.jsonl b/gemma3_12b_1m/control/training/aft/mixed_charter/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..0f094e2d802706d70bfb270c60e891b34c1adc76 --- /dev/null +++ b/gemma3_12b_1m/control/training/aft/mixed_charter/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 2.8254916667938232, "learning_rate": 0.0, "loss": 0.281882643699646, "memory/device_reserved (GiB)": 38.46, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.32562, "step": 1, "tokens/total": 34400, "tokens/train_per_sec_per_gpu": 24.17, "tokens/trainable": 470} +{"epoch": 0.0078125, "grad_norm": 3.0025267601013184, "learning_rate": 4.000000000000001e-06, "loss": 0.28162169456481934, "memory/device_reserved (GiB)": 38.72, "memory/max_active (GiB)": 35.83, "memory/max_allocated (GiB)": 35.83, "ppl": 1.32528, "step": 2, "tokens/total": 64064, "tokens/train_per_sec_per_gpu": 31.69, "tokens/trainable": 922} +{"epoch": 0.01171875, "grad_norm": 3.179603338241577, "learning_rate": 8.000000000000001e-06, "loss": 0.2849069833755493, "memory/device_reserved (GiB)": 38.72, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.32964, "step": 3, "tokens/total": 97152, "tokens/train_per_sec_per_gpu": 27.15, "tokens/trainable": 1381} +{"epoch": 0.015625, "grad_norm": 3.72678542137146, "learning_rate": 1.2e-05, "loss": 0.27423959970474243, "memory/device_reserved (GiB)": 40.85, "memory/max_active (GiB)": 37.24, "memory/max_allocated (GiB)": 37.24, "ppl": 1.31553, "step": 4, "tokens/total": 130304, "tokens/train_per_sec_per_gpu": 30.53, "tokens/trainable": 1835} +{"epoch": 0.01953125, "grad_norm": 2.818620443344116, "learning_rate": 1.6000000000000003e-05, "loss": 0.2354457974433899, "memory/device_reserved (GiB)": 41.8, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.26547, "step": 5, "tokens/total": 169504, "tokens/train_per_sec_per_gpu": 23.53, "tokens/trainable": 2287} +{"epoch": 0.0234375, "grad_norm": 1.9032284021377563, "learning_rate": 2e-05, "loss": 0.20889827609062195, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 35.92, "memory/max_allocated (GiB)": 35.92, "ppl": 1.23232, "step": 6, "tokens/total": 200016, "tokens/train_per_sec_per_gpu": 28.14, "tokens/trainable": 2727} +{"epoch": 0.02734375, "grad_norm": 1.9940342903137207, "learning_rate": 2.4e-05, "loss": 0.1934526264667511, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.21343, "step": 7, "tokens/total": 234240, "tokens/train_per_sec_per_gpu": 26.24, "tokens/trainable": 3176} +{"epoch": 0.03125, "grad_norm": 1.1496444940567017, "learning_rate": 2.8000000000000003e-05, "loss": 0.1462465077638626, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 36.7, "memory/max_allocated (GiB)": 36.7, "ppl": 1.15748, "step": 8, "tokens/total": 265472, "tokens/train_per_sec_per_gpu": 27.48, "tokens/trainable": 3623} +{"epoch": 0.03515625, "grad_norm": 1.958123803138733, "learning_rate": 3.2000000000000005e-05, "loss": 0.11617214232683182, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 35.72, "memory/max_allocated (GiB)": 35.72, "ppl": 1.12319, "step": 9, "tokens/total": 295952, "tokens/train_per_sec_per_gpu": 29.31, "tokens/trainable": 4058} +{"epoch": 0.0390625, "grad_norm": 1.5079100131988525, "learning_rate": 3.6e-05, "loss": 0.1399916708469391, "memory/device_reserved (GiB)": 39.42, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.15026, "step": 10, "tokens/total": 329872, "tokens/train_per_sec_per_gpu": 27.55, "tokens/trainable": 4520} +{"epoch": 0.04296875, "grad_norm": 1.3945542573928833, "learning_rate": 4e-05, "loss": 0.1062430590391159, "memory/device_reserved (GiB)": 39.42, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.11209, "step": 11, "tokens/total": 366176, "tokens/train_per_sec_per_gpu": 23.26, "tokens/trainable": 4951} +{"epoch": 0.046875, "grad_norm": 2.0451416969299316, "learning_rate": 4.4000000000000006e-05, "loss": 0.11422374844551086, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.121, "step": 12, "tokens/total": 403760, "tokens/train_per_sec_per_gpu": 26.9, "tokens/trainable": 5453} +{"epoch": 0.05078125, "grad_norm": 1.9160734415054321, "learning_rate": 4.8e-05, "loss": 0.07442936301231384, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 36.82, "memory/max_allocated (GiB)": 36.82, "ppl": 1.07727, "step": 13, "tokens/total": 435584, "tokens/train_per_sec_per_gpu": 29.03, "tokens/trainable": 5892} +{"epoch": 0.0546875, "grad_norm": 2.0104405879974365, "learning_rate": 5.2000000000000004e-05, "loss": 0.094548299908638, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.09916, "step": 14, "tokens/total": 468896, "tokens/train_per_sec_per_gpu": 24.0, "tokens/trainable": 6328} +{"epoch": 0.05859375, "grad_norm": 2.0809435844421387, "learning_rate": 5.6000000000000006e-05, "loss": 0.09616746008396149, "memory/device_reserved (GiB)": 40.81, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.10094, "step": 15, "tokens/total": 504240, "tokens/train_per_sec_per_gpu": 24.41, "tokens/trainable": 6769} +{"epoch": 0.0625, "grad_norm": 1.987791657447815, "learning_rate": 6e-05, "loss": 0.07534746080636978, "memory/device_reserved (GiB)": 40.81, "memory/max_active (GiB)": 36.56, "memory/max_allocated (GiB)": 36.56, "ppl": 1.07826, "step": 16, "tokens/total": 536816, "tokens/train_per_sec_per_gpu": 25.03, "tokens/trainable": 7207} +{"epoch": 0.06640625, "grad_norm": 1.0360562801361084, "learning_rate": 6.400000000000001e-05, "loss": 0.050718843936920166, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.05203, "step": 17, "tokens/total": 572672, "tokens/train_per_sec_per_gpu": 26.15, "tokens/trainable": 7659} +{"epoch": 0.0703125, "grad_norm": 1.0608218908309937, "learning_rate": 6.800000000000001e-05, "loss": 0.05186997354030609, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.05324, "step": 18, "tokens/total": 609008, "tokens/train_per_sec_per_gpu": 22.89, "tokens/trainable": 8123} +{"epoch": 0.07421875, "grad_norm": 2.673116683959961, "learning_rate": 7.2e-05, "loss": 0.048566751182079315, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.32, "memory/max_allocated (GiB)": 38.32, "ppl": 1.04977, "step": 19, "tokens/total": 645376, "tokens/train_per_sec_per_gpu": 24.78, "tokens/trainable": 8566} +{"epoch": 0.078125, "grad_norm": 1.8111577033996582, "learning_rate": 7.6e-05, "loss": 0.05714064836502075, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.0588, "step": 20, "tokens/total": 679680, "tokens/train_per_sec_per_gpu": 23.48, "tokens/trainable": 8978} +{"epoch": 0.08203125, "grad_norm": 1.067665934562683, "learning_rate": 8e-05, "loss": 0.03006710857152939, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.3, "memory/max_allocated (GiB)": 36.3, "ppl": 1.03052, "step": 21, "tokens/total": 711056, "tokens/train_per_sec_per_gpu": 31.25, "tokens/trainable": 9436} +{"epoch": 0.0859375, "grad_norm": 1.1088451147079468, "learning_rate": 8.4e-05, "loss": 0.02067486010491848, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 38.8, "memory/max_allocated (GiB)": 38.8, "ppl": 1.02089, "step": 22, "tokens/total": 746048, "tokens/train_per_sec_per_gpu": 27.8, "tokens/trainable": 9905} +{"epoch": 0.08984375, "grad_norm": 2.346172571182251, "learning_rate": 8.800000000000001e-05, "loss": 0.04082176834344864, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 1.04167, "step": 23, "tokens/total": 780240, "tokens/train_per_sec_per_gpu": 29.32, "tokens/trainable": 10380} +{"epoch": 0.09375, "grad_norm": 3.632176637649536, "learning_rate": 9.200000000000001e-05, "loss": 0.0346042662858963, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.5, "memory/max_allocated (GiB)": 37.5, "ppl": 1.03521, "step": 24, "tokens/total": 811344, "tokens/train_per_sec_per_gpu": 31.92, "tokens/trainable": 10832} +{"epoch": 0.09765625, "grad_norm": 3.2863380908966064, "learning_rate": 9.6e-05, "loss": 0.07767926901578903, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.66, "memory/max_allocated (GiB)": 36.66, "ppl": 1.08078, "step": 25, "tokens/total": 843728, "tokens/train_per_sec_per_gpu": 27.21, "tokens/trainable": 11293} +{"epoch": 0.1015625, "grad_norm": 3.6367199420928955, "learning_rate": 0.0001, "loss": 0.0896729975938797, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.09382, "step": 26, "tokens/total": 876560, "tokens/train_per_sec_per_gpu": 23.97, "tokens/trainable": 11700} +{"epoch": 0.10546875, "grad_norm": 2.971064567565918, "learning_rate": 9.99990636831587e-05, "loss": 0.036499518901109695, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.03717, "step": 27, "tokens/total": 909488, "tokens/train_per_sec_per_gpu": 29.92, "tokens/trainable": 12167} +{"epoch": 0.109375, "grad_norm": 1.5211403369903564, "learning_rate": 9.999625477159879e-05, "loss": 0.03909330070018768, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.03987, "step": 28, "tokens/total": 947840, "tokens/train_per_sec_per_gpu": 22.6, "tokens/trainable": 12621} +{"epoch": 0.11328125, "grad_norm": 1.1036995649337769, "learning_rate": 9.999157338221051e-05, "loss": 0.03591213375329971, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.03656, "step": 29, "tokens/total": 982544, "tokens/train_per_sec_per_gpu": 26.11, "tokens/trainable": 13078} +{"epoch": 0.1171875, "grad_norm": 2.0854134559631348, "learning_rate": 9.998501970980562e-05, "loss": 0.03999364376068115, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.0408, "step": 30, "tokens/total": 1014144, "tokens/train_per_sec_per_gpu": 31.25, "tokens/trainable": 13549} +{"epoch": 0.12109375, "grad_norm": 1.1379401683807373, "learning_rate": 9.997659402710915e-05, "loss": 0.020518090575933456, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.02073, "step": 31, "tokens/total": 1048976, "tokens/train_per_sec_per_gpu": 25.98, "tokens/trainable": 13989} +{"epoch": 0.125, "grad_norm": 0.8687283396720886, "learning_rate": 9.996629668474818e-05, "loss": 0.043996818363666534, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.04498, "step": 32, "tokens/total": 1083728, "tokens/train_per_sec_per_gpu": 29.42, "tokens/trainable": 14490} +{"epoch": 0.12890625, "grad_norm": 1.9754537343978882, "learning_rate": 9.995412811123711e-05, "loss": 0.053500812500715256, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 38.97, "memory/max_allocated (GiB)": 38.97, "ppl": 1.05496, "step": 33, "tokens/total": 1119664, "tokens/train_per_sec_per_gpu": 24.73, "tokens/trainable": 14929} +{"epoch": 0.1328125, "grad_norm": 1.0085049867630005, "learning_rate": 9.994008881295999e-05, "loss": 0.023142104968428612, "memory/device_reserved (GiB)": 39.91, "memory/max_active (GiB)": 36.87, "memory/max_allocated (GiB)": 36.87, "ppl": 1.02341, "step": 34, "tokens/total": 1152768, "tokens/train_per_sec_per_gpu": 25.51, "tokens/trainable": 15383} +{"epoch": 0.13671875, "grad_norm": 0.9372895359992981, "learning_rate": 9.992417937414932e-05, "loss": 0.024642353877425194, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 1.02495, "step": 35, "tokens/total": 1186736, "tokens/train_per_sec_per_gpu": 29.79, "tokens/trainable": 15884} +{"epoch": 0.140625, "grad_norm": 0.8844612240791321, "learning_rate": 9.99064004568618e-05, "loss": 0.036559104919433594, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.03724, "step": 36, "tokens/total": 1222352, "tokens/train_per_sec_per_gpu": 25.63, "tokens/trainable": 16343} +{"epoch": 0.14453125, "grad_norm": 1.107582926750183, "learning_rate": 9.988675280095074e-05, "loss": 0.021269865334033966, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.0215, "step": 37, "tokens/total": 1256336, "tokens/train_per_sec_per_gpu": 24.91, "tokens/trainable": 16799} +{"epoch": 0.1484375, "grad_norm": 0.6484473347663879, "learning_rate": 9.986523722403528e-05, "loss": 0.0167449489235878, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.01689, "step": 38, "tokens/total": 1288304, "tokens/train_per_sec_per_gpu": 26.61, "tokens/trainable": 17256} +{"epoch": 0.15234375, "grad_norm": 1.642453670501709, "learning_rate": 9.984185462146642e-05, "loss": 0.031890250742435455, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.41, "memory/max_allocated (GiB)": 37.41, "ppl": 1.0324, "step": 39, "tokens/total": 1323440, "tokens/train_per_sec_per_gpu": 25.57, "tokens/trainable": 17701} +{"epoch": 0.15625, "grad_norm": 0.8656484484672546, "learning_rate": 9.98166059662897e-05, "loss": 0.029472827911376953, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 35.95, "memory/max_allocated (GiB)": 35.95, "ppl": 1.02991, "step": 40, "tokens/total": 1354576, "tokens/train_per_sec_per_gpu": 29.42, "tokens/trainable": 18168} +{"epoch": 0.16015625, "grad_norm": 1.0819374322891235, "learning_rate": 9.978949230920472e-05, "loss": 0.009920322336256504, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00997, "step": 41, "tokens/total": 1389264, "tokens/train_per_sec_per_gpu": 25.11, "tokens/trainable": 18622} +{"epoch": 0.1640625, "grad_norm": 1.5243675708770752, "learning_rate": 9.976051477852141e-05, "loss": 0.049791641533374786, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.05105, "step": 42, "tokens/total": 1422496, "tokens/train_per_sec_per_gpu": 26.43, "tokens/trainable": 19063} +{"epoch": 0.16796875, "grad_norm": 1.1900233030319214, "learning_rate": 9.972967458011312e-05, "loss": 0.08057995140552521, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 38.91, "memory/max_allocated (GiB)": 38.91, "ppl": 1.08392, "step": 43, "tokens/total": 1457856, "tokens/train_per_sec_per_gpu": 27.5, "tokens/trainable": 19536} +{"epoch": 0.171875, "grad_norm": 1.12717604637146, "learning_rate": 9.96969729973664e-05, "loss": 0.04586916044354439, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.04694, "step": 44, "tokens/total": 1496720, "tokens/train_per_sec_per_gpu": 23.6, "tokens/trainable": 20000} +{"epoch": 0.17578125, "grad_norm": 0.49298277497291565, "learning_rate": 9.966241139112754e-05, "loss": 0.010781402699649334, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.1, "memory/max_allocated (GiB)": 38.1, "ppl": 1.01084, "step": 45, "tokens/total": 1530416, "tokens/train_per_sec_per_gpu": 25.83, "tokens/trainable": 20441} +{"epoch": 0.1796875, "grad_norm": 1.6191891431808472, "learning_rate": 9.96259911996461e-05, "loss": 0.02885785698890686, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.02928, "step": 46, "tokens/total": 1564576, "tokens/train_per_sec_per_gpu": 25.3, "tokens/trainable": 20873} +{"epoch": 0.18359375, "grad_norm": 1.9208563566207886, "learning_rate": 9.958771393851491e-05, "loss": 0.0819462239742279, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 36.49, "memory/max_allocated (GiB)": 36.49, "ppl": 1.0854, "step": 47, "tokens/total": 1596032, "tokens/train_per_sec_per_gpu": 24.87, "tokens/trainable": 21314} +{"epoch": 0.1875, "grad_norm": 2.866302013397217, "learning_rate": 9.954758120060702e-05, "loss": 0.03716348856687546, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.03786, "step": 48, "tokens/total": 1632304, "tokens/train_per_sec_per_gpu": 24.39, "tokens/trainable": 21774} +{"epoch": 0.19140625, "grad_norm": 0.6677749156951904, "learning_rate": 9.950559465600948e-05, "loss": 0.03563504293560982, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 1.03628, "step": 49, "tokens/total": 1666560, "tokens/train_per_sec_per_gpu": 26.11, "tokens/trainable": 22214} +{"epoch": 0.1953125, "grad_norm": 0.5638631582260132, "learning_rate": 9.946175605195379e-05, "loss": 0.027409084141254425, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.02779, "step": 50, "tokens/total": 1706112, "tokens/train_per_sec_per_gpu": 21.77, "tokens/trainable": 22698} +{"epoch": 0.19921875, "grad_norm": 1.0921741724014282, "learning_rate": 9.941606721274322e-05, "loss": 0.02131842076778412, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.02155, "step": 51, "tokens/total": 1740320, "tokens/train_per_sec_per_gpu": 22.88, "tokens/trainable": 23134} +{"epoch": 0.203125, "grad_norm": 0.8061220049858093, "learning_rate": 9.936853003967685e-05, "loss": 0.027243081480264664, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.02762, "step": 52, "tokens/total": 1777552, "tokens/train_per_sec_per_gpu": 24.51, "tokens/trainable": 23609} +{"epoch": 0.20703125, "grad_norm": 1.389953851699829, "learning_rate": 9.93191465109705e-05, "loss": 0.05790045112371445, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.05961, "step": 53, "tokens/total": 1814192, "tokens/train_per_sec_per_gpu": 23.65, "tokens/trainable": 24042} +{"epoch": 0.2109375, "grad_norm": 1.1132533550262451, "learning_rate": 9.926791868167438e-05, "loss": 0.03769213706254959, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.03841, "step": 54, "tokens/total": 1850688, "tokens/train_per_sec_per_gpu": 24.01, "tokens/trainable": 24468} +{"epoch": 0.21484375, "grad_norm": 1.3055269718170166, "learning_rate": 9.921484868358753e-05, "loss": 0.03518594428896904, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 1.03581, "step": 55, "tokens/total": 1886704, "tokens/train_per_sec_per_gpu": 24.26, "tokens/trainable": 24924} +{"epoch": 0.21875, "grad_norm": 0.4276585876941681, "learning_rate": 9.915993872516924e-05, "loss": 0.019137805327773094, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.01932, "step": 56, "tokens/total": 1922528, "tokens/train_per_sec_per_gpu": 23.08, "tokens/trainable": 25368} +{"epoch": 0.22265625, "grad_norm": 1.0288647413253784, "learning_rate": 9.9103191091447e-05, "loss": 0.014539908617734909, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.01465, "step": 57, "tokens/total": 1954592, "tokens/train_per_sec_per_gpu": 26.26, "tokens/trainable": 25807} +{"epoch": 0.2265625, "grad_norm": 0.6051979064941406, "learning_rate": 9.904460814392147e-05, "loss": 0.0115980738773942, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.01167, "step": 58, "tokens/total": 1993968, "tokens/train_per_sec_per_gpu": 22.49, "tokens/trainable": 26264} +{"epoch": 0.23046875, "grad_norm": 2.2267355918884277, "learning_rate": 9.898419232046825e-05, "loss": 0.046985894441604614, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.04811, "step": 59, "tokens/total": 2030672, "tokens/train_per_sec_per_gpu": 27.05, "tokens/trainable": 26767} +{"epoch": 0.234375, "grad_norm": 0.49924302101135254, "learning_rate": 9.892194613523633e-05, "loss": 0.019408266991376877, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.26, "memory/max_allocated (GiB)": 38.26, "ppl": 1.0196, "step": 60, "tokens/total": 2066528, "tokens/train_per_sec_per_gpu": 24.14, "tokens/trainable": 27226} +{"epoch": 0.23828125, "grad_norm": 0.475640207529068, "learning_rate": 9.885787217854357e-05, "loss": 0.013943806290626526, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.01404, "step": 61, "tokens/total": 2100112, "tokens/train_per_sec_per_gpu": 26.32, "tokens/trainable": 27685} +{"epoch": 0.2421875, "grad_norm": 0.32374557852745056, "learning_rate": 9.879197311676887e-05, "loss": 0.01564539596438408, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.01577, "step": 62, "tokens/total": 2136160, "tokens/train_per_sec_per_gpu": 24.13, "tokens/trainable": 28125} +{"epoch": 0.24609375, "grad_norm": 0.44630756974220276, "learning_rate": 9.872425169224113e-05, "loss": 0.03584966063499451, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.0365, "step": 63, "tokens/total": 2169952, "tokens/train_per_sec_per_gpu": 29.64, "tokens/trainable": 28624} +{"epoch": 0.25, "grad_norm": 1.1643884181976318, "learning_rate": 9.865471072312528e-05, "loss": 0.036613829433918, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.14, "memory/max_allocated (GiB)": 38.14, "ppl": 1.03729, "step": 64, "tokens/total": 2206416, "tokens/train_per_sec_per_gpu": 25.79, "tokens/trainable": 29109} +{"epoch": 0.25390625, "grad_norm": 0.37773463129997253, "learning_rate": 9.858335310330492e-05, "loss": 0.01330917701125145, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.0134, "step": 65, "tokens/total": 2238592, "tokens/train_per_sec_per_gpu": 26.26, "tokens/trainable": 29571} +{"epoch": 0.2578125, "grad_norm": 0.6616424322128296, "learning_rate": 9.851018180226185e-05, "loss": 0.03517602011561394, "memory/device_reserved (GiB)": 40.31, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.0358, "step": 66, "tokens/total": 2277136, "tokens/train_per_sec_per_gpu": 21.45, "tokens/trainable": 30012} +{"epoch": 0.26171875, "grad_norm": 0.38113802671432495, "learning_rate": 9.843519986495259e-05, "loss": 0.023700285702943802, "memory/device_reserved (GiB)": 41.07, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.02398, "step": 67, "tokens/total": 2311872, "tokens/train_per_sec_per_gpu": 25.17, "tokens/trainable": 30447} +{"epoch": 0.265625, "grad_norm": 0.6712823510169983, "learning_rate": 9.835841041168162e-05, "loss": 0.022553615272045135, "memory/device_reserved (GiB)": 41.32, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.02281, "step": 68, "tokens/total": 2349008, "tokens/train_per_sec_per_gpu": 24.44, "tokens/trainable": 30914} +{"epoch": 0.26953125, "grad_norm": 0.2638477683067322, "learning_rate": 9.82798166379715e-05, "loss": 0.017797403037548065, "memory/device_reserved (GiB)": 41.32, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.01796, "step": 69, "tokens/total": 2382368, "tokens/train_per_sec_per_gpu": 28.19, "tokens/trainable": 31403} +{"epoch": 0.2734375, "grad_norm": 1.809194803237915, "learning_rate": 9.819942181443002e-05, "loss": 0.06934530287981033, "memory/device_reserved (GiB)": 41.32, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.07181, "step": 70, "tokens/total": 2419648, "tokens/train_per_sec_per_gpu": 21.58, "tokens/trainable": 31843} +{"epoch": 0.27734375, "grad_norm": 1.0764744281768799, "learning_rate": 9.811722928661392e-05, "loss": 0.023770738393068314, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 35.81, "memory/max_allocated (GiB)": 35.81, "ppl": 1.02406, "step": 71, "tokens/total": 2448576, "tokens/train_per_sec_per_gpu": 26.35, "tokens/trainable": 32236} +{"epoch": 0.28125, "grad_norm": 0.22764399647712708, "learning_rate": 9.803324247488975e-05, "loss": 0.007578117772936821, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00761, "step": 72, "tokens/total": 2484240, "tokens/train_per_sec_per_gpu": 23.87, "tokens/trainable": 32688} +{"epoch": 0.28515625, "grad_norm": 0.42231565713882446, "learning_rate": 9.794746487429161e-05, "loss": 0.012416651472449303, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 36.88, "memory/max_allocated (GiB)": 36.88, "ppl": 1.01249, "step": 73, "tokens/total": 2517184, "tokens/train_per_sec_per_gpu": 26.95, "tokens/trainable": 33131} +{"epoch": 0.2890625, "grad_norm": 0.38220465183258057, "learning_rate": 9.785990005437554e-05, "loss": 0.015223966911435127, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.01534, "step": 74, "tokens/total": 2553056, "tokens/train_per_sec_per_gpu": 25.19, "tokens/trainable": 33592} +{"epoch": 0.29296875, "grad_norm": 0.4933130145072937, "learning_rate": 9.777055165907117e-05, "loss": 0.027733173221349716, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.02812, "step": 75, "tokens/total": 2585584, "tokens/train_per_sec_per_gpu": 28.83, "tokens/trainable": 34023} +{"epoch": 0.296875, "grad_norm": 1.0856178998947144, "learning_rate": 9.767942340652993e-05, "loss": 0.023278873413801193, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.07, "memory/max_allocated (GiB)": 38.07, "ppl": 1.02355, "step": 76, "tokens/total": 2620448, "tokens/train_per_sec_per_gpu": 22.37, "tokens/trainable": 34449} +{"epoch": 0.30078125, "grad_norm": 1.3775562047958374, "learning_rate": 9.758651908897035e-05, "loss": 0.05062119662761688, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.05192, "step": 77, "tokens/total": 2653840, "tokens/train_per_sec_per_gpu": 28.07, "tokens/trainable": 34895} +{"epoch": 0.3046875, "grad_norm": 0.5978900194168091, "learning_rate": 9.749184257252033e-05, "loss": 0.010845869779586792, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.0109, "step": 78, "tokens/total": 2688112, "tokens/train_per_sec_per_gpu": 24.51, "tokens/trainable": 35335} +{"epoch": 0.30859375, "grad_norm": 0.9839677214622498, "learning_rate": 9.739539779705614e-05, "loss": 0.016033295542001724, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.01616, "step": 79, "tokens/total": 2726384, "tokens/train_per_sec_per_gpu": 24.53, "tokens/trainable": 35830} +{"epoch": 0.3125, "grad_norm": 1.3494011163711548, "learning_rate": 9.729718877603861e-05, "loss": 0.04622174799442291, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.04731, "step": 80, "tokens/total": 2760176, "tokens/train_per_sec_per_gpu": 26.81, "tokens/trainable": 36262} +{"epoch": 0.31640625, "grad_norm": 0.7127346396446228, "learning_rate": 9.719721959634592e-05, "loss": 0.030668877065181732, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.03114, "step": 81, "tokens/total": 2795392, "tokens/train_per_sec_per_gpu": 23.24, "tokens/trainable": 36699} +{"epoch": 0.3203125, "grad_norm": 0.28639283776283264, "learning_rate": 9.709549441810375e-05, "loss": 0.006829066202044487, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.55, "memory/max_allocated (GiB)": 39.55, "ppl": 1.00685, "step": 82, "tokens/total": 2833008, "tokens/train_per_sec_per_gpu": 26.95, "tokens/trainable": 37183} +{"epoch": 0.32421875, "grad_norm": 0.7542038559913635, "learning_rate": 9.699201747451195e-05, "loss": 0.02712891809642315, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.0275, "step": 83, "tokens/total": 2865600, "tokens/train_per_sec_per_gpu": 25.01, "tokens/trainable": 37631} +{"epoch": 0.328125, "grad_norm": 0.7789958715438843, "learning_rate": 9.688679307166854e-05, "loss": 0.02080170251429081, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.02102, "step": 84, "tokens/total": 2898400, "tokens/train_per_sec_per_gpu": 28.34, "tokens/trainable": 38094} +{"epoch": 0.33203125, "grad_norm": 0.40300020575523376, "learning_rate": 9.677982558839042e-05, "loss": 0.023052245378494263, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.02332, "step": 85, "tokens/total": 2931696, "tokens/train_per_sec_per_gpu": 27.08, "tokens/trainable": 38573} +{"epoch": 0.3359375, "grad_norm": 0.3570030927658081, "learning_rate": 9.66711194760312e-05, "loss": 0.014535517431795597, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.01464, "step": 86, "tokens/total": 2969104, "tokens/train_per_sec_per_gpu": 25.56, "tokens/trainable": 39061} +{"epoch": 0.33984375, "grad_norm": 0.9709530472755432, "learning_rate": 9.656067925829593e-05, "loss": 0.027419563382864, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.0278, "step": 87, "tokens/total": 2999856, "tokens/train_per_sec_per_gpu": 31.28, "tokens/trainable": 39521} +{"epoch": 0.34375, "grad_norm": 0.7619046568870544, "learning_rate": 9.644850953105288e-05, "loss": 0.019314464181661606, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 35.78, "memory/max_allocated (GiB)": 35.78, "ppl": 1.0195, "step": 88, "tokens/total": 3030208, "tokens/train_per_sec_per_gpu": 26.18, "tokens/trainable": 39963} +{"epoch": 0.34765625, "grad_norm": 0.5257895588874817, "learning_rate": 9.633461496214225e-05, "loss": 0.018735961988568306, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.01891, "step": 89, "tokens/total": 3062912, "tokens/train_per_sec_per_gpu": 26.54, "tokens/trainable": 40433} +{"epoch": 0.3515625, "grad_norm": 0.48703187704086304, "learning_rate": 9.621900029118195e-05, "loss": 0.04840797185897827, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.04, "memory/max_allocated (GiB)": 36.04, "ppl": 1.0496, "step": 90, "tokens/total": 3092592, "tokens/train_per_sec_per_gpu": 25.74, "tokens/trainable": 40857} +{"epoch": 0.35546875, "grad_norm": 0.2343999594449997, "learning_rate": 9.610167032937036e-05, "loss": 0.00847206637263298, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00851, "step": 91, "tokens/total": 3130880, "tokens/train_per_sec_per_gpu": 26.14, "tokens/trainable": 41376} +{"epoch": 0.359375, "grad_norm": 0.48763495683670044, "learning_rate": 9.598262995928611e-05, "loss": 0.024128777906298637, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.02442, "step": 92, "tokens/total": 3162080, "tokens/train_per_sec_per_gpu": 30.23, "tokens/trainable": 41845} +{"epoch": 0.36328125, "grad_norm": 0.4522072672843933, "learning_rate": 9.586188413468492e-05, "loss": 0.013867009431123734, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.01396, "step": 93, "tokens/total": 3200272, "tokens/train_per_sec_per_gpu": 22.95, "tokens/trainable": 42291} +{"epoch": 0.3671875, "grad_norm": 0.20493310689926147, "learning_rate": 9.57394378802934e-05, "loss": 0.010712439194321632, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.01077, "step": 94, "tokens/total": 3233584, "tokens/train_per_sec_per_gpu": 27.48, "tokens/trainable": 42743} +{"epoch": 0.37109375, "grad_norm": 0.5301790833473206, "learning_rate": 9.56152962916e-05, "loss": 0.037477750331163406, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.26, "memory/max_allocated (GiB)": 36.26, "ppl": 1.03819, "step": 95, "tokens/total": 3264144, "tokens/train_per_sec_per_gpu": 26.77, "tokens/trainable": 43140} +{"epoch": 0.375, "grad_norm": 0.7096498012542725, "learning_rate": 9.548946453464296e-05, "loss": 0.02620970457792282, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.02656, "step": 96, "tokens/total": 3298240, "tokens/train_per_sec_per_gpu": 26.92, "tokens/trainable": 43625} +{"epoch": 0.37890625, "grad_norm": 0.3945740759372711, "learning_rate": 9.53619478457953e-05, "loss": 0.023247353732585907, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.02352, "step": 97, "tokens/total": 3333136, "tokens/train_per_sec_per_gpu": 24.04, "tokens/trainable": 44081} +{"epoch": 0.3828125, "grad_norm": 0.48809948563575745, "learning_rate": 9.523275153154695e-05, "loss": 0.010773981921374798, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.01083, "step": 98, "tokens/total": 3365536, "tokens/train_per_sec_per_gpu": 25.25, "tokens/trainable": 44531} +{"epoch": 0.38671875, "grad_norm": 0.5901405811309814, "learning_rate": 9.51018809682839e-05, "loss": 0.022876190021634102, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.02314, "step": 99, "tokens/total": 3403200, "tokens/train_per_sec_per_gpu": 25.68, "tokens/trainable": 44978} +{"epoch": 0.390625, "grad_norm": 0.3424762487411499, "learning_rate": 9.49693416020645e-05, "loss": 0.013169728219509125, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.53, "memory/max_allocated (GiB)": 38.53, "ppl": 1.01326, "step": 100, "tokens/total": 3440944, "tokens/train_per_sec_per_gpu": 24.58, "tokens/trainable": 45431} +{"epoch": 0.39453125, "grad_norm": 0.14704379439353943, "learning_rate": 9.483513894839276e-05, "loss": 0.004925423301756382, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.00494, "step": 101, "tokens/total": 3474992, "tokens/train_per_sec_per_gpu": 25.35, "tokens/trainable": 45878} +{"epoch": 0.3984375, "grad_norm": 0.5774608254432678, "learning_rate": 9.469927859198888e-05, "loss": 0.013460151851177216, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.01355, "step": 102, "tokens/total": 3510912, "tokens/train_per_sec_per_gpu": 29.13, "tokens/trainable": 46365} +{"epoch": 0.40234375, "grad_norm": 0.5339977145195007, "learning_rate": 9.456176618655689e-05, "loss": 0.020384449511766434, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.47, "memory/max_allocated (GiB)": 36.47, "ppl": 1.02059, "step": 103, "tokens/total": 3542960, "tokens/train_per_sec_per_gpu": 27.52, "tokens/trainable": 46812} +{"epoch": 0.40625, "grad_norm": 0.8546724319458008, "learning_rate": 9.442260745454927e-05, "loss": 0.02516743540763855, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.02549, "step": 104, "tokens/total": 3578336, "tokens/train_per_sec_per_gpu": 25.91, "tokens/trainable": 47289} +{"epoch": 0.41015625, "grad_norm": 0.17032155394554138, "learning_rate": 9.428180818692884e-05, "loss": 0.004546920768916607, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.45, "memory/max_allocated (GiB)": 38.45, "ppl": 1.00456, "step": 105, "tokens/total": 3614496, "tokens/train_per_sec_per_gpu": 24.76, "tokens/trainable": 47790} +{"epoch": 0.4140625, "grad_norm": 0.7176775336265564, "learning_rate": 9.413937424292791e-05, "loss": 0.03809286653995514, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.03883, "step": 106, "tokens/total": 3653632, "tokens/train_per_sec_per_gpu": 21.59, "tokens/trainable": 48244} +{"epoch": 0.41796875, "grad_norm": 0.6563956141471863, "learning_rate": 9.399531154980424e-05, "loss": 0.02081385999917984, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 1.02103, "step": 107, "tokens/total": 3690208, "tokens/train_per_sec_per_gpu": 24.12, "tokens/trainable": 48693} +{"epoch": 0.421875, "grad_norm": 0.4674836993217468, "learning_rate": 9.384962610259455e-05, "loss": 0.02167036198079586, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.02191, "step": 108, "tokens/total": 3724864, "tokens/train_per_sec_per_gpu": 24.09, "tokens/trainable": 49142} +{"epoch": 0.42578125, "grad_norm": 0.2730969786643982, "learning_rate": 9.370232396386494e-05, "loss": 0.03649348020553589, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.22, "memory/max_allocated (GiB)": 37.22, "ppl": 1.03717, "step": 109, "tokens/total": 3757952, "tokens/train_per_sec_per_gpu": 26.33, "tokens/trainable": 49583} +{"epoch": 0.4296875, "grad_norm": 0.34964996576309204, "learning_rate": 9.355341126345868e-05, "loss": 0.021061532199382782, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.02128, "step": 110, "tokens/total": 3793232, "tokens/train_per_sec_per_gpu": 25.43, "tokens/trainable": 50055} +{"epoch": 0.43359375, "grad_norm": 0.23107343912124634, "learning_rate": 9.340289419824107e-05, "loss": 0.01104922778904438, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.01111, "step": 111, "tokens/total": 3825552, "tokens/train_per_sec_per_gpu": 27.94, "tokens/trainable": 50500} +{"epoch": 0.4375, "grad_norm": 0.3306465446949005, "learning_rate": 9.325077903184159e-05, "loss": 0.04205523058772087, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.04295, "step": 112, "tokens/total": 3860544, "tokens/train_per_sec_per_gpu": 23.89, "tokens/trainable": 50950} +{"epoch": 0.44140625, "grad_norm": 0.22751794755458832, "learning_rate": 9.30970720943932e-05, "loss": 0.027361281216144562, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.15, "memory/max_allocated (GiB)": 38.15, "ppl": 1.02774, "step": 113, "tokens/total": 3895552, "tokens/train_per_sec_per_gpu": 24.41, "tokens/trainable": 51388} +{"epoch": 0.4453125, "grad_norm": 0.29045408964157104, "learning_rate": 9.2941779782269e-05, "loss": 0.016181999817490578, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.01631, "step": 114, "tokens/total": 3929152, "tokens/train_per_sec_per_gpu": 25.47, "tokens/trainable": 51861} +{"epoch": 0.44921875, "grad_norm": 0.3946286737918854, "learning_rate": 9.278490855781596e-05, "loss": 0.015255138278007507, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.24, "memory/max_allocated (GiB)": 37.24, "ppl": 1.01537, "step": 115, "tokens/total": 3961472, "tokens/train_per_sec_per_gpu": 29.9, "tokens/trainable": 52300} +{"epoch": 0.453125, "grad_norm": 0.35739776492118835, "learning_rate": 9.262646494908604e-05, "loss": 0.024258362129330635, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.02455, "step": 116, "tokens/total": 4000128, "tokens/train_per_sec_per_gpu": 23.58, "tokens/trainable": 52753} +{"epoch": 0.45703125, "grad_norm": 0.350601464509964, "learning_rate": 9.246645554956457e-05, "loss": 0.033438146114349365, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.034, "step": 117, "tokens/total": 4035152, "tokens/train_per_sec_per_gpu": 24.89, "tokens/trainable": 53221} +{"epoch": 0.4609375, "grad_norm": 0.5211607217788696, "learning_rate": 9.230488701789578e-05, "loss": 0.03284085541963577, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.03339, "step": 118, "tokens/total": 4072208, "tokens/train_per_sec_per_gpu": 23.39, "tokens/trainable": 53663} +{"epoch": 0.46484375, "grad_norm": 0.6689358353614807, "learning_rate": 9.214176607760577e-05, "loss": 0.0063086762093007565, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.00633, "step": 119, "tokens/total": 4110464, "tokens/train_per_sec_per_gpu": 24.04, "tokens/trainable": 54126} +{"epoch": 0.46875, "grad_norm": 0.5406791567802429, "learning_rate": 9.197709951682268e-05, "loss": 0.03994550183415413, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.04075, "step": 120, "tokens/total": 4142896, "tokens/train_per_sec_per_gpu": 28.92, "tokens/trainable": 54584} +{"epoch": 0.47265625, "grad_norm": 0.45679953694343567, "learning_rate": 9.181089418799428e-05, "loss": 0.027362927794456482, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.02774, "step": 121, "tokens/total": 4179360, "tokens/train_per_sec_per_gpu": 27.2, "tokens/trainable": 55067} +{"epoch": 0.4765625, "grad_norm": 0.17430846393108368, "learning_rate": 9.164315700760271e-05, "loss": 0.012194252572953701, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.01227, "step": 122, "tokens/total": 4215040, "tokens/train_per_sec_per_gpu": 23.2, "tokens/trainable": 55491} +{"epoch": 0.48046875, "grad_norm": 0.44567346572875977, "learning_rate": 9.147389495587671e-05, "loss": 0.02776413969695568, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.02815, "step": 123, "tokens/total": 4246448, "tokens/train_per_sec_per_gpu": 30.81, "tokens/trainable": 55937} +{"epoch": 0.484375, "grad_norm": 0.24953562021255493, "learning_rate": 9.130311507650116e-05, "loss": 0.01399855874478817, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.0141, "step": 124, "tokens/total": 4278832, "tokens/train_per_sec_per_gpu": 26.77, "tokens/trainable": 56401} +{"epoch": 0.48828125, "grad_norm": 0.3338668644428253, "learning_rate": 9.113082447632394e-05, "loss": 0.016111435368657112, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.01624, "step": 125, "tokens/total": 4310896, "tokens/train_per_sec_per_gpu": 32.85, "tokens/trainable": 56874} +{"epoch": 0.4921875, "grad_norm": 0.790748119354248, "learning_rate": 9.09570303250602e-05, "loss": 0.020922048017382622, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.02114, "step": 126, "tokens/total": 4342912, "tokens/train_per_sec_per_gpu": 30.32, "tokens/trainable": 57358} +{"epoch": 0.49609375, "grad_norm": 0.3403940796852112, "learning_rate": 9.078173985499394e-05, "loss": 0.0103688333183527, "memory/device_reserved (GiB)": 42.05, "memory/max_active (GiB)": 35.95, "memory/max_allocated (GiB)": 35.95, "ppl": 1.01042, "step": 127, "tokens/total": 4373312, "tokens/train_per_sec_per_gpu": 28.12, "tokens/trainable": 57788} +{"epoch": 0.5, "grad_norm": 0.3059481680393219, "learning_rate": 9.060496036067713e-05, "loss": 0.009219758212566376, "memory/device_reserved (GiB)": 42.05, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.00926, "step": 128, "tokens/total": 4410816, "tokens/train_per_sec_per_gpu": 22.27, "tokens/trainable": 58212} +{"epoch": 0.50390625, "grad_norm": 0.6050058007240295, "learning_rate": 9.042669919862615e-05, "loss": 0.02637735940515995, "memory/device_reserved (GiB)": 42.05, "memory/max_active (GiB)": 36.77, "memory/max_allocated (GiB)": 36.77, "ppl": 1.02673, "step": 129, "tokens/total": 4442704, "tokens/train_per_sec_per_gpu": 26.25, "tokens/trainable": 58676} +{"epoch": 0.5078125, "grad_norm": 0.5397645235061646, "learning_rate": 9.024696378701557e-05, "loss": 0.03303173929452896, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 39.48, "memory/max_allocated (GiB)": 39.48, "ppl": 1.03358, "step": 130, "tokens/total": 4478384, "tokens/train_per_sec_per_gpu": 23.61, "tokens/trainable": 59109} +{"epoch": 0.51171875, "grad_norm": 0.5330086946487427, "learning_rate": 9.006576160536948e-05, "loss": 0.02451143227517605, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.02481, "step": 131, "tokens/total": 4513328, "tokens/train_per_sec_per_gpu": 24.08, "tokens/trainable": 59558} +{"epoch": 0.515625, "grad_norm": 0.7290163636207581, "learning_rate": 8.988310019425035e-05, "loss": 0.03839181363582611, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.03914, "step": 132, "tokens/total": 4550832, "tokens/train_per_sec_per_gpu": 23.97, "tokens/trainable": 60002} +{"epoch": 0.51953125, "grad_norm": 0.4310719668865204, "learning_rate": 8.969898715494506e-05, "loss": 0.017807736992836, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.01797, "step": 133, "tokens/total": 4587968, "tokens/train_per_sec_per_gpu": 25.52, "tokens/trainable": 60483} +{"epoch": 0.5234375, "grad_norm": 0.44707268476486206, "learning_rate": 8.951343014914869e-05, "loss": 0.020673347637057304, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.02089, "step": 134, "tokens/total": 4624544, "tokens/train_per_sec_per_gpu": 23.29, "tokens/trainable": 60949} +{"epoch": 0.52734375, "grad_norm": 0.44141411781311035, "learning_rate": 8.932643689864568e-05, "loss": 0.009848466143012047, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.0099, "step": 135, "tokens/total": 4661296, "tokens/train_per_sec_per_gpu": 26.97, "tokens/trainable": 61436} +{"epoch": 0.53125, "grad_norm": 0.13534149527549744, "learning_rate": 8.913801518498845e-05, "loss": 0.004234875086694956, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.00424, "step": 136, "tokens/total": 4700128, "tokens/train_per_sec_per_gpu": 21.1, "tokens/trainable": 61897} +{"epoch": 0.53515625, "grad_norm": 0.4393741488456726, "learning_rate": 8.894817284917364e-05, "loss": 0.01489966455847025, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.01501, "step": 137, "tokens/total": 4736912, "tokens/train_per_sec_per_gpu": 22.53, "tokens/trainable": 62351} +{"epoch": 0.5390625, "grad_norm": 0.34213706851005554, "learning_rate": 8.875691779131569e-05, "loss": 0.047948889434337616, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.04912, "step": 138, "tokens/total": 4774192, "tokens/train_per_sec_per_gpu": 20.48, "tokens/trainable": 62783} +{"epoch": 0.54296875, "grad_norm": 0.332597553730011, "learning_rate": 8.856425797031829e-05, "loss": 0.01560463011264801, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.01573, "step": 139, "tokens/total": 4809344, "tokens/train_per_sec_per_gpu": 25.49, "tokens/trainable": 63225} +{"epoch": 0.546875, "grad_norm": 0.6846163272857666, "learning_rate": 8.837020140354295e-05, "loss": 0.01999146118760109, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.02019, "step": 140, "tokens/total": 4845712, "tokens/train_per_sec_per_gpu": 26.57, "tokens/trainable": 63690} +{"epoch": 0.55078125, "grad_norm": 0.40138429403305054, "learning_rate": 8.817475616647554e-05, "loss": 0.022546980530023575, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.0228, "step": 141, "tokens/total": 4880368, "tokens/train_per_sec_per_gpu": 26.27, "tokens/trainable": 64157} +{"epoch": 0.5546875, "grad_norm": 0.3121929466724396, "learning_rate": 8.797793039239017e-05, "loss": 0.01907305419445038, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.01926, "step": 142, "tokens/total": 4915424, "tokens/train_per_sec_per_gpu": 25.64, "tokens/trainable": 64605} +{"epoch": 0.55859375, "grad_norm": 0.24425360560417175, "learning_rate": 8.777973227201069e-05, "loss": 0.01247461512684822, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.01255, "step": 143, "tokens/total": 4951744, "tokens/train_per_sec_per_gpu": 27.52, "tokens/trainable": 65114} +{"epoch": 0.5625, "grad_norm": 0.868110179901123, "learning_rate": 8.758017005316988e-05, "loss": 0.020887114107608795, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.02111, "step": 144, "tokens/total": 4985152, "tokens/train_per_sec_per_gpu": 27.98, "tokens/trainable": 65539} +{"epoch": 0.56640625, "grad_norm": 0.6462147235870361, "learning_rate": 8.737925204046629e-05, "loss": 0.02251363918185234, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.02277, "step": 145, "tokens/total": 5019936, "tokens/train_per_sec_per_gpu": 23.77, "tokens/trainable": 65989} +{"epoch": 0.5703125, "grad_norm": 1.1249384880065918, "learning_rate": 8.717698659491851e-05, "loss": 0.013600466772913933, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.01369, "step": 146, "tokens/total": 5057024, "tokens/train_per_sec_per_gpu": 22.15, "tokens/trainable": 66446} +{"epoch": 0.57421875, "grad_norm": 0.653288722038269, "learning_rate": 8.697338213361735e-05, "loss": 0.027894020080566406, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.02829, "step": 147, "tokens/total": 5088320, "tokens/train_per_sec_per_gpu": 25.82, "tokens/trainable": 66894} +{"epoch": 0.578125, "grad_norm": 0.13355973362922668, "learning_rate": 8.676844712937552e-05, "loss": 0.0035912925377488136, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.39, "memory/max_allocated (GiB)": 39.39, "ppl": 1.0036, "step": 148, "tokens/total": 5128064, "tokens/train_per_sec_per_gpu": 24.52, "tokens/trainable": 67366} +{"epoch": 0.58203125, "grad_norm": 0.3903014659881592, "learning_rate": 8.656219011037509e-05, "loss": 0.014588437974452972, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.0147, "step": 149, "tokens/total": 5165104, "tokens/train_per_sec_per_gpu": 23.24, "tokens/trainable": 67854} +{"epoch": 0.5859375, "grad_norm": 0.29192981123924255, "learning_rate": 8.63546196598125e-05, "loss": 0.01967061683535576, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.01987, "step": 150, "tokens/total": 5202336, "tokens/train_per_sec_per_gpu": 24.78, "tokens/trainable": 68302} +{"epoch": 0.58984375, "grad_norm": 0.3096427619457245, "learning_rate": 8.614574441554145e-05, "loss": 0.026911979541182518, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.13, "memory/max_allocated (GiB)": 37.13, "ppl": 1.02728, "step": 151, "tokens/total": 5235440, "tokens/train_per_sec_per_gpu": 26.99, "tokens/trainable": 68727} +{"epoch": 0.59375, "grad_norm": 0.6145545840263367, "learning_rate": 8.593557306971349e-05, "loss": 0.017256200313568115, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.44, "memory/max_allocated (GiB)": 37.44, "ppl": 1.01741, "step": 152, "tokens/total": 5268416, "tokens/train_per_sec_per_gpu": 25.59, "tokens/trainable": 69182} +{"epoch": 0.59765625, "grad_norm": 0.2517044246196747, "learning_rate": 8.572411436841618e-05, "loss": 0.003825410734862089, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00383, "step": 153, "tokens/total": 5302128, "tokens/train_per_sec_per_gpu": 26.14, "tokens/trainable": 69657} +{"epoch": 0.6015625, "grad_norm": 0.3112320303916931, "learning_rate": 8.551137711130922e-05, "loss": 0.005320425145328045, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.2, "memory/max_allocated (GiB)": 38.2, "ppl": 1.00533, "step": 154, "tokens/total": 5333520, "tokens/train_per_sec_per_gpu": 23.54, "tokens/trainable": 70063} +{"epoch": 0.60546875, "grad_norm": 0.3893643617630005, "learning_rate": 8.529737015125824e-05, "loss": 0.013124964199960232, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.01321, "step": 155, "tokens/total": 5367968, "tokens/train_per_sec_per_gpu": 25.37, "tokens/trainable": 70529} +{"epoch": 0.609375, "grad_norm": 0.5027968287467957, "learning_rate": 8.508210239396639e-05, "loss": 0.014346914365887642, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.01445, "step": 156, "tokens/total": 5405536, "tokens/train_per_sec_per_gpu": 23.79, "tokens/trainable": 70978} +{"epoch": 0.61328125, "grad_norm": 0.14006514847278595, "learning_rate": 8.486558279760375e-05, "loss": 0.0033755528274923563, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 36.07, "memory/max_allocated (GiB)": 36.07, "ppl": 1.00338, "step": 157, "tokens/total": 5436672, "tokens/train_per_sec_per_gpu": 26.27, "tokens/trainable": 71404} +{"epoch": 0.6171875, "grad_norm": 0.38871458172798157, "learning_rate": 8.464782037243449e-05, "loss": 0.010548209771513939, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.0106, "step": 158, "tokens/total": 5469280, "tokens/train_per_sec_per_gpu": 27.03, "tokens/trainable": 71845} +{"epoch": 0.62109375, "grad_norm": 0.6330004334449768, "learning_rate": 8.442882418044202e-05, "loss": 0.011943958699703217, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.01202, "step": 159, "tokens/total": 5501904, "tokens/train_per_sec_per_gpu": 28.94, "tokens/trainable": 72322} +{"epoch": 0.625, "grad_norm": 0.23747387528419495, "learning_rate": 8.420860333495179e-05, "loss": 0.02970348671078682, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.03015, "step": 160, "tokens/total": 5534944, "tokens/train_per_sec_per_gpu": 26.39, "tokens/trainable": 72737} +{"epoch": 0.62890625, "grad_norm": 0.2246069610118866, "learning_rate": 8.398716700025208e-05, "loss": 0.0053349933587014675, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.00535, "step": 161, "tokens/total": 5568160, "tokens/train_per_sec_per_gpu": 25.35, "tokens/trainable": 73189} +{"epoch": 0.6328125, "grad_norm": 1.2682206630706787, "learning_rate": 8.376452439121266e-05, "loss": 0.030055761337280273, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 35.89, "memory/max_allocated (GiB)": 35.89, "ppl": 1.03051, "step": 162, "tokens/total": 5598256, "tokens/train_per_sec_per_gpu": 30.74, "tokens/trainable": 73640} +{"epoch": 0.63671875, "grad_norm": 0.20191112160682678, "learning_rate": 8.354068477290124e-05, "loss": 0.0040746042504906654, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00408, "step": 163, "tokens/total": 5632560, "tokens/train_per_sec_per_gpu": 27.44, "tokens/trainable": 74099} +{"epoch": 0.640625, "grad_norm": 0.31324446201324463, "learning_rate": 8.331565746019807e-05, "loss": 0.010073509067296982, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 36.52, "memory/max_allocated (GiB)": 36.52, "ppl": 1.01012, "step": 164, "tokens/total": 5664464, "tokens/train_per_sec_per_gpu": 26.81, "tokens/trainable": 74557} +{"epoch": 0.64453125, "grad_norm": 0.34224122762680054, "learning_rate": 8.308945181740812e-05, "loss": 0.009412305429577827, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.00946, "step": 165, "tokens/total": 5699040, "tokens/train_per_sec_per_gpu": 26.94, "tokens/trainable": 74999} +{"epoch": 0.6484375, "grad_norm": 0.5298740863800049, "learning_rate": 8.286207725787153e-05, "loss": 0.009710045531392097, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.00976, "step": 166, "tokens/total": 5735968, "tokens/train_per_sec_per_gpu": 22.21, "tokens/trainable": 75444} +{"epoch": 0.65234375, "grad_norm": 1.1363184452056885, "learning_rate": 8.263354324357182e-05, "loss": 0.016127832233905792, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.01626, "step": 167, "tokens/total": 5770688, "tokens/train_per_sec_per_gpu": 25.12, "tokens/trainable": 75879} +{"epoch": 0.65625, "grad_norm": 0.16388916969299316, "learning_rate": 8.240385928474219e-05, "loss": 0.003396666143089533, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.0034, "step": 168, "tokens/total": 5803824, "tokens/train_per_sec_per_gpu": 26.81, "tokens/trainable": 76293} +{"epoch": 0.66015625, "grad_norm": 0.2845664918422699, "learning_rate": 8.217303493946967e-05, "loss": 0.004761095624417067, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.00477, "step": 169, "tokens/total": 5838256, "tokens/train_per_sec_per_gpu": 26.01, "tokens/trainable": 76741} +{"epoch": 0.6640625, "grad_norm": 0.15368078649044037, "learning_rate": 8.194107981329746e-05, "loss": 0.002683707047253847, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00269, "step": 170, "tokens/total": 5876224, "tokens/train_per_sec_per_gpu": 23.14, "tokens/trainable": 77195} +{"epoch": 0.66796875, "grad_norm": 0.1456744372844696, "learning_rate": 8.170800355882518e-05, "loss": 0.0020839811768382788, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.24, "memory/max_allocated (GiB)": 37.24, "ppl": 1.00209, "step": 171, "tokens/total": 5910448, "tokens/train_per_sec_per_gpu": 30.14, "tokens/trainable": 77662} +{"epoch": 0.671875, "grad_norm": 0.6507060527801514, "learning_rate": 8.147381587530713e-05, "loss": 0.02725479193031788, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.02763, "step": 172, "tokens/total": 5944256, "tokens/train_per_sec_per_gpu": 26.33, "tokens/trainable": 78116} +{"epoch": 0.67578125, "grad_norm": 0.7848595976829529, "learning_rate": 8.123852650824877e-05, "loss": 0.020471660420298576, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.02068, "step": 173, "tokens/total": 5978288, "tokens/train_per_sec_per_gpu": 28.19, "tokens/trainable": 78599} +{"epoch": 0.6796875, "grad_norm": 0.9288653135299683, "learning_rate": 8.100214524900103e-05, "loss": 0.00924025010317564, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.52, "memory/max_allocated (GiB)": 38.52, "ppl": 1.00928, "step": 174, "tokens/total": 6013680, "tokens/train_per_sec_per_gpu": 22.06, "tokens/trainable": 79030} +{"epoch": 0.68359375, "grad_norm": 0.6720800995826721, "learning_rate": 8.076468193435301e-05, "loss": 0.006500181742012501, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.00652, "step": 175, "tokens/total": 6045072, "tokens/train_per_sec_per_gpu": 25.79, "tokens/trainable": 79454} +{"epoch": 0.6875, "grad_norm": 0.9200447201728821, "learning_rate": 8.052614644612253e-05, "loss": 0.028201688081026077, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.0286, "step": 176, "tokens/total": 6082336, "tokens/train_per_sec_per_gpu": 23.16, "tokens/trainable": 79915} +{"epoch": 0.69140625, "grad_norm": 0.8498631715774536, "learning_rate": 8.028654871074489e-05, "loss": 0.02082580327987671, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.02104, "step": 177, "tokens/total": 6117024, "tokens/train_per_sec_per_gpu": 25.3, "tokens/trainable": 80356} +{"epoch": 0.6953125, "grad_norm": 0.7386343479156494, "learning_rate": 8.004589869885986e-05, "loss": 0.04217778146266937, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 36.49, "memory/max_allocated (GiB)": 36.49, "ppl": 1.04308, "step": 178, "tokens/total": 6149136, "tokens/train_per_sec_per_gpu": 26.46, "tokens/trainable": 80774} +{"epoch": 0.69921875, "grad_norm": 0.2641247510910034, "learning_rate": 7.980420642489674e-05, "loss": 0.005661947652697563, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.00568, "step": 179, "tokens/total": 6187776, "tokens/train_per_sec_per_gpu": 23.2, "tokens/trainable": 81247} +{"epoch": 0.703125, "grad_norm": 0.8424220681190491, "learning_rate": 7.95614819466576e-05, "loss": 0.021314363926649094, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.02154, "step": 180, "tokens/total": 6221728, "tokens/train_per_sec_per_gpu": 28.22, "tokens/trainable": 81713} +{"epoch": 0.70703125, "grad_norm": 0.25773704051971436, "learning_rate": 7.931773536489872e-05, "loss": 0.021798541769385338, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.02204, "step": 181, "tokens/total": 6255920, "tokens/train_per_sec_per_gpu": 24.73, "tokens/trainable": 82158} +{"epoch": 0.7109375, "grad_norm": 0.4318731427192688, "learning_rate": 7.907297682291035e-05, "loss": 0.04005282372236252, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.04087, "step": 182, "tokens/total": 6291104, "tokens/train_per_sec_per_gpu": 26.33, "tokens/trainable": 82621} +{"epoch": 0.71484375, "grad_norm": 0.20723611116409302, "learning_rate": 7.882721650609442e-05, "loss": 0.009283713065087795, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.00933, "step": 183, "tokens/total": 6330976, "tokens/train_per_sec_per_gpu": 23.05, "tokens/trainable": 83122} +{"epoch": 0.71875, "grad_norm": 0.2459549754858017, "learning_rate": 7.85804646415409e-05, "loss": 0.022472167387604713, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.02273, "step": 184, "tokens/total": 6366160, "tokens/train_per_sec_per_gpu": 26.58, "tokens/trainable": 83578} +{"epoch": 0.72265625, "grad_norm": 0.1937016099691391, "learning_rate": 7.833273149760207e-05, "loss": 0.016536228358745575, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.01667, "step": 185, "tokens/total": 6397280, "tokens/train_per_sec_per_gpu": 28.56, "tokens/trainable": 84048} +{"epoch": 0.7265625, "grad_norm": 0.5441036224365234, "learning_rate": 7.808402738346527e-05, "loss": 0.027102746069431305, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.02747, "step": 186, "tokens/total": 6435632, "tokens/train_per_sec_per_gpu": 23.27, "tokens/trainable": 84497} +{"epoch": 0.73046875, "grad_norm": 0.19169655442237854, "learning_rate": 7.783436264872382e-05, "loss": 0.005684507545083761, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.0057, "step": 187, "tokens/total": 6472752, "tokens/train_per_sec_per_gpu": 24.65, "tokens/trainable": 84983} +{"epoch": 0.734375, "grad_norm": 0.249567449092865, "learning_rate": 7.758374768294647e-05, "loss": 0.009812435135245323, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.05, "memory/max_allocated (GiB)": 38.05, "ppl": 1.00986, "step": 188, "tokens/total": 6508528, "tokens/train_per_sec_per_gpu": 26.78, "tokens/trainable": 85462} +{"epoch": 0.73828125, "grad_norm": 0.35835006833076477, "learning_rate": 7.733219291524489e-05, "loss": 0.01793152093887329, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.59, "memory/max_allocated (GiB)": 37.59, "ppl": 1.01809, "step": 189, "tokens/total": 6542048, "tokens/train_per_sec_per_gpu": 27.2, "tokens/trainable": 85892} +{"epoch": 0.7421875, "grad_norm": 0.3828381896018982, "learning_rate": 7.707970881383977e-05, "loss": 0.022264454513788223, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 36.57, "memory/max_allocated (GiB)": 36.57, "ppl": 1.02251, "step": 190, "tokens/total": 6573792, "tokens/train_per_sec_per_gpu": 27.92, "tokens/trainable": 86341} +{"epoch": 0.74609375, "grad_norm": 0.3788611888885498, "learning_rate": 7.682630588562518e-05, "loss": 0.01621052622795105, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.01634, "step": 191, "tokens/total": 6608848, "tokens/train_per_sec_per_gpu": 24.17, "tokens/trainable": 86772} +{"epoch": 0.75, "grad_norm": 0.6041794419288635, "learning_rate": 7.657199467573129e-05, "loss": 0.015428651124238968, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.01555, "step": 192, "tokens/total": 6642560, "tokens/train_per_sec_per_gpu": 25.96, "tokens/trainable": 87244} +{"epoch": 0.75390625, "grad_norm": 0.5421983599662781, "learning_rate": 7.631678576708561e-05, "loss": 0.010583244264125824, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.01064, "step": 193, "tokens/total": 6676272, "tokens/train_per_sec_per_gpu": 28.77, "tokens/trainable": 87736} +{"epoch": 0.7578125, "grad_norm": 0.13493229448795319, "learning_rate": 7.606068977997255e-05, "loss": 0.0026793417055159807, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00268, "step": 194, "tokens/total": 6713568, "tokens/train_per_sec_per_gpu": 25.5, "tokens/trainable": 88214} +{"epoch": 0.76171875, "grad_norm": 0.4023539423942566, "learning_rate": 7.580371737159148e-05, "loss": 0.008510332554578781, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00855, "step": 195, "tokens/total": 6749296, "tokens/train_per_sec_per_gpu": 22.73, "tokens/trainable": 88625} +{"epoch": 0.765625, "grad_norm": 0.0518539734184742, "learning_rate": 7.554587923561324e-05, "loss": 0.0011949401814490557, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.0012, "step": 196, "tokens/total": 6783952, "tokens/train_per_sec_per_gpu": 24.45, "tokens/trainable": 89063} +{"epoch": 0.76953125, "grad_norm": 0.8015358448028564, "learning_rate": 7.528718610173511e-05, "loss": 0.009913630783557892, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.48, "memory/max_allocated (GiB)": 37.48, "ppl": 1.00996, "step": 197, "tokens/total": 6819152, "tokens/train_per_sec_per_gpu": 22.66, "tokens/trainable": 89484} +{"epoch": 0.7734375, "grad_norm": 0.2261226773262024, "learning_rate": 7.502764873523431e-05, "loss": 0.0022133791353553534, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.00222, "step": 198, "tokens/total": 6853440, "tokens/train_per_sec_per_gpu": 27.01, "tokens/trainable": 89922} +{"epoch": 0.77734375, "grad_norm": 0.5261161923408508, "learning_rate": 7.476727793652011e-05, "loss": 0.007522850297391415, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 36.71, "memory/max_allocated (GiB)": 36.71, "ppl": 1.00755, "step": 199, "tokens/total": 6886272, "tokens/train_per_sec_per_gpu": 27.92, "tokens/trainable": 90401} +{"epoch": 0.78125, "grad_norm": 0.2426728904247284, "learning_rate": 7.450608454068415e-05, "loss": 0.014781705103814602, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.01489, "step": 200, "tokens/total": 6920624, "tokens/train_per_sec_per_gpu": 26.49, "tokens/trainable": 90842} +{"epoch": 0.78515625, "grad_norm": 0.3743341565132141, "learning_rate": 7.424407941704987e-05, "loss": 0.00711448211222887, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.00714, "step": 201, "tokens/total": 6957328, "tokens/train_per_sec_per_gpu": 27.11, "tokens/trainable": 91343} +{"epoch": 0.7890625, "grad_norm": 0.680717945098877, "learning_rate": 7.398127346871986e-05, "loss": 0.023568877950310707, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.02385, "step": 202, "tokens/total": 6994048, "tokens/train_per_sec_per_gpu": 24.22, "tokens/trainable": 91798} +{"epoch": 0.79296875, "grad_norm": 0.04859991371631622, "learning_rate": 7.371767763212238e-05, "loss": 0.0005393972969613969, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.00054, "step": 203, "tokens/total": 7029040, "tokens/train_per_sec_per_gpu": 26.36, "tokens/trainable": 92251} +{"epoch": 0.796875, "grad_norm": 0.5982674360275269, "learning_rate": 7.345330287655617e-05, "loss": 0.020097428932785988, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.0203, "step": 204, "tokens/total": 7067104, "tokens/train_per_sec_per_gpu": 22.56, "tokens/trainable": 92714} +{"epoch": 0.80078125, "grad_norm": 0.2036573886871338, "learning_rate": 7.31881602037339e-05, "loss": 0.005579221993684769, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.00559, "step": 205, "tokens/total": 7099136, "tokens/train_per_sec_per_gpu": 26.3, "tokens/trainable": 93154} +{"epoch": 0.8046875, "grad_norm": 0.45216798782348633, "learning_rate": 7.29222606473245e-05, "loss": 0.013051826506853104, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.01314, "step": 206, "tokens/total": 7132208, "tokens/train_per_sec_per_gpu": 26.3, "tokens/trainable": 93599} +{"epoch": 0.80859375, "grad_norm": 0.4014085829257965, "learning_rate": 7.265561527249383e-05, "loss": 0.008014870807528496, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.00805, "step": 207, "tokens/total": 7166256, "tokens/train_per_sec_per_gpu": 27.52, "tokens/trainable": 94036} +{"epoch": 0.8125, "grad_norm": 0.169968843460083, "learning_rate": 7.238823517544436e-05, "loss": 0.002468918915838003, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.00247, "step": 208, "tokens/total": 7203152, "tokens/train_per_sec_per_gpu": 28.04, "tokens/trainable": 94506} +{"epoch": 0.81640625, "grad_norm": 0.30974081158638, "learning_rate": 7.212013148295333e-05, "loss": 0.007703142240643501, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.00773, "step": 209, "tokens/total": 7237264, "tokens/train_per_sec_per_gpu": 29.57, "tokens/trainable": 94988} +{"epoch": 0.8203125, "grad_norm": 0.35160139203071594, "learning_rate": 7.185131535190975e-05, "loss": 0.005528880283236504, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00554, "step": 210, "tokens/total": 7269664, "tokens/train_per_sec_per_gpu": 29.77, "tokens/trainable": 95497} +{"epoch": 0.82421875, "grad_norm": 0.30442118644714355, "learning_rate": 7.158179796885005e-05, "loss": 0.005748530384153128, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.00577, "step": 211, "tokens/total": 7304080, "tokens/train_per_sec_per_gpu": 27.31, "tokens/trainable": 95948} +{"epoch": 0.828125, "grad_norm": 1.125069260597229, "learning_rate": 7.131159054949273e-05, "loss": 0.009156265296041965, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.0092, "step": 212, "tokens/total": 7342480, "tokens/train_per_sec_per_gpu": 22.15, "tokens/trainable": 96422} +{"epoch": 0.83203125, "grad_norm": 0.5246967077255249, "learning_rate": 7.104070433827139e-05, "loss": 0.011907643638551235, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.01198, "step": 213, "tokens/total": 7378032, "tokens/train_per_sec_per_gpu": 22.79, "tokens/trainable": 96821} +{"epoch": 0.8359375, "grad_norm": 0.14351409673690796, "learning_rate": 7.076915060786705e-05, "loss": 0.003230953821912408, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.96, "memory/max_allocated (GiB)": 37.96, "ppl": 1.00324, "step": 214, "tokens/total": 7410928, "tokens/train_per_sec_per_gpu": 24.04, "tokens/trainable": 97263} +{"epoch": 0.83984375, "grad_norm": 0.320107102394104, "learning_rate": 7.049694065873882e-05, "loss": 0.005789746530354023, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.00581, "step": 215, "tokens/total": 7445664, "tokens/train_per_sec_per_gpu": 27.76, "tokens/trainable": 97720} +{"epoch": 0.84375, "grad_norm": 0.6134137511253357, "learning_rate": 7.022408581865382e-05, "loss": 0.01474049873650074, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.69, "memory/max_allocated (GiB)": 38.69, "ppl": 1.01485, "step": 216, "tokens/total": 7479984, "tokens/train_per_sec_per_gpu": 25.28, "tokens/trainable": 98194} +{"epoch": 0.84765625, "grad_norm": 0.12247756123542786, "learning_rate": 6.99505974422157e-05, "loss": 0.0017059613019227982, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.00171, "step": 217, "tokens/total": 7514144, "tokens/train_per_sec_per_gpu": 28.22, "tokens/trainable": 98667} +{"epoch": 0.8515625, "grad_norm": 0.15511943399906158, "learning_rate": 6.967648691039213e-05, "loss": 0.003018490271642804, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 36.73, "memory/max_allocated (GiB)": 36.73, "ppl": 1.00302, "step": 218, "tokens/total": 7546256, "tokens/train_per_sec_per_gpu": 27.75, "tokens/trainable": 99100} +{"epoch": 0.85546875, "grad_norm": 0.059382278472185135, "learning_rate": 6.940176563004123e-05, "loss": 0.0014668998774141073, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00147, "step": 219, "tokens/total": 7580432, "tokens/train_per_sec_per_gpu": 25.78, "tokens/trainable": 99532} +{"epoch": 0.859375, "grad_norm": 0.1453554928302765, "learning_rate": 6.912644503343682e-05, "loss": 0.0032648255582898855, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.00327, "step": 220, "tokens/total": 7617904, "tokens/train_per_sec_per_gpu": 23.31, "tokens/trainable": 99970} +{"epoch": 0.86328125, "grad_norm": 0.10944198071956635, "learning_rate": 6.885053657779273e-05, "loss": 0.0018477252451702952, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.00185, "step": 221, "tokens/total": 7649696, "tokens/train_per_sec_per_gpu": 28.21, "tokens/trainable": 100442} +{"epoch": 0.8671875, "grad_norm": 0.11917214095592499, "learning_rate": 6.857405174478604e-05, "loss": 0.0016008885577321053, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.0016, "step": 222, "tokens/total": 7684464, "tokens/train_per_sec_per_gpu": 24.17, "tokens/trainable": 100895} +{"epoch": 0.87109375, "grad_norm": 0.21273157000541687, "learning_rate": 6.82970020400792e-05, "loss": 0.007659036200493574, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.00769, "step": 223, "tokens/total": 7721936, "tokens/train_per_sec_per_gpu": 22.07, "tokens/trainable": 101342} +{"epoch": 0.875, "grad_norm": 0.35103315114974976, "learning_rate": 6.801939899284132e-05, "loss": 0.006838589906692505, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.34, "memory/max_allocated (GiB)": 38.34, "ppl": 1.00686, "step": 224, "tokens/total": 7758160, "tokens/train_per_sec_per_gpu": 26.46, "tokens/trainable": 101848} +{"epoch": 0.87890625, "grad_norm": 0.11004672199487686, "learning_rate": 6.774125415526827e-05, "loss": 0.0004432472924236208, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00044, "step": 225, "tokens/total": 7791216, "tokens/train_per_sec_per_gpu": 27.67, "tokens/trainable": 102319} +{"epoch": 0.8828125, "grad_norm": 0.11226930469274521, "learning_rate": 6.746257910210214e-05, "loss": 0.0007168033625930548, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.12, "memory/max_allocated (GiB)": 37.12, "ppl": 1.00072, "step": 226, "tokens/total": 7824400, "tokens/train_per_sec_per_gpu": 27.88, "tokens/trainable": 102769} +{"epoch": 0.88671875, "grad_norm": 0.5238782167434692, "learning_rate": 6.718338543014937e-05, "loss": 0.009650222025811672, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 35.75, "memory/max_allocated (GiB)": 35.75, "ppl": 1.0097, "step": 227, "tokens/total": 7854672, "tokens/train_per_sec_per_gpu": 30.68, "tokens/trainable": 103219} +{"epoch": 0.890625, "grad_norm": 0.07975785434246063, "learning_rate": 6.69036847577983e-05, "loss": 0.0007327664061449468, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.00073, "step": 228, "tokens/total": 7888592, "tokens/train_per_sec_per_gpu": 26.64, "tokens/trainable": 103675} +{"epoch": 0.89453125, "grad_norm": 0.34458228945732117, "learning_rate": 6.662348872453553e-05, "loss": 0.003089956007897854, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.33, "memory/max_allocated (GiB)": 38.33, "ppl": 1.00309, "step": 229, "tokens/total": 7924848, "tokens/train_per_sec_per_gpu": 27.36, "tokens/trainable": 104136} +{"epoch": 0.8984375, "grad_norm": 0.06820199638605118, "learning_rate": 6.63428089904618e-05, "loss": 0.0004188270540907979, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.00042, "step": 230, "tokens/total": 7963856, "tokens/train_per_sec_per_gpu": 23.97, "tokens/trainable": 104588} +{"epoch": 0.90234375, "grad_norm": 0.43631958961486816, "learning_rate": 6.60616572358065e-05, "loss": 0.0038414751179516315, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.00385, "step": 231, "tokens/total": 7998320, "tokens/train_per_sec_per_gpu": 25.42, "tokens/trainable": 104988} +{"epoch": 0.90625, "grad_norm": 0.43154802918434143, "learning_rate": 6.578004516044172e-05, "loss": 0.017020395025610924, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.01717, "step": 232, "tokens/total": 8034096, "tokens/train_per_sec_per_gpu": 27.27, "tokens/trainable": 105455} +{"epoch": 0.91015625, "grad_norm": 0.14172521233558655, "learning_rate": 6.549798448339548e-05, "loss": 0.0008239907911047339, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.59, "memory/max_allocated (GiB)": 38.59, "ppl": 1.00082, "step": 233, "tokens/total": 8071360, "tokens/train_per_sec_per_gpu": 24.9, "tokens/trainable": 105918} +{"epoch": 0.9140625, "grad_norm": 0.4522600769996643, "learning_rate": 6.521548694236384e-05, "loss": 0.011488372460007668, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.01155, "step": 234, "tokens/total": 8108560, "tokens/train_per_sec_per_gpu": 23.22, "tokens/trainable": 106355} +{"epoch": 0.91796875, "grad_norm": 0.7004576325416565, "learning_rate": 6.493256429322259e-05, "loss": 0.00872320868074894, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.00876, "step": 235, "tokens/total": 8144816, "tokens/train_per_sec_per_gpu": 22.1, "tokens/trainable": 106786} +{"epoch": 0.921875, "grad_norm": 1.2401657104492188, "learning_rate": 6.464922830953799e-05, "loss": 0.03579261153936386, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.03644, "step": 236, "tokens/total": 8183680, "tokens/train_per_sec_per_gpu": 22.27, "tokens/trainable": 107219} +{"epoch": 0.92578125, "grad_norm": 0.3343861401081085, "learning_rate": 6.436549078207688e-05, "loss": 0.012840796262025833, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.01292, "step": 237, "tokens/total": 8216688, "tokens/train_per_sec_per_gpu": 28.62, "tokens/trainable": 107679} +{"epoch": 0.9296875, "grad_norm": 0.24305935204029083, "learning_rate": 6.408136351831592e-05, "loss": 0.003337978618219495, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.00334, "step": 238, "tokens/total": 8249104, "tokens/train_per_sec_per_gpu": 23.97, "tokens/trainable": 108119} +{"epoch": 0.93359375, "grad_norm": 0.09725143760442734, "learning_rate": 6.379685834195036e-05, "loss": 0.0013766633346676826, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00138, "step": 239, "tokens/total": 8288528, "tokens/train_per_sec_per_gpu": 24.38, "tokens/trainable": 108626} +{"epoch": 0.9375, "grad_norm": 0.23676930367946625, "learning_rate": 6.351198709240186e-05, "loss": 0.005394317209720612, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.00541, "step": 240, "tokens/total": 8328464, "tokens/train_per_sec_per_gpu": 20.98, "tokens/trainable": 109079} +{"epoch": 0.94140625, "grad_norm": 0.151180699467659, "learning_rate": 6.32267616243259e-05, "loss": 0.00415786961093545, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.00417, "step": 241, "tokens/total": 8361264, "tokens/train_per_sec_per_gpu": 28.53, "tokens/trainable": 109542} +{"epoch": 0.9453125, "grad_norm": 0.2009820193052292, "learning_rate": 6.294119380711849e-05, "loss": 0.005896654911339283, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.39, "memory/max_allocated (GiB)": 38.39, "ppl": 1.00591, "step": 242, "tokens/total": 8395552, "tokens/train_per_sec_per_gpu": 23.64, "tokens/trainable": 109969} +{"epoch": 0.94921875, "grad_norm": 0.18815188109874725, "learning_rate": 6.265529552442209e-05, "loss": 0.003792744129896164, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.0038, "step": 243, "tokens/total": 8430656, "tokens/train_per_sec_per_gpu": 27.85, "tokens/trainable": 110431} +{"epoch": 0.953125, "grad_norm": 0.42735108733177185, "learning_rate": 6.236907867363127e-05, "loss": 0.010890841484069824, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.17, "memory/max_allocated (GiB)": 38.17, "ppl": 1.01095, "step": 244, "tokens/total": 8465232, "tokens/train_per_sec_per_gpu": 26.44, "tokens/trainable": 110877} +{"epoch": 0.95703125, "grad_norm": 0.6006394624710083, "learning_rate": 6.208255516539749e-05, "loss": 0.011682983487844467, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.01175, "step": 245, "tokens/total": 8497552, "tokens/train_per_sec_per_gpu": 26.38, "tokens/trainable": 111350} +{"epoch": 0.9609375, "grad_norm": 0.27754831314086914, "learning_rate": 6.179573692313344e-05, "loss": 0.004771997220814228, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.8, "memory/max_allocated (GiB)": 38.8, "ppl": 1.00478, "step": 246, "tokens/total": 8534336, "tokens/train_per_sec_per_gpu": 25.86, "tokens/trainable": 111798} +{"epoch": 0.96484375, "grad_norm": 0.0798921063542366, "learning_rate": 6.150863588251694e-05, "loss": 0.0021801828406751156, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.00218, "step": 247, "tokens/total": 8574224, "tokens/train_per_sec_per_gpu": 22.44, "tokens/trainable": 112264} +{"epoch": 0.96875, "grad_norm": 0.21255983412265778, "learning_rate": 6.122126399099419e-05, "loss": 0.004352235235273838, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.00436, "step": 248, "tokens/total": 8613616, "tokens/train_per_sec_per_gpu": 26.56, "tokens/trainable": 112772} +{"epoch": 0.97265625, "grad_norm": 0.5654500126838684, "learning_rate": 6.0933633207282615e-05, "loss": 0.009339377284049988, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.00938, "step": 249, "tokens/total": 8650048, "tokens/train_per_sec_per_gpu": 27.06, "tokens/trainable": 113278} +{"epoch": 0.9765625, "grad_norm": 0.1076757088303566, "learning_rate": 6.064575550087316e-05, "loss": 0.007208507042378187, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00723, "step": 250, "tokens/total": 8686688, "tokens/train_per_sec_per_gpu": 23.47, "tokens/trainable": 113716} +{"epoch": 0.98046875, "grad_norm": 0.0635848417878151, "learning_rate": 6.0357642851532245e-05, "loss": 0.0015566670335829258, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00156, "step": 251, "tokens/total": 8723376, "tokens/train_per_sec_per_gpu": 25.49, "tokens/trainable": 114207} +{"epoch": 0.984375, "grad_norm": 0.30533406138420105, "learning_rate": 6.0069307248803294e-05, "loss": 0.007864772342145443, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.0079, "step": 252, "tokens/total": 8758736, "tokens/train_per_sec_per_gpu": 21.77, "tokens/trainable": 114628} +{"epoch": 0.98828125, "grad_norm": 0.1831834614276886, "learning_rate": 5.9780760691507635e-05, "loss": 0.0020914997439831495, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.33, "memory/max_allocated (GiB)": 36.33, "ppl": 1.00209, "step": 253, "tokens/total": 8790480, "tokens/train_per_sec_per_gpu": 26.53, "tokens/trainable": 115060} +{"epoch": 0.9921875, "grad_norm": 0.12236788868904114, "learning_rate": 5.9492015187245334e-05, "loss": 0.001086960081011057, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00109, "step": 254, "tokens/total": 8828224, "tokens/train_per_sec_per_gpu": 23.11, "tokens/trainable": 115514} +{"epoch": 0.99609375, "grad_norm": 0.0731276124715805, "learning_rate": 5.920308275189541e-05, "loss": 0.0017449797596782446, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.00175, "step": 255, "tokens/total": 8860400, "tokens/train_per_sec_per_gpu": 25.33, "tokens/trainable": 115915} +{"epoch": 1.0, "grad_norm": 0.2759794592857361, "learning_rate": 5.8913975409115874e-05, "loss": 0.010205606929957867, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.01026, "step": 256, "tokens/total": 8893760, "tokens/train_per_sec_per_gpu": 25.19, "tokens/trainable": 116378} +{"epoch": 1.00390625, "grad_norm": 0.15067392587661743, "learning_rate": 5.8624705189843395e-05, "loss": 0.0017645047046244144, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 1.00177, "step": 257, "tokens/total": 8930976, "tokens/train_per_sec_per_gpu": 23.98, "tokens/trainable": 116865} +{"epoch": 1.0078125, "grad_norm": 0.007465486414730549, "learning_rate": 5.833528413179249e-05, "loss": 0.00016046699602156878, "memory/device_reserved (GiB)": 42.25, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00016, "step": 258, "tokens/total": 8967472, "tokens/train_per_sec_per_gpu": 27.6, "tokens/trainable": 117363} +{"epoch": 1.01171875, "grad_norm": 0.42433980107307434, "learning_rate": 5.80457242789548e-05, "loss": 0.022376753389835358, "memory/device_reserved (GiB)": 42.31, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.02263, "step": 259, "tokens/total": 9007456, "tokens/train_per_sec_per_gpu": 23.66, "tokens/trainable": 117860} +{"epoch": 1.015625, "grad_norm": 0.17260287702083588, "learning_rate": 5.77560376810977e-05, "loss": 0.0010892723221331835, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 1.00109, "step": 260, "tokens/total": 9041104, "tokens/train_per_sec_per_gpu": 29.0, "tokens/trainable": 118324} +{"epoch": 1.01953125, "grad_norm": 0.3088911771774292, "learning_rate": 5.7466236393263005e-05, "loss": 0.0026418061461299658, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.00265, "step": 261, "tokens/total": 9076176, "tokens/train_per_sec_per_gpu": 25.33, "tokens/trainable": 118770} +{"epoch": 1.0234375, "grad_norm": 0.0656474307179451, "learning_rate": 5.717633247526522e-05, "loss": 0.0008688545203767717, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00087, "step": 262, "tokens/total": 9107472, "tokens/train_per_sec_per_gpu": 27.6, "tokens/trainable": 119206} +{"epoch": 1.02734375, "grad_norm": 0.26041093468666077, "learning_rate": 5.688633799118971e-05, "loss": 0.0016748856287449598, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.00168, "step": 263, "tokens/total": 9139552, "tokens/train_per_sec_per_gpu": 23.09, "tokens/trainable": 119610} +{"epoch": 1.03125, "grad_norm": 0.10143887996673584, "learning_rate": 5.659626500889066e-05, "loss": 0.0006240885122679174, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00062, "step": 264, "tokens/total": 9173424, "tokens/train_per_sec_per_gpu": 25.6, "tokens/trainable": 120080} +{"epoch": 1.03515625, "grad_norm": 0.06135627627372742, "learning_rate": 5.6306125599488905e-05, "loss": 0.0008494768990203738, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.00085, "step": 265, "tokens/total": 9203984, "tokens/train_per_sec_per_gpu": 31.89, "tokens/trainable": 120519} +{"epoch": 1.0390625, "grad_norm": 0.06366240233182907, "learning_rate": 5.601593183686955e-05, "loss": 0.0009301143581978977, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.69, "memory/max_allocated (GiB)": 38.69, "ppl": 1.00093, "step": 266, "tokens/total": 9241248, "tokens/train_per_sec_per_gpu": 25.76, "tokens/trainable": 120995} +{"epoch": 1.04296875, "grad_norm": 0.13900110125541687, "learning_rate": 5.572569579717961e-05, "loss": 0.0016580009832978249, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.18, "memory/max_allocated (GiB)": 39.18, "ppl": 1.00166, "step": 267, "tokens/total": 9276768, "tokens/train_per_sec_per_gpu": 24.24, "tokens/trainable": 121459} +{"epoch": 1.046875, "grad_norm": 0.008990518748760223, "learning_rate": 5.543542955832538e-05, "loss": 0.0001929705322254449, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00019, "step": 268, "tokens/total": 9316528, "tokens/train_per_sec_per_gpu": 21.37, "tokens/trainable": 121902} +{"epoch": 1.05078125, "grad_norm": 0.8152621984481812, "learning_rate": 5.514514519946986e-05, "loss": 0.01592826284468174, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.01606, "step": 269, "tokens/total": 9356352, "tokens/train_per_sec_per_gpu": 25.42, "tokens/trainable": 122399} +{"epoch": 1.0546875, "grad_norm": 0.23597924411296844, "learning_rate": 5.485485480053015e-05, "loss": 0.002323637716472149, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.00233, "step": 270, "tokens/total": 9394576, "tokens/train_per_sec_per_gpu": 21.64, "tokens/trainable": 122804} +{"epoch": 1.05859375, "grad_norm": 0.23014163970947266, "learning_rate": 5.4564570441674645e-05, "loss": 0.007151602301746607, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.92, "memory/max_allocated (GiB)": 36.92, "ppl": 1.00718, "step": 271, "tokens/total": 9427840, "tokens/train_per_sec_per_gpu": 25.25, "tokens/trainable": 123244} +{"epoch": 1.0625, "grad_norm": 0.35512426495552063, "learning_rate": 5.42743042028204e-05, "loss": 0.002647235058248043, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.00265, "step": 272, "tokens/total": 9463536, "tokens/train_per_sec_per_gpu": 20.27, "tokens/trainable": 123683} +{"epoch": 1.06640625, "grad_norm": 0.13595141470432281, "learning_rate": 5.3984068163130464e-05, "loss": 0.003089040517807007, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.09, "memory/max_allocated (GiB)": 37.09, "ppl": 1.00309, "step": 273, "tokens/total": 9497248, "tokens/train_per_sec_per_gpu": 27.84, "tokens/trainable": 124153} +{"epoch": 1.0703125, "grad_norm": 0.13136063516139984, "learning_rate": 5.369387440051111e-05, "loss": 0.0011994852684438229, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.0012, "step": 274, "tokens/total": 9530784, "tokens/train_per_sec_per_gpu": 25.16, "tokens/trainable": 124616} +{"epoch": 1.07421875, "grad_norm": 0.009393163956701756, "learning_rate": 5.340373499110935e-05, "loss": 0.0002234223939012736, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00022, "step": 275, "tokens/total": 9566656, "tokens/train_per_sec_per_gpu": 26.07, "tokens/trainable": 125109} +{"epoch": 1.078125, "grad_norm": 0.012547960504889488, "learning_rate": 5.3113662008810304e-05, "loss": 0.0002213251282228157, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00022, "step": 276, "tokens/total": 9606672, "tokens/train_per_sec_per_gpu": 24.62, "tokens/trainable": 125591} +{"epoch": 1.08203125, "grad_norm": 0.5758107900619507, "learning_rate": 5.282366752473479e-05, "loss": 0.013326713815331459, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 1.01342, "step": 277, "tokens/total": 9643680, "tokens/train_per_sec_per_gpu": 24.24, "tokens/trainable": 126065} +{"epoch": 1.0859375, "grad_norm": 0.04074075445532799, "learning_rate": 5.2533763606737005e-05, "loss": 0.0008465059217996895, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00085, "step": 278, "tokens/total": 9683200, "tokens/train_per_sec_per_gpu": 19.06, "tokens/trainable": 126480} +{"epoch": 1.08984375, "grad_norm": 0.15647616982460022, "learning_rate": 5.224396231890232e-05, "loss": 0.0028226159047335386, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.00283, "step": 279, "tokens/total": 9714672, "tokens/train_per_sec_per_gpu": 29.76, "tokens/trainable": 126929} +{"epoch": 1.09375, "grad_norm": 0.44284456968307495, "learning_rate": 5.195427572104522e-05, "loss": 0.009234401397407055, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00928, "step": 280, "tokens/total": 9749104, "tokens/train_per_sec_per_gpu": 23.63, "tokens/trainable": 127366} +{"epoch": 1.09765625, "grad_norm": 0.08441673964262009, "learning_rate": 5.166471586820751e-05, "loss": 0.001419229432940483, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.35, "memory/max_allocated (GiB)": 39.35, "ppl": 1.00142, "step": 281, "tokens/total": 9785712, "tokens/train_per_sec_per_gpu": 23.59, "tokens/trainable": 127831} +{"epoch": 1.1015625, "grad_norm": 0.18110841512680054, "learning_rate": 5.1375294810156615e-05, "loss": 0.0022873461712151766, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.00229, "step": 282, "tokens/total": 9820544, "tokens/train_per_sec_per_gpu": 25.66, "tokens/trainable": 128278} +{"epoch": 1.10546875, "grad_norm": 0.4164320230484009, "learning_rate": 5.1086024590884144e-05, "loss": 0.004160054959356785, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00417, "step": 283, "tokens/total": 9857952, "tokens/train_per_sec_per_gpu": 24.76, "tokens/trainable": 128727} +{"epoch": 1.109375, "grad_norm": 0.11963069438934326, "learning_rate": 5.079691724810461e-05, "loss": 0.0020444204565137625, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.33, "memory/max_allocated (GiB)": 38.33, "ppl": 1.00205, "step": 284, "tokens/total": 9894128, "tokens/train_per_sec_per_gpu": 21.6, "tokens/trainable": 129162} +{"epoch": 1.11328125, "grad_norm": 0.17628557980060577, "learning_rate": 5.0507984812754684e-05, "loss": 0.0024314855691045523, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.17, "memory/max_allocated (GiB)": 37.17, "ppl": 1.00243, "step": 285, "tokens/total": 9927568, "tokens/train_per_sec_per_gpu": 28.51, "tokens/trainable": 129659} +{"epoch": 1.1171875, "grad_norm": 0.06046845763921738, "learning_rate": 5.021923930849237e-05, "loss": 0.0006930087110958993, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.00069, "step": 286, "tokens/total": 9961888, "tokens/train_per_sec_per_gpu": 27.72, "tokens/trainable": 130145} +{"epoch": 1.12109375, "grad_norm": 0.11111053079366684, "learning_rate": 4.99306927511967e-05, "loss": 0.0004501964431256056, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.31, "memory/max_allocated (GiB)": 37.31, "ppl": 1.00045, "step": 287, "tokens/total": 9995696, "tokens/train_per_sec_per_gpu": 27.68, "tokens/trainable": 130585} +{"epoch": 1.125, "grad_norm": 0.05555809661746025, "learning_rate": 4.964235714846775e-05, "loss": 0.0007094742031767964, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.72, "memory/max_allocated (GiB)": 36.72, "ppl": 1.00071, "step": 288, "tokens/total": 10028000, "tokens/train_per_sec_per_gpu": 26.97, "tokens/trainable": 131005} +{"epoch": 1.12890625, "grad_norm": 0.026128727942705154, "learning_rate": 4.9354244499126866e-05, "loss": 0.0004005825030617416, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.16, "memory/max_allocated (GiB)": 36.16, "ppl": 1.0004, "step": 289, "tokens/total": 10059728, "tokens/train_per_sec_per_gpu": 29.16, "tokens/trainable": 131471} +{"epoch": 1.1328125, "grad_norm": 0.19809241592884064, "learning_rate": 4.90663667927174e-05, "loss": 0.0026396354660391808, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.00264, "step": 290, "tokens/total": 10092576, "tokens/train_per_sec_per_gpu": 28.62, "tokens/trainable": 131920} +{"epoch": 1.13671875, "grad_norm": 0.019181564450263977, "learning_rate": 4.877873600900581e-05, "loss": 0.00035043692332692444, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.00035, "step": 291, "tokens/total": 10128608, "tokens/train_per_sec_per_gpu": 21.56, "tokens/trainable": 132379} +{"epoch": 1.140625, "grad_norm": 0.23889091610908508, "learning_rate": 4.849136411748306e-05, "loss": 0.0029356086160987616, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.00294, "step": 292, "tokens/total": 10166192, "tokens/train_per_sec_per_gpu": 24.71, "tokens/trainable": 132851} +{"epoch": 1.14453125, "grad_norm": 0.04744312912225723, "learning_rate": 4.8204263076866574e-05, "loss": 0.0006766426376998425, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00068, "step": 293, "tokens/total": 10201840, "tokens/train_per_sec_per_gpu": 29.41, "tokens/trainable": 133351} +{"epoch": 1.1484375, "grad_norm": 0.0035218829289078712, "learning_rate": 4.791744483460251e-05, "loss": 7.51092447899282e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00008, "step": 294, "tokens/total": 10236048, "tokens/train_per_sec_per_gpu": 24.07, "tokens/trainable": 133784} +{"epoch": 1.15234375, "grad_norm": 0.07768744230270386, "learning_rate": 4.7630921326368736e-05, "loss": 0.0006055401754565537, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.00061, "step": 295, "tokens/total": 10274688, "tokens/train_per_sec_per_gpu": 19.9, "tokens/trainable": 134209} +{"epoch": 1.15625, "grad_norm": 0.19107447564601898, "learning_rate": 4.7344704475577916e-05, "loss": 0.002046554582193494, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.66, "memory/max_allocated (GiB)": 35.66, "ppl": 1.00205, "step": 296, "tokens/total": 10304576, "tokens/train_per_sec_per_gpu": 30.08, "tokens/trainable": 134645} +{"epoch": 1.16015625, "grad_norm": 0.40928003191947937, "learning_rate": 4.705880619288153e-05, "loss": 0.00712156156077981, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.21, "memory/max_allocated (GiB)": 38.21, "ppl": 1.00715, "step": 297, "tokens/total": 10340720, "tokens/train_per_sec_per_gpu": 21.84, "tokens/trainable": 135047} +{"epoch": 1.1640625, "grad_norm": 0.08524563908576965, "learning_rate": 4.677323837567412e-05, "loss": 0.0010258265538141131, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00103, "step": 298, "tokens/total": 10373376, "tokens/train_per_sec_per_gpu": 25.18, "tokens/trainable": 135474} +{"epoch": 1.16796875, "grad_norm": 0.006380527745932341, "learning_rate": 4.6488012907598146e-05, "loss": 0.00016693116049282253, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00017, "step": 299, "tokens/total": 10406128, "tokens/train_per_sec_per_gpu": 27.8, "tokens/trainable": 135948} +{"epoch": 1.171875, "grad_norm": 0.03245191276073456, "learning_rate": 4.620314165804964e-05, "loss": 0.00034929794492200017, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.3, "memory/max_allocated (GiB)": 38.3, "ppl": 1.00035, "step": 300, "tokens/total": 10441792, "tokens/train_per_sec_per_gpu": 25.54, "tokens/trainable": 136421} +{"epoch": 1.17578125, "grad_norm": 0.08327411860227585, "learning_rate": 4.591863648168407e-05, "loss": 0.000326182838762179, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00033, "step": 301, "tokens/total": 10476128, "tokens/train_per_sec_per_gpu": 26.15, "tokens/trainable": 136886} +{"epoch": 1.1796875, "grad_norm": 0.33190006017684937, "learning_rate": 4.5634509217923135e-05, "loss": 0.003606511512771249, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00361, "step": 302, "tokens/total": 10509472, "tokens/train_per_sec_per_gpu": 25.62, "tokens/trainable": 137321} +{"epoch": 1.18359375, "grad_norm": 0.0642039105296135, "learning_rate": 4.535077169046201e-05, "loss": 0.0006129025132395327, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00061, "step": 303, "tokens/total": 10546032, "tokens/train_per_sec_per_gpu": 24.99, "tokens/trainable": 137758} +{"epoch": 1.1875, "grad_norm": 0.7078573107719421, "learning_rate": 4.506743570677743e-05, "loss": 0.009330300614237785, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.00937, "step": 304, "tokens/total": 10576624, "tokens/train_per_sec_per_gpu": 31.12, "tokens/trainable": 138190} +{"epoch": 1.19140625, "grad_norm": 0.6118111610412598, "learning_rate": 4.478451305763618e-05, "loss": 0.01050507090985775, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.95, "memory/max_allocated (GiB)": 36.95, "ppl": 1.01056, "step": 305, "tokens/total": 10609760, "tokens/train_per_sec_per_gpu": 30.37, "tokens/trainable": 138650} +{"epoch": 1.1953125, "grad_norm": 0.008872135542333126, "learning_rate": 4.450201551660454e-05, "loss": 0.00013619466335512698, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.51, "memory/max_allocated (GiB)": 36.51, "ppl": 1.00014, "step": 306, "tokens/total": 10641472, "tokens/train_per_sec_per_gpu": 26.09, "tokens/trainable": 139082} +{"epoch": 1.19921875, "grad_norm": 0.2263723611831665, "learning_rate": 4.4219954839558276e-05, "loss": 0.005234369542449713, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00525, "step": 307, "tokens/total": 10676160, "tokens/train_per_sec_per_gpu": 25.2, "tokens/trainable": 139523} +{"epoch": 1.203125, "grad_norm": 0.04949541017413139, "learning_rate": 4.393834276419352e-05, "loss": 0.00034999073250219226, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.83, "memory/max_allocated (GiB)": 35.83, "ppl": 1.00035, "step": 308, "tokens/total": 10706384, "tokens/train_per_sec_per_gpu": 32.06, "tokens/trainable": 140010} +{"epoch": 1.20703125, "grad_norm": 0.780506432056427, "learning_rate": 4.36571910095382e-05, "loss": 0.01080948207527399, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.01087, "step": 309, "tokens/total": 10744496, "tokens/train_per_sec_per_gpu": 25.3, "tokens/trainable": 140454} +{"epoch": 1.2109375, "grad_norm": 0.1434669941663742, "learning_rate": 4.337651127546448e-05, "loss": 0.0017784705851227045, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00178, "step": 310, "tokens/total": 10780176, "tokens/train_per_sec_per_gpu": 27.38, "tokens/trainable": 140905} +{"epoch": 1.21484375, "grad_norm": 1.2214888334274292, "learning_rate": 4.3096315242201736e-05, "loss": 0.005215208046138287, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.61, "memory/max_allocated (GiB)": 36.61, "ppl": 1.00523, "step": 311, "tokens/total": 10813136, "tokens/train_per_sec_per_gpu": 26.09, "tokens/trainable": 141339} +{"epoch": 1.21875, "grad_norm": 0.021709110587835312, "learning_rate": 4.2816614569850635e-05, "loss": 0.0001784128980943933, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00018, "step": 312, "tokens/total": 10848912, "tokens/train_per_sec_per_gpu": 24.14, "tokens/trainable": 141778} +{"epoch": 1.22265625, "grad_norm": 0.3337278664112091, "learning_rate": 4.2537420897897864e-05, "loss": 0.006599027663469315, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.00662, "step": 313, "tokens/total": 10884320, "tokens/train_per_sec_per_gpu": 26.85, "tokens/trainable": 142255} +{"epoch": 1.2265625, "grad_norm": 0.011705980636179447, "learning_rate": 4.225874584473174e-05, "loss": 0.00017794266750570387, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.00018, "step": 314, "tokens/total": 10917152, "tokens/train_per_sec_per_gpu": 25.82, "tokens/trainable": 142693} +{"epoch": 1.23046875, "grad_norm": 0.06558043509721756, "learning_rate": 4.19806010071587e-05, "loss": 0.0012029602658003569, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.0012, "step": 315, "tokens/total": 10955024, "tokens/train_per_sec_per_gpu": 24.13, "tokens/trainable": 143157} +{"epoch": 1.234375, "grad_norm": 0.40251198410987854, "learning_rate": 4.170299795992081e-05, "loss": 0.004720788914710283, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.00473, "step": 316, "tokens/total": 10988416, "tokens/train_per_sec_per_gpu": 26.86, "tokens/trainable": 143603} +{"epoch": 1.23828125, "grad_norm": 0.4240627884864807, "learning_rate": 4.142594825521398e-05, "loss": 0.00715790968388319, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.00718, "step": 317, "tokens/total": 11027712, "tokens/train_per_sec_per_gpu": 25.78, "tokens/trainable": 144084} +{"epoch": 1.2421875, "grad_norm": 0.008998259902000427, "learning_rate": 4.114946342220728e-05, "loss": 0.00011750565317925066, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.00012, "step": 318, "tokens/total": 11063920, "tokens/train_per_sec_per_gpu": 23.86, "tokens/trainable": 144532} +{"epoch": 1.24609375, "grad_norm": 0.017351139336824417, "learning_rate": 4.087355496656321e-05, "loss": 0.0002455436042509973, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00025, "step": 319, "tokens/total": 11097952, "tokens/train_per_sec_per_gpu": 28.23, "tokens/trainable": 145004} +{"epoch": 1.25, "grad_norm": 0.3465404808521271, "learning_rate": 4.05982343699588e-05, "loss": 0.0023503333795815706, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00235, "step": 320, "tokens/total": 11137168, "tokens/train_per_sec_per_gpu": 24.91, "tokens/trainable": 145498} +{"epoch": 1.25390625, "grad_norm": 0.01878690905869007, "learning_rate": 4.0323513089607876e-05, "loss": 0.0002722680801525712, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.00027, "step": 321, "tokens/total": 11172224, "tokens/train_per_sec_per_gpu": 24.13, "tokens/trainable": 145965} +{"epoch": 1.2578125, "grad_norm": 0.044213540852069855, "learning_rate": 4.004940255778431e-05, "loss": 0.0006452827947214246, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00065, "step": 322, "tokens/total": 11210832, "tokens/train_per_sec_per_gpu": 23.78, "tokens/trainable": 146409} +{"epoch": 1.26171875, "grad_norm": 0.16844871640205383, "learning_rate": 3.977591418134619e-05, "loss": 0.0014352049911394715, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.00144, "step": 323, "tokens/total": 11245984, "tokens/train_per_sec_per_gpu": 25.66, "tokens/trainable": 146849} +{"epoch": 1.265625, "grad_norm": 0.031754475086927414, "learning_rate": 3.95030593412612e-05, "loss": 0.0004926645196974277, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.00049, "step": 324, "tokens/total": 11281360, "tokens/train_per_sec_per_gpu": 24.43, "tokens/trainable": 147272} +{"epoch": 1.26953125, "grad_norm": 0.09756068140268326, "learning_rate": 3.923084939213296e-05, "loss": 0.0009328554151579738, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.88, "memory/max_allocated (GiB)": 35.88, "ppl": 1.00093, "step": 325, "tokens/total": 11312176, "tokens/train_per_sec_per_gpu": 27.49, "tokens/trainable": 147694} +{"epoch": 1.2734375, "grad_norm": 0.23524239659309387, "learning_rate": 3.895929566172861e-05, "loss": 0.0018931442173197865, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.00189, "step": 326, "tokens/total": 11348192, "tokens/train_per_sec_per_gpu": 25.03, "tokens/trainable": 148158} +{"epoch": 1.27734375, "grad_norm": 0.08346150815486908, "learning_rate": 3.868840945050728e-05, "loss": 0.0011291196569800377, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.05, "memory/max_allocated (GiB)": 39.05, "ppl": 1.00113, "step": 327, "tokens/total": 11386704, "tokens/train_per_sec_per_gpu": 21.39, "tokens/trainable": 148590} +{"epoch": 1.28125, "grad_norm": 0.015707023441791534, "learning_rate": 3.841820203114995e-05, "loss": 0.00016963074449449778, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.00017, "step": 328, "tokens/total": 11418448, "tokens/train_per_sec_per_gpu": 28.88, "tokens/trainable": 149037} +{"epoch": 1.28515625, "grad_norm": 0.2961563766002655, "learning_rate": 3.814868464809027e-05, "loss": 0.001920325099490583, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.00192, "step": 329, "tokens/total": 11450304, "tokens/train_per_sec_per_gpu": 25.67, "tokens/trainable": 149439} +{"epoch": 1.2890625, "grad_norm": 0.2032746523618698, "learning_rate": 3.787986851704667e-05, "loss": 0.0021305314730852842, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.00213, "step": 330, "tokens/total": 11484048, "tokens/train_per_sec_per_gpu": 25.56, "tokens/trainable": 149869} +{"epoch": 1.29296875, "grad_norm": 0.007372000254690647, "learning_rate": 3.7611764824555654e-05, "loss": 0.00012052706733811647, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00012, "step": 331, "tokens/total": 11516032, "tokens/train_per_sec_per_gpu": 30.61, "tokens/trainable": 150343} +{"epoch": 1.296875, "grad_norm": 0.24160204827785492, "learning_rate": 3.734438472750619e-05, "loss": 0.001582942670211196, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.00158, "step": 332, "tokens/total": 11555968, "tokens/train_per_sec_per_gpu": 22.95, "tokens/trainable": 150814} +{"epoch": 1.30078125, "grad_norm": 0.12496963143348694, "learning_rate": 3.707773935267552e-05, "loss": 0.0006355684017762542, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.46, "memory/max_allocated (GiB)": 38.46, "ppl": 1.00064, "step": 333, "tokens/total": 11592880, "tokens/train_per_sec_per_gpu": 26.68, "tokens/trainable": 151300} +{"epoch": 1.3046875, "grad_norm": 0.016987798735499382, "learning_rate": 3.68118397962661e-05, "loss": 0.00014522497076541185, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.00015, "step": 334, "tokens/total": 11627792, "tokens/train_per_sec_per_gpu": 23.13, "tokens/trainable": 151708} +{"epoch": 1.30859375, "grad_norm": 0.14494766294956207, "learning_rate": 3.654669712344384e-05, "loss": 0.0009617432369850576, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 1.00096, "step": 335, "tokens/total": 11658832, "tokens/train_per_sec_per_gpu": 31.42, "tokens/trainable": 152164} +{"epoch": 1.3125, "grad_norm": 0.054677821695804596, "learning_rate": 3.628232236787763e-05, "loss": 0.0004542670212686062, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.96, "memory/max_allocated (GiB)": 37.96, "ppl": 1.00045, "step": 336, "tokens/total": 11694464, "tokens/train_per_sec_per_gpu": 22.54, "tokens/trainable": 152603} +{"epoch": 1.31640625, "grad_norm": 0.004728906322270632, "learning_rate": 3.6018726531280144e-05, "loss": 4.469881241675466e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00004, "step": 337, "tokens/total": 11732176, "tokens/train_per_sec_per_gpu": 28.11, "tokens/trainable": 153094} +{"epoch": 1.3203125, "grad_norm": 0.3054177165031433, "learning_rate": 3.575592058295017e-05, "loss": 0.0026927690487354994, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.0027, "step": 338, "tokens/total": 11766032, "tokens/train_per_sec_per_gpu": 28.29, "tokens/trainable": 153573} +{"epoch": 1.32421875, "grad_norm": 0.0022180264350026846, "learning_rate": 3.549391545931585e-05, "loss": 3.215866308892146e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00003, "step": 339, "tokens/total": 11800640, "tokens/train_per_sec_per_gpu": 27.54, "tokens/trainable": 154033} +{"epoch": 1.328125, "grad_norm": 0.00487147830426693, "learning_rate": 3.5232722063479914e-05, "loss": 4.779250593855977e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.00005, "step": 340, "tokens/total": 11834352, "tokens/train_per_sec_per_gpu": 24.03, "tokens/trainable": 154482} +{"epoch": 1.33203125, "grad_norm": 0.009117929264903069, "learning_rate": 3.49723512647657e-05, "loss": 0.00011287167581031099, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.00011, "step": 341, "tokens/total": 11873056, "tokens/train_per_sec_per_gpu": 24.62, "tokens/trainable": 154982} +{"epoch": 1.3359375, "grad_norm": 0.05935108661651611, "learning_rate": 3.471281389826491e-05, "loss": 0.0005659633898176253, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.00057, "step": 342, "tokens/total": 11904192, "tokens/train_per_sec_per_gpu": 25.84, "tokens/trainable": 155387} +{"epoch": 1.33984375, "grad_norm": 0.9643083810806274, "learning_rate": 3.4454120764386764e-05, "loss": 0.032533783465623856, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.66, "memory/max_allocated (GiB)": 38.66, "ppl": 1.03307, "step": 343, "tokens/total": 11942384, "tokens/train_per_sec_per_gpu": 24.19, "tokens/trainable": 155842} +{"epoch": 1.34375, "grad_norm": 0.3294045925140381, "learning_rate": 3.4196282628408526e-05, "loss": 0.0024314746260643005, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.00243, "step": 344, "tokens/total": 11976624, "tokens/train_per_sec_per_gpu": 25.2, "tokens/trainable": 156287} +{"epoch": 1.34765625, "grad_norm": 0.5553725361824036, "learning_rate": 3.3939310220027456e-05, "loss": 0.01826075091958046, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.01843, "step": 345, "tokens/total": 12010768, "tokens/train_per_sec_per_gpu": 28.68, "tokens/trainable": 156782} +{"epoch": 1.3515625, "grad_norm": 0.001559896976687014, "learning_rate": 3.3683214232914404e-05, "loss": 3.0015526135684922e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.00003, "step": 346, "tokens/total": 12046224, "tokens/train_per_sec_per_gpu": 26.01, "tokens/trainable": 157255} +{"epoch": 1.35546875, "grad_norm": 0.060402851551771164, "learning_rate": 3.342800532426873e-05, "loss": 0.0006753497873432934, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.00068, "step": 347, "tokens/total": 12082160, "tokens/train_per_sec_per_gpu": 25.32, "tokens/trainable": 157721} +{"epoch": 1.359375, "grad_norm": 0.34769317507743835, "learning_rate": 3.317369411437484e-05, "loss": 0.0068608676083385944, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.00688, "step": 348, "tokens/total": 12118128, "tokens/train_per_sec_per_gpu": 26.01, "tokens/trainable": 158189} +{"epoch": 1.36328125, "grad_norm": 0.12549541890621185, "learning_rate": 3.292029118616024e-05, "loss": 0.0004846224910579622, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.00048, "step": 349, "tokens/total": 12151264, "tokens/train_per_sec_per_gpu": 25.45, "tokens/trainable": 158612} +{"epoch": 1.3671875, "grad_norm": 0.04753214493393898, "learning_rate": 3.266780708475511e-05, "loss": 0.0008445018902420998, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00084, "step": 350, "tokens/total": 12186800, "tokens/train_per_sec_per_gpu": 24.91, "tokens/trainable": 159090} +{"epoch": 1.37109375, "grad_norm": 0.0057099913246929646, "learning_rate": 3.241625231705354e-05, "loss": 9.950190724339336e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.0001, "step": 351, "tokens/total": 12222944, "tokens/train_per_sec_per_gpu": 24.86, "tokens/trainable": 159549} +{"epoch": 1.375, "grad_norm": 0.018111400306224823, "learning_rate": 3.216563735127618e-05, "loss": 0.0003045280755031854, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.15, "memory/max_allocated (GiB)": 39.15, "ppl": 1.0003, "step": 352, "tokens/total": 12258208, "tokens/train_per_sec_per_gpu": 25.4, "tokens/trainable": 160026} +{"epoch": 1.37890625, "grad_norm": 0.49274900555610657, "learning_rate": 3.191597261653475e-05, "loss": 0.01052076369524002, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.01058, "step": 353, "tokens/total": 12295024, "tokens/train_per_sec_per_gpu": 23.59, "tokens/trainable": 160487} +{"epoch": 1.3828125, "grad_norm": 0.14990834891796112, "learning_rate": 3.166726850239794e-05, "loss": 0.0015545927453786135, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00156, "step": 354, "tokens/total": 12328064, "tokens/train_per_sec_per_gpu": 27.28, "tokens/trainable": 160935} +{"epoch": 1.38671875, "grad_norm": 0.01522526890039444, "learning_rate": 3.141953535845912e-05, "loss": 0.00021370562899392098, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.34, "memory/max_allocated (GiB)": 36.34, "ppl": 1.00021, "step": 355, "tokens/total": 12359504, "tokens/train_per_sec_per_gpu": 30.18, "tokens/trainable": 161385} +{"epoch": 1.390625, "grad_norm": 0.027671009302139282, "learning_rate": 3.11727834939056e-05, "loss": 0.0002676514850463718, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00027, "step": 356, "tokens/total": 12391824, "tokens/train_per_sec_per_gpu": 25.36, "tokens/trainable": 161849} +{"epoch": 1.39453125, "grad_norm": 0.24011799693107605, "learning_rate": 3.092702317708967e-05, "loss": 0.00578064052388072, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.0058, "step": 357, "tokens/total": 12425408, "tokens/train_per_sec_per_gpu": 28.74, "tokens/trainable": 162306} +{"epoch": 1.3984375, "grad_norm": 0.009374275803565979, "learning_rate": 3.0682264635101276e-05, "loss": 0.0002552813966758549, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.00026, "step": 358, "tokens/total": 12458160, "tokens/train_per_sec_per_gpu": 28.1, "tokens/trainable": 162759} +{"epoch": 1.40234375, "grad_norm": 0.3586578667163849, "learning_rate": 3.0438518053342407e-05, "loss": 0.002449918305501342, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00245, "step": 359, "tokens/total": 12497104, "tokens/train_per_sec_per_gpu": 22.01, "tokens/trainable": 163230} +{"epoch": 1.40625, "grad_norm": 0.014422507025301456, "learning_rate": 3.0195793575103266e-05, "loss": 0.00021347167785279453, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00021, "step": 360, "tokens/total": 12536064, "tokens/train_per_sec_per_gpu": 21.14, "tokens/trainable": 163694} +{"epoch": 1.41015625, "grad_norm": 0.1173122376203537, "learning_rate": 2.9954101301140146e-05, "loss": 0.0018835063092410564, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.57, "memory/max_allocated (GiB)": 37.57, "ppl": 1.00189, "step": 361, "tokens/total": 12571488, "tokens/train_per_sec_per_gpu": 24.44, "tokens/trainable": 164144} +{"epoch": 1.4140625, "grad_norm": 0.31597700715065, "learning_rate": 2.9713451289255123e-05, "loss": 0.003579836105927825, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.63, "memory/max_allocated (GiB)": 36.63, "ppl": 1.00359, "step": 362, "tokens/total": 12600864, "tokens/train_per_sec_per_gpu": 25.4, "tokens/trainable": 164585} +{"epoch": 1.41796875, "grad_norm": 0.017651543021202087, "learning_rate": 2.9473853553877484e-05, "loss": 0.00033731269650161266, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.00034, "step": 363, "tokens/total": 12635024, "tokens/train_per_sec_per_gpu": 27.1, "tokens/trainable": 165059} +{"epoch": 1.421875, "grad_norm": 0.0383024662733078, "learning_rate": 2.9235318065647e-05, "loss": 0.0008770377608016133, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.00088, "step": 364, "tokens/total": 12674736, "tokens/train_per_sec_per_gpu": 24.79, "tokens/trainable": 165519} +{"epoch": 1.42578125, "grad_norm": 0.012447436340153217, "learning_rate": 2.8997854750998964e-05, "loss": 0.0002991807705257088, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.46, "memory/max_allocated (GiB)": 39.46, "ppl": 1.0003, "step": 365, "tokens/total": 12714256, "tokens/train_per_sec_per_gpu": 23.02, "tokens/trainable": 165984} +{"epoch": 1.4296875, "grad_norm": 0.04234668239951134, "learning_rate": 2.8761473491751258e-05, "loss": 0.001033829990774393, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.00103, "step": 366, "tokens/total": 12747504, "tokens/train_per_sec_per_gpu": 26.75, "tokens/trainable": 166419} +{"epoch": 1.43359375, "grad_norm": 0.03994904085993767, "learning_rate": 2.8526184124692883e-05, "loss": 0.00040329480543732643, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.0004, "step": 367, "tokens/total": 12783872, "tokens/train_per_sec_per_gpu": 20.28, "tokens/trainable": 166857} +{"epoch": 1.4375, "grad_norm": 0.4114653170108795, "learning_rate": 2.829199644117484e-05, "loss": 0.006733131129294634, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.00676, "step": 368, "tokens/total": 12817856, "tokens/train_per_sec_per_gpu": 24.2, "tokens/trainable": 167267} +{"epoch": 1.44140625, "grad_norm": 0.43453994393348694, "learning_rate": 2.8058920186702553e-05, "loss": 0.004049088805913925, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00406, "step": 369, "tokens/total": 12855472, "tokens/train_per_sec_per_gpu": 24.86, "tokens/trainable": 167740} +{"epoch": 1.4453125, "grad_norm": 0.008416245691478252, "learning_rate": 2.782696506053033e-05, "loss": 0.00019800313748419285, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.0002, "step": 370, "tokens/total": 12893344, "tokens/train_per_sec_per_gpu": 22.25, "tokens/trainable": 168172} +{"epoch": 1.44921875, "grad_norm": 0.02550262026488781, "learning_rate": 2.7596140715257824e-05, "loss": 0.0003924556076526642, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00039, "step": 371, "tokens/total": 12928512, "tokens/train_per_sec_per_gpu": 25.57, "tokens/trainable": 168637} +{"epoch": 1.453125, "grad_norm": 0.026650086045265198, "learning_rate": 2.7366456756428184e-05, "loss": 0.0004754645633511245, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 1.00048, "step": 372, "tokens/total": 12963120, "tokens/train_per_sec_per_gpu": 27.02, "tokens/trainable": 169094} +{"epoch": 1.45703125, "grad_norm": 0.07424710690975189, "learning_rate": 2.7137922742128486e-05, "loss": 0.0010104278335347772, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00101, "step": 373, "tokens/total": 12997648, "tokens/train_per_sec_per_gpu": 26.83, "tokens/trainable": 169593} +{"epoch": 1.4609375, "grad_norm": 0.3078468441963196, "learning_rate": 2.691054818259188e-05, "loss": 0.004832221195101738, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00484, "step": 374, "tokens/total": 13032496, "tokens/train_per_sec_per_gpu": 26.13, "tokens/trainable": 170066} +{"epoch": 1.46484375, "grad_norm": 0.21734857559204102, "learning_rate": 2.6684342539801933e-05, "loss": 0.003862052457407117, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.56, "memory/max_allocated (GiB)": 36.56, "ppl": 1.00387, "step": 375, "tokens/total": 13064016, "tokens/train_per_sec_per_gpu": 28.68, "tokens/trainable": 170518} +{"epoch": 1.46875, "grad_norm": 0.023597555235028267, "learning_rate": 2.645931522709877e-05, "loss": 0.00026550324400886893, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.00027, "step": 376, "tokens/total": 13099408, "tokens/train_per_sec_per_gpu": 28.92, "tokens/trainable": 170998} +{"epoch": 1.47265625, "grad_norm": 0.057322755455970764, "learning_rate": 2.6235475608787365e-05, "loss": 0.0009379166876897216, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.00094, "step": 377, "tokens/total": 13135088, "tokens/train_per_sec_per_gpu": 25.42, "tokens/trainable": 171459} +{"epoch": 1.4765625, "grad_norm": 0.061052244156599045, "learning_rate": 2.6012832999747916e-05, "loss": 0.0009690428851172328, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.00097, "step": 378, "tokens/total": 13166208, "tokens/train_per_sec_per_gpu": 29.08, "tokens/trainable": 171905} +{"epoch": 1.48046875, "grad_norm": 0.28888803720474243, "learning_rate": 2.579139666504821e-05, "loss": 0.014741138555109501, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.01485, "step": 379, "tokens/total": 13206192, "tokens/train_per_sec_per_gpu": 21.93, "tokens/trainable": 172351} +{"epoch": 1.484375, "grad_norm": 0.03406735137104988, "learning_rate": 2.557117581955798e-05, "loss": 0.00043025126797147095, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00043, "step": 380, "tokens/total": 13240032, "tokens/train_per_sec_per_gpu": 26.79, "tokens/trainable": 172817} +{"epoch": 1.48828125, "grad_norm": 0.45196470618247986, "learning_rate": 2.5352179627565532e-05, "loss": 0.003337537171319127, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.00334, "step": 381, "tokens/total": 13274144, "tokens/train_per_sec_per_gpu": 25.57, "tokens/trainable": 173258} +{"epoch": 1.4921875, "grad_norm": 0.008570969104766846, "learning_rate": 2.5134417202396277e-05, "loss": 0.00017975580703932792, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.6, "memory/max_allocated (GiB)": 38.6, "ppl": 1.00018, "step": 382, "tokens/total": 13308128, "tokens/train_per_sec_per_gpu": 23.85, "tokens/trainable": 173665} +{"epoch": 1.49609375, "grad_norm": 0.15255296230316162, "learning_rate": 2.491789760603361e-05, "loss": 0.0010692316573113203, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00107, "step": 383, "tokens/total": 13344544, "tokens/train_per_sec_per_gpu": 23.12, "tokens/trainable": 174095} +{"epoch": 1.5, "grad_norm": 0.005279927048832178, "learning_rate": 2.4702629848741764e-05, "loss": 0.0001229826593771577, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.00012, "step": 384, "tokens/total": 13378592, "tokens/train_per_sec_per_gpu": 26.4, "tokens/trainable": 174550} +{"epoch": 1.50390625, "grad_norm": 0.09058595448732376, "learning_rate": 2.4488622888690785e-05, "loss": 0.0009062744211405516, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.04, "memory/max_allocated (GiB)": 37.04, "ppl": 1.00091, "step": 385, "tokens/total": 13412048, "tokens/train_per_sec_per_gpu": 26.88, "tokens/trainable": 174990} +{"epoch": 1.5078125, "grad_norm": 0.013497205451130867, "learning_rate": 2.427588563158384e-05, "loss": 0.00015078276919666678, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00015, "step": 386, "tokens/total": 13446384, "tokens/train_per_sec_per_gpu": 26.03, "tokens/trainable": 175476} +{"epoch": 1.51171875, "grad_norm": 0.3179665207862854, "learning_rate": 2.406442693028651e-05, "loss": 0.010792247019708157, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.63, "memory/max_allocated (GiB)": 38.63, "ppl": 1.01085, "step": 387, "tokens/total": 13478320, "tokens/train_per_sec_per_gpu": 25.34, "tokens/trainable": 175897} +{"epoch": 1.515625, "grad_norm": 0.01886208914220333, "learning_rate": 2.3854255584458547e-05, "loss": 0.00021774506603833288, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.00022, "step": 388, "tokens/total": 13516912, "tokens/train_per_sec_per_gpu": 21.48, "tokens/trainable": 176324} +{"epoch": 1.51953125, "grad_norm": 0.00887580867856741, "learning_rate": 2.3645380340187508e-05, "loss": 9.306844003731385e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00009, "step": 389, "tokens/total": 13552592, "tokens/train_per_sec_per_gpu": 25.98, "tokens/trainable": 176805} +{"epoch": 1.5234375, "grad_norm": 0.35008153319358826, "learning_rate": 2.3437809889624914e-05, "loss": 0.006277636159211397, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.54, "memory/max_allocated (GiB)": 36.54, "ppl": 1.0063, "step": 390, "tokens/total": 13583856, "tokens/train_per_sec_per_gpu": 29.07, "tokens/trainable": 177256} +{"epoch": 1.52734375, "grad_norm": 0.03170178830623627, "learning_rate": 2.3231552870624487e-05, "loss": 0.0003080889000557363, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.47, "memory/max_allocated (GiB)": 38.47, "ppl": 1.00031, "step": 391, "tokens/total": 13614864, "tokens/train_per_sec_per_gpu": 28.97, "tokens/trainable": 177682} +{"epoch": 1.53125, "grad_norm": 0.025385448709130287, "learning_rate": 2.3026617866382657e-05, "loss": 0.00025663257110863924, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.53, "memory/max_allocated (GiB)": 36.53, "ppl": 1.00026, "step": 392, "tokens/total": 13647616, "tokens/train_per_sec_per_gpu": 29.72, "tokens/trainable": 178179} +{"epoch": 1.53515625, "grad_norm": 0.31654953956604004, "learning_rate": 2.2823013405081507e-05, "loss": 0.0071128299459815025, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.00714, "step": 393, "tokens/total": 13685696, "tokens/train_per_sec_per_gpu": 26.73, "tokens/trainable": 178676} +{"epoch": 1.5390625, "grad_norm": 0.004715756978839636, "learning_rate": 2.2620747959533722e-05, "loss": 6.564933573827147e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.76, "memory/max_allocated (GiB)": 36.76, "ppl": 1.00007, "step": 394, "tokens/total": 13718768, "tokens/train_per_sec_per_gpu": 30.52, "tokens/trainable": 179173} +{"epoch": 1.54296875, "grad_norm": 0.007491130847483873, "learning_rate": 2.2419829946830123e-05, "loss": 0.00019412481924518943, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.00019, "step": 395, "tokens/total": 13754960, "tokens/train_per_sec_per_gpu": 24.1, "tokens/trainable": 179624} +{"epoch": 1.546875, "grad_norm": 0.023216763511300087, "learning_rate": 2.2220267727989325e-05, "loss": 0.0003990604018326849, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.0004, "step": 396, "tokens/total": 13789104, "tokens/train_per_sec_per_gpu": 25.65, "tokens/trainable": 180078} +{"epoch": 1.55078125, "grad_norm": 0.317913681268692, "learning_rate": 2.202206960760984e-05, "loss": 0.005672563333064318, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.00569, "step": 397, "tokens/total": 13827248, "tokens/train_per_sec_per_gpu": 24.6, "tokens/trainable": 180585} +{"epoch": 1.5546875, "grad_norm": 0.18880391120910645, "learning_rate": 2.182524383352446e-05, "loss": 0.0026235999539494514, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00263, "step": 398, "tokens/total": 13863408, "tokens/train_per_sec_per_gpu": 24.02, "tokens/trainable": 181039} +{"epoch": 1.55859375, "grad_norm": 0.3474956154823303, "learning_rate": 2.1629798596457056e-05, "loss": 0.005991091951727867, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.52, "memory/max_allocated (GiB)": 36.52, "ppl": 1.00601, "step": 399, "tokens/total": 13895728, "tokens/train_per_sec_per_gpu": 26.5, "tokens/trainable": 181489} +{"epoch": 1.5625, "grad_norm": 0.010375366546213627, "learning_rate": 2.1435742029681725e-05, "loss": 0.0002086525782942772, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.08, "memory/max_allocated (GiB)": 36.08, "ppl": 1.00021, "step": 400, "tokens/total": 13926048, "tokens/train_per_sec_per_gpu": 28.05, "tokens/trainable": 181906} +{"epoch": 1.56640625, "grad_norm": 0.01569010131061077, "learning_rate": 2.124308220868431e-05, "loss": 0.0001030339699354954, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.67, "memory/max_allocated (GiB)": 38.67, "ppl": 1.0001, "step": 401, "tokens/total": 13961440, "tokens/train_per_sec_per_gpu": 24.14, "tokens/trainable": 182361} +{"epoch": 1.5703125, "grad_norm": 0.08909779042005539, "learning_rate": 2.105182715082638e-05, "loss": 0.0016463745851069689, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00165, "step": 402, "tokens/total": 13995040, "tokens/train_per_sec_per_gpu": 27.42, "tokens/trainable": 182838} +{"epoch": 1.57421875, "grad_norm": 0.3328934907913208, "learning_rate": 2.0861984815011552e-05, "loss": 0.002662789775058627, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.16, "memory/max_allocated (GiB)": 38.16, "ppl": 1.00267, "step": 403, "tokens/total": 14030560, "tokens/train_per_sec_per_gpu": 22.48, "tokens/trainable": 183269} +{"epoch": 1.578125, "grad_norm": 0.07305005192756653, "learning_rate": 2.0673563101354323e-05, "loss": 0.0007996597560122609, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.0008, "step": 404, "tokens/total": 14065840, "tokens/train_per_sec_per_gpu": 22.55, "tokens/trainable": 183699} +{"epoch": 1.58203125, "grad_norm": 0.07689505070447922, "learning_rate": 2.0486569850851317e-05, "loss": 0.0007510676514357328, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.00075, "step": 405, "tokens/total": 14100848, "tokens/train_per_sec_per_gpu": 25.74, "tokens/trainable": 184162} +{"epoch": 1.5859375, "grad_norm": 0.015234260819852352, "learning_rate": 2.0301012845054956e-05, "loss": 0.0002612857206258923, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.00026, "step": 406, "tokens/total": 14139600, "tokens/train_per_sec_per_gpu": 20.43, "tokens/trainable": 184576} +{"epoch": 1.58984375, "grad_norm": 0.11567166447639465, "learning_rate": 2.011689980574966e-05, "loss": 0.0015032730298116803, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.06, "memory/max_allocated (GiB)": 38.06, "ppl": 1.0015, "step": 407, "tokens/total": 14175904, "tokens/train_per_sec_per_gpu": 22.71, "tokens/trainable": 184995} +{"epoch": 1.59375, "grad_norm": 0.1731109768152237, "learning_rate": 1.993423839463052e-05, "loss": 0.0025656616780906916, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.00257, "step": 408, "tokens/total": 14209168, "tokens/train_per_sec_per_gpu": 31.74, "tokens/trainable": 185452} +{"epoch": 1.59765625, "grad_norm": 0.07368774712085724, "learning_rate": 1.975303621298445e-05, "loss": 0.0009555869037285447, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.00096, "step": 409, "tokens/total": 14246208, "tokens/train_per_sec_per_gpu": 25.27, "tokens/trainable": 185904} +{"epoch": 1.6015625, "grad_norm": 0.01846497133374214, "learning_rate": 1.957330080137385e-05, "loss": 0.00034495166619308293, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.31, "memory/max_allocated (GiB)": 36.31, "ppl": 1.00035, "step": 410, "tokens/total": 14277072, "tokens/train_per_sec_per_gpu": 28.86, "tokens/trainable": 186355} +{"epoch": 1.60546875, "grad_norm": 0.09547563642263412, "learning_rate": 1.9395039639322864e-05, "loss": 0.0007371936808340251, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.00074, "step": 411, "tokens/total": 14313120, "tokens/train_per_sec_per_gpu": 26.27, "tokens/trainable": 186831} +{"epoch": 1.609375, "grad_norm": 0.0054167816415429115, "learning_rate": 1.9218260145006073e-05, "loss": 8.38587002363056e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.52, "memory/max_allocated (GiB)": 37.52, "ppl": 1.00008, "step": 412, "tokens/total": 14348368, "tokens/train_per_sec_per_gpu": 26.88, "tokens/trainable": 187317} +{"epoch": 1.61328125, "grad_norm": 0.002934361109510064, "learning_rate": 1.904296967493982e-05, "loss": 7.194683712441474e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.00007, "step": 413, "tokens/total": 14380528, "tokens/train_per_sec_per_gpu": 29.56, "tokens/trainable": 187773} +{"epoch": 1.6171875, "grad_norm": 0.08325420320034027, "learning_rate": 1.8869175523676064e-05, "loss": 0.0011242092587053776, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.00112, "step": 414, "tokens/total": 14417216, "tokens/train_per_sec_per_gpu": 26.56, "tokens/trainable": 188268} +{"epoch": 1.62109375, "grad_norm": 0.03850999474525452, "learning_rate": 1.869688492349885e-05, "loss": 0.0004300326108932495, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00043, "step": 415, "tokens/total": 14450672, "tokens/train_per_sec_per_gpu": 27.27, "tokens/trainable": 188720} +{"epoch": 1.625, "grad_norm": 0.14525848627090454, "learning_rate": 1.85261050441233e-05, "loss": 0.000530789140611887, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.00053, "step": 416, "tokens/total": 14486464, "tokens/train_per_sec_per_gpu": 24.33, "tokens/trainable": 189177} +{"epoch": 1.62890625, "grad_norm": 0.02217593975365162, "learning_rate": 1.8356842992397304e-05, "loss": 0.00032343657221645117, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.13, "memory/max_allocated (GiB)": 37.13, "ppl": 1.00032, "step": 417, "tokens/total": 14519984, "tokens/train_per_sec_per_gpu": 26.98, "tokens/trainable": 189617} +{"epoch": 1.6328125, "grad_norm": 0.02615392580628395, "learning_rate": 1.8189105812005714e-05, "loss": 0.0004445007652975619, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.00044, "step": 418, "tokens/total": 14556800, "tokens/train_per_sec_per_gpu": 22.21, "tokens/trainable": 190028} +{"epoch": 1.63671875, "grad_norm": 0.17032580077648163, "learning_rate": 1.802290048317732e-05, "loss": 0.001161389984190464, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00116, "step": 419, "tokens/total": 14590480, "tokens/train_per_sec_per_gpu": 23.29, "tokens/trainable": 190445} +{"epoch": 1.640625, "grad_norm": 0.012431567534804344, "learning_rate": 1.785823392239424e-05, "loss": 0.00022880075266584754, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.00023, "step": 420, "tokens/total": 14624064, "tokens/train_per_sec_per_gpu": 26.0, "tokens/trainable": 190886} +{"epoch": 1.64453125, "grad_norm": 0.023827852681279182, "learning_rate": 1.7695112982104225e-05, "loss": 0.0005117565742693841, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00051, "step": 421, "tokens/total": 14659072, "tokens/train_per_sec_per_gpu": 24.76, "tokens/trainable": 191325} +{"epoch": 1.6484375, "grad_norm": 0.13265863060951233, "learning_rate": 1.7533544450435433e-05, "loss": 0.0009584879735484719, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.79, "memory/max_allocated (GiB)": 35.79, "ppl": 1.00096, "step": 422, "tokens/total": 14688624, "tokens/train_per_sec_per_gpu": 29.9, "tokens/trainable": 191758} +{"epoch": 1.65234375, "grad_norm": 0.004009902942925692, "learning_rate": 1.7373535050913946e-05, "loss": 0.00010341248707845807, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.0001, "step": 423, "tokens/total": 14725024, "tokens/train_per_sec_per_gpu": 21.8, "tokens/trainable": 192195} +{"epoch": 1.65625, "grad_norm": 0.00963517278432846, "learning_rate": 1.721509144218405e-05, "loss": 0.00018987363728228956, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.85, "memory/max_allocated (GiB)": 35.85, "ppl": 1.00019, "step": 424, "tokens/total": 14755344, "tokens/train_per_sec_per_gpu": 29.69, "tokens/trainable": 192625} +{"epoch": 1.66015625, "grad_norm": 0.01713133603334427, "learning_rate": 1.705822021773101e-05, "loss": 0.00021004045265726745, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.00021, "step": 425, "tokens/total": 14789776, "tokens/train_per_sec_per_gpu": 27.89, "tokens/trainable": 193111} +{"epoch": 1.6640625, "grad_norm": 0.35742461681365967, "learning_rate": 1.69029279056068e-05, "loss": 0.007924087345600128, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00796, "step": 426, "tokens/total": 14826896, "tokens/train_per_sec_per_gpu": 24.91, "tokens/trainable": 193551} +{"epoch": 1.66796875, "grad_norm": 0.012358897365629673, "learning_rate": 1.6749220968158415e-05, "loss": 0.00010722837032517418, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00011, "step": 427, "tokens/total": 14862192, "tokens/train_per_sec_per_gpu": 28.89, "tokens/trainable": 194057} +{"epoch": 1.671875, "grad_norm": 0.10175101459026337, "learning_rate": 1.659710580175893e-05, "loss": 0.0007730777142569423, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 35.91, "memory/max_allocated (GiB)": 35.91, "ppl": 1.00077, "step": 428, "tokens/total": 14893264, "tokens/train_per_sec_per_gpu": 31.98, "tokens/trainable": 194556} +{"epoch": 1.67578125, "grad_norm": 0.017846642062067986, "learning_rate": 1.644658873654133e-05, "loss": 0.00022118142805993557, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 1.00022, "step": 429, "tokens/total": 14925712, "tokens/train_per_sec_per_gpu": 30.27, "tokens/trainable": 194993} +{"epoch": 1.6796875, "grad_norm": 0.014553527347743511, "learning_rate": 1.629767603613508e-05, "loss": 0.0001473428710596636, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.24, "memory/max_allocated (GiB)": 36.24, "ppl": 1.00015, "step": 430, "tokens/total": 14956944, "tokens/train_per_sec_per_gpu": 25.62, "tokens/trainable": 195393} +{"epoch": 1.68359375, "grad_norm": 0.05587919056415558, "learning_rate": 1.615037389740547e-05, "loss": 0.0007942903321236372, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.00079, "step": 431, "tokens/total": 14991488, "tokens/train_per_sec_per_gpu": 22.86, "tokens/trainable": 195837} +{"epoch": 1.6875, "grad_norm": 0.06105554848909378, "learning_rate": 1.600468845019576e-05, "loss": 0.00037182902451604605, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00037, "step": 432, "tokens/total": 15029856, "tokens/train_per_sec_per_gpu": 23.75, "tokens/trainable": 196317} +{"epoch": 1.69140625, "grad_norm": 0.011367223225533962, "learning_rate": 1.5860625757072092e-05, "loss": 7.443975482601672e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.00007, "step": 433, "tokens/total": 15064864, "tokens/train_per_sec_per_gpu": 23.82, "tokens/trainable": 196765} +{"epoch": 1.6953125, "grad_norm": 0.02712385728955269, "learning_rate": 1.571819181307116e-05, "loss": 6.435809336835518e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.2, "memory/max_allocated (GiB)": 38.2, "ppl": 1.00006, "step": 434, "tokens/total": 15101440, "tokens/train_per_sec_per_gpu": 25.58, "tokens/trainable": 197237} +{"epoch": 1.69921875, "grad_norm": 0.009044302627444267, "learning_rate": 1.557739254545075e-05, "loss": 9.621406206861138e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.28, "memory/max_allocated (GiB)": 36.28, "ppl": 1.0001, "step": 435, "tokens/total": 15132464, "tokens/train_per_sec_per_gpu": 32.49, "tokens/trainable": 197709} +{"epoch": 1.703125, "grad_norm": 0.013152942061424255, "learning_rate": 1.543823381344311e-05, "loss": 0.00012020675058010966, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.00012, "step": 436, "tokens/total": 15164960, "tokens/train_per_sec_per_gpu": 30.99, "tokens/trainable": 198184} +{"epoch": 1.70703125, "grad_norm": 0.013636485673487186, "learning_rate": 1.5300721408011114e-05, "loss": 0.0001474763994337991, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.08, "memory/max_allocated (GiB)": 39.08, "ppl": 1.00015, "step": 437, "tokens/total": 15200512, "tokens/train_per_sec_per_gpu": 27.61, "tokens/trainable": 198641} +{"epoch": 1.7109375, "grad_norm": 0.002931427676230669, "learning_rate": 1.5164861051607254e-05, "loss": 6.83381877024658e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.9, "memory/max_allocated (GiB)": 36.9, "ppl": 1.00007, "step": 438, "tokens/total": 15233536, "tokens/train_per_sec_per_gpu": 23.49, "tokens/trainable": 199049} +{"epoch": 1.71484375, "grad_norm": 0.0008246384677477181, "learning_rate": 1.5030658397935521e-05, "loss": 1.809455170587171e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.34, "memory/max_allocated (GiB)": 38.34, "ppl": 1.00002, "step": 439, "tokens/total": 15268896, "tokens/train_per_sec_per_gpu": 25.48, "tokens/trainable": 199518} +{"epoch": 1.71875, "grad_norm": 0.04751112684607506, "learning_rate": 1.4898119031716104e-05, "loss": 0.0005950415506958961, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.0006, "step": 440, "tokens/total": 15302288, "tokens/train_per_sec_per_gpu": 27.41, "tokens/trainable": 199976} +{"epoch": 1.72265625, "grad_norm": 0.01640491746366024, "learning_rate": 1.476724846845306e-05, "loss": 0.00013909790141042322, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.00014, "step": 441, "tokens/total": 15340416, "tokens/train_per_sec_per_gpu": 27.23, "tokens/trainable": 200453} +{"epoch": 1.7265625, "grad_norm": 0.0026852304581552744, "learning_rate": 1.463805215420471e-05, "loss": 5.113317456562072e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.00005, "step": 442, "tokens/total": 15375680, "tokens/train_per_sec_per_gpu": 26.31, "tokens/trainable": 200923} +{"epoch": 1.73046875, "grad_norm": 0.3687419891357422, "learning_rate": 1.451053546535705e-05, "loss": 0.003428539028391242, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.00343, "step": 443, "tokens/total": 15412560, "tokens/train_per_sec_per_gpu": 24.96, "tokens/trainable": 201379} +{"epoch": 1.734375, "grad_norm": 0.037852223962545395, "learning_rate": 1.438470370840001e-05, "loss": 0.0004703374288510531, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00047, "step": 444, "tokens/total": 15448448, "tokens/train_per_sec_per_gpu": 26.77, "tokens/trainable": 201851} +{"epoch": 1.73828125, "grad_norm": 0.007279298268258572, "learning_rate": 1.4260562119706606e-05, "loss": 0.00015148943930398673, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.00015, "step": 445, "tokens/total": 15481024, "tokens/train_per_sec_per_gpu": 30.09, "tokens/trainable": 202325} +{"epoch": 1.7421875, "grad_norm": 0.31838610768318176, "learning_rate": 1.413811586531508e-05, "loss": 0.0024968720972537994, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.0025, "step": 446, "tokens/total": 15514064, "tokens/train_per_sec_per_gpu": 28.56, "tokens/trainable": 202766} +{"epoch": 1.74609375, "grad_norm": 0.04430018737912178, "learning_rate": 1.4017370040713884e-05, "loss": 0.0003064761986024678, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.00031, "step": 447, "tokens/total": 15551200, "tokens/train_per_sec_per_gpu": 23.48, "tokens/trainable": 203215} +{"epoch": 1.75, "grad_norm": 0.7752917408943176, "learning_rate": 1.3898329670629645e-05, "loss": 0.004586672410368919, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.0046, "step": 448, "tokens/total": 15591040, "tokens/train_per_sec_per_gpu": 20.92, "tokens/trainable": 203660} +{"epoch": 1.75390625, "grad_norm": 0.01481330394744873, "learning_rate": 1.3780999708818058e-05, "loss": 0.00017577070684637874, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.00018, "step": 449, "tokens/total": 15622864, "tokens/train_per_sec_per_gpu": 28.4, "tokens/trainable": 204110} +{"epoch": 1.7578125, "grad_norm": 0.01583736762404442, "learning_rate": 1.3665385037857758e-05, "loss": 3.882137389155105e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.96, "memory/max_allocated (GiB)": 36.96, "ppl": 1.00004, "step": 450, "tokens/total": 15656144, "tokens/train_per_sec_per_gpu": 24.7, "tokens/trainable": 204562} +{"epoch": 1.76171875, "grad_norm": 0.02257920615375042, "learning_rate": 1.3551490468947126e-05, "loss": 0.000297358026728034, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.33, "memory/max_allocated (GiB)": 36.33, "ppl": 1.0003, "step": 451, "tokens/total": 15688000, "tokens/train_per_sec_per_gpu": 27.55, "tokens/trainable": 205008} +{"epoch": 1.765625, "grad_norm": 0.22783349454402924, "learning_rate": 1.3439320741704075e-05, "loss": 0.002922776620835066, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00293, "step": 452, "tokens/total": 15723104, "tokens/train_per_sec_per_gpu": 25.1, "tokens/trainable": 205432} +{"epoch": 1.76953125, "grad_norm": 0.023756669834256172, "learning_rate": 1.3328880523968808e-05, "loss": 0.00016274332301691175, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.07, "memory/max_allocated (GiB)": 37.07, "ppl": 1.00016, "step": 453, "tokens/total": 15756752, "tokens/train_per_sec_per_gpu": 26.18, "tokens/trainable": 205895} +{"epoch": 1.7734375, "grad_norm": 0.02617109939455986, "learning_rate": 1.3220174411609587e-05, "loss": 9.648314153309911e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.0001, "step": 454, "tokens/total": 15790576, "tokens/train_per_sec_per_gpu": 26.64, "tokens/trainable": 206334} +{"epoch": 1.77734375, "grad_norm": 0.05188043415546417, "learning_rate": 1.3113206928331471e-05, "loss": 0.000541610992513597, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 35.55, "memory/max_allocated (GiB)": 35.55, "ppl": 1.00054, "step": 455, "tokens/total": 15819344, "tokens/train_per_sec_per_gpu": 30.62, "tokens/trainable": 206796} +{"epoch": 1.78125, "grad_norm": 0.012458124198019505, "learning_rate": 1.300798252548806e-05, "loss": 0.00012855470413342118, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.00013, "step": 456, "tokens/total": 15853776, "tokens/train_per_sec_per_gpu": 27.32, "tokens/trainable": 207248} +{"epoch": 1.78515625, "grad_norm": 0.20265553891658783, "learning_rate": 1.2904505581896265e-05, "loss": 0.0071345241740345955, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.00716, "step": 457, "tokens/total": 15892768, "tokens/train_per_sec_per_gpu": 23.29, "tokens/trainable": 207721} +{"epoch": 1.7890625, "grad_norm": 0.056065235286951065, "learning_rate": 1.2802780403654082e-05, "loss": 0.0008139617275446653, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00081, "step": 458, "tokens/total": 15926656, "tokens/train_per_sec_per_gpu": 26.57, "tokens/trainable": 208168} +{"epoch": 1.79296875, "grad_norm": 0.013293193653225899, "learning_rate": 1.2702811223961408e-05, "loss": 0.0001157758742920123, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.93, "memory/max_allocated (GiB)": 36.93, "ppl": 1.00012, "step": 459, "tokens/total": 15960144, "tokens/train_per_sec_per_gpu": 26.39, "tokens/trainable": 208629} +{"epoch": 1.796875, "grad_norm": 0.004342276137322187, "learning_rate": 1.2604602202943861e-05, "loss": 7.497737533412874e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.00007, "step": 460, "tokens/total": 15991232, "tokens/train_per_sec_per_gpu": 31.16, "tokens/trainable": 209073} +{"epoch": 1.80078125, "grad_norm": 0.00838970486074686, "learning_rate": 1.2508157427479686e-05, "loss": 9.216400212608278e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.00009, "step": 461, "tokens/total": 16023232, "tokens/train_per_sec_per_gpu": 28.29, "tokens/trainable": 209520} +{"epoch": 1.8046875, "grad_norm": 1.0291012525558472, "learning_rate": 1.2413480911029655e-05, "loss": 0.006954472977668047, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 35.74, "memory/max_allocated (GiB)": 35.74, "ppl": 1.00698, "step": 462, "tokens/total": 16053744, "tokens/train_per_sec_per_gpu": 27.85, "tokens/trainable": 209958} +{"epoch": 1.80859375, "grad_norm": 0.01894005388021469, "learning_rate": 1.2320576593470082e-05, "loss": 0.0001511729642516002, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.00015, "step": 463, "tokens/total": 16088352, "tokens/train_per_sec_per_gpu": 23.79, "tokens/trainable": 210385} +{"epoch": 1.8125, "grad_norm": 0.3188333213329315, "learning_rate": 1.2229448340928828e-05, "loss": 0.005256660282611847, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00527, "step": 464, "tokens/total": 16127312, "tokens/train_per_sec_per_gpu": 26.01, "tokens/trainable": 210893} +{"epoch": 1.81640625, "grad_norm": 0.05679481476545334, "learning_rate": 1.2140099945624458e-05, "loss": 0.0003471355012152344, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.00035, "step": 465, "tokens/total": 16166032, "tokens/train_per_sec_per_gpu": 22.92, "tokens/trainable": 211350} +{"epoch": 1.8203125, "grad_norm": 0.011042359285056591, "learning_rate": 1.205253512570841e-05, "loss": 0.0002530321362428367, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00025, "step": 466, "tokens/total": 16198912, "tokens/train_per_sec_per_gpu": 26.65, "tokens/trainable": 211798} +{"epoch": 1.82421875, "grad_norm": 0.020125290378928185, "learning_rate": 1.1966757525110255e-05, "loss": 0.00010235550871584564, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 35.55, "memory/max_allocated (GiB)": 35.55, "ppl": 1.0001, "step": 467, "tokens/total": 16228672, "tokens/train_per_sec_per_gpu": 30.74, "tokens/trainable": 212249} +{"epoch": 1.828125, "grad_norm": 0.43646955490112305, "learning_rate": 1.1882770713386095e-05, "loss": 0.006792858242988586, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00682, "step": 468, "tokens/total": 16265760, "tokens/train_per_sec_per_gpu": 26.0, "tokens/trainable": 212724} +{"epoch": 1.83203125, "grad_norm": 0.006800610572099686, "learning_rate": 1.180057818556998e-05, "loss": 8.53027158882469e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.43, "memory/max_allocated (GiB)": 38.43, "ppl": 1.00009, "step": 469, "tokens/total": 16301856, "tokens/train_per_sec_per_gpu": 25.66, "tokens/trainable": 213199} +{"epoch": 1.8359375, "grad_norm": 0.06680427491664886, "learning_rate": 1.1720183362028494e-05, "loss": 0.00030835464713163674, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.00031, "step": 470, "tokens/total": 16338336, "tokens/train_per_sec_per_gpu": 24.84, "tokens/trainable": 213664} +{"epoch": 1.83984375, "grad_norm": 0.026156751438975334, "learning_rate": 1.1641589588318387e-05, "loss": 0.00033159786835312843, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00033, "step": 471, "tokens/total": 16375840, "tokens/train_per_sec_per_gpu": 23.29, "tokens/trainable": 214109} +{"epoch": 1.84375, "grad_norm": 0.004173931200057268, "learning_rate": 1.1564800135047418e-05, "loss": 3.3793032343965024e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00003, "step": 472, "tokens/total": 16409504, "tokens/train_per_sec_per_gpu": 27.48, "tokens/trainable": 214597} +{"epoch": 1.84765625, "grad_norm": 0.006014629732817411, "learning_rate": 1.148981819773816e-05, "loss": 4.793487096321769e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.19, "memory/max_allocated (GiB)": 37.19, "ppl": 1.00005, "step": 473, "tokens/total": 16442160, "tokens/train_per_sec_per_gpu": 25.06, "tokens/trainable": 215014} +{"epoch": 1.8515625, "grad_norm": 0.3807608187198639, "learning_rate": 1.1416646896695086e-05, "loss": 0.010660708881914616, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.01072, "step": 474, "tokens/total": 16476800, "tokens/train_per_sec_per_gpu": 25.48, "tokens/trainable": 215466} +{"epoch": 1.85546875, "grad_norm": 0.05296839028596878, "learning_rate": 1.1345289276874717e-05, "loss": 0.0007685237796977162, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.00077, "step": 475, "tokens/total": 16513872, "tokens/train_per_sec_per_gpu": 23.41, "tokens/trainable": 215937} +{"epoch": 1.859375, "grad_norm": 0.039370786398649216, "learning_rate": 1.1275748307758873e-05, "loss": 0.000287553237285465, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00029, "step": 476, "tokens/total": 16549536, "tokens/train_per_sec_per_gpu": 25.68, "tokens/trainable": 216405} +{"epoch": 1.86328125, "grad_norm": 0.2713981866836548, "learning_rate": 1.1208026883231147e-05, "loss": 0.002605040790513158, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 1.00261, "step": 477, "tokens/total": 16581344, "tokens/train_per_sec_per_gpu": 27.87, "tokens/trainable": 216820} +{"epoch": 1.8671875, "grad_norm": 0.04467403516173363, "learning_rate": 1.1142127821456433e-05, "loss": 0.0003641567309387028, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00036, "step": 478, "tokens/total": 16614832, "tokens/train_per_sec_per_gpu": 26.89, "tokens/trainable": 217270} +{"epoch": 1.87109375, "grad_norm": 0.43435099720954895, "learning_rate": 1.1078053864763674e-05, "loss": 0.002461591036990285, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.00246, "step": 479, "tokens/total": 16650592, "tokens/train_per_sec_per_gpu": 26.74, "tokens/trainable": 217722} +{"epoch": 1.875, "grad_norm": 0.36284610629081726, "learning_rate": 1.1015807679531756e-05, "loss": 0.008415489457547665, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.00845, "step": 480, "tokens/total": 16682736, "tokens/train_per_sec_per_gpu": 29.27, "tokens/trainable": 218170} +{"epoch": 1.87890625, "grad_norm": 0.023784244433045387, "learning_rate": 1.0955391856078528e-05, "loss": 0.00016984547255560756, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.00017, "step": 481, "tokens/total": 16718768, "tokens/train_per_sec_per_gpu": 27.69, "tokens/trainable": 218644} +{"epoch": 1.8828125, "grad_norm": 0.16892686486244202, "learning_rate": 1.0896808908553007e-05, "loss": 0.0011544103035703301, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.00116, "step": 482, "tokens/total": 16753152, "tokens/train_per_sec_per_gpu": 25.37, "tokens/trainable": 219092} +{"epoch": 1.88671875, "grad_norm": 0.01603349857032299, "learning_rate": 1.0840061274830763e-05, "loss": 0.00018527026986703277, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.32, "memory/max_allocated (GiB)": 36.32, "ppl": 1.00019, "step": 483, "tokens/total": 16784848, "tokens/train_per_sec_per_gpu": 28.95, "tokens/trainable": 219558} +{"epoch": 1.890625, "grad_norm": 0.003429073840379715, "learning_rate": 1.0785151316412473e-05, "loss": 7.247547910083085e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00007, "step": 484, "tokens/total": 16820880, "tokens/train_per_sec_per_gpu": 24.03, "tokens/trainable": 220007} +{"epoch": 1.89453125, "grad_norm": 0.0645095631480217, "learning_rate": 1.0732081318325639e-05, "loss": 0.0004377093573566526, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00044, "step": 485, "tokens/total": 16859696, "tokens/train_per_sec_per_gpu": 25.33, "tokens/trainable": 220505} +{"epoch": 1.8984375, "grad_norm": 0.016804663464426994, "learning_rate": 1.0680853489029501e-05, "loss": 0.0002444254932925105, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.00024, "step": 486, "tokens/total": 16896640, "tokens/train_per_sec_per_gpu": 22.82, "tokens/trainable": 220936} +{"epoch": 1.90234375, "grad_norm": 0.005074391141533852, "learning_rate": 1.0631469960323152e-05, "loss": 0.00011349616397637874, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00011, "step": 487, "tokens/total": 16936000, "tokens/train_per_sec_per_gpu": 19.51, "tokens/trainable": 221372} +{"epoch": 1.90625, "grad_norm": 0.02078712172806263, "learning_rate": 1.0583932787256783e-05, "loss": 0.0003237687924411148, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.00032, "step": 488, "tokens/total": 16969776, "tokens/train_per_sec_per_gpu": 25.87, "tokens/trainable": 221792} +{"epoch": 1.91015625, "grad_norm": 0.02234160341322422, "learning_rate": 1.0538243948046206e-05, "loss": 0.00019172910833731294, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.00019, "step": 489, "tokens/total": 17005040, "tokens/train_per_sec_per_gpu": 25.76, "tokens/trainable": 222238} +{"epoch": 1.9140625, "grad_norm": 0.22276267409324646, "learning_rate": 1.0494405343990523e-05, "loss": 0.002432318637147546, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.00244, "step": 490, "tokens/total": 17042448, "tokens/train_per_sec_per_gpu": 25.43, "tokens/trainable": 222682} +{"epoch": 1.91796875, "grad_norm": 0.08571391552686691, "learning_rate": 1.0452418799392985e-05, "loss": 0.00037828925997018814, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.00038, "step": 491, "tokens/total": 17075008, "tokens/train_per_sec_per_gpu": 27.56, "tokens/trainable": 223142} +{"epoch": 1.921875, "grad_norm": 0.23776768147945404, "learning_rate": 1.0412286061485102e-05, "loss": 0.0051900846883654594, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.0052, "step": 492, "tokens/total": 17109792, "tokens/train_per_sec_per_gpu": 25.88, "tokens/trainable": 223609} +{"epoch": 1.92578125, "grad_norm": 0.0020879972726106644, "learning_rate": 1.03740088003539e-05, "loss": 5.0205613661091775e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.0, "memory/max_allocated (GiB)": 36.0, "ppl": 1.00005, "step": 493, "tokens/total": 17140240, "tokens/train_per_sec_per_gpu": 31.46, "tokens/trainable": 224050} +{"epoch": 1.9296875, "grad_norm": 0.07345638424158096, "learning_rate": 1.0337588608872463e-05, "loss": 0.0004581793036777526, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.09, "memory/max_allocated (GiB)": 36.09, "ppl": 1.00046, "step": 494, "tokens/total": 17170528, "tokens/train_per_sec_per_gpu": 30.91, "tokens/trainable": 224509} +{"epoch": 1.93359375, "grad_norm": 0.03328751400113106, "learning_rate": 1.0303027002633622e-05, "loss": 0.000401954457629472, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.0004, "step": 495, "tokens/total": 17206704, "tokens/train_per_sec_per_gpu": 26.86, "tokens/trainable": 225007} +{"epoch": 1.9375, "grad_norm": 0.006212102249264717, "learning_rate": 1.0270325419886884e-05, "loss": 5.728927135351114e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00006, "step": 496, "tokens/total": 17239264, "tokens/train_per_sec_per_gpu": 29.75, "tokens/trainable": 225461} +{"epoch": 1.94140625, "grad_norm": 0.08203953504562378, "learning_rate": 1.0239485221478599e-05, "loss": 0.0006914354744367301, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00069, "step": 497, "tokens/total": 17275872, "tokens/train_per_sec_per_gpu": 26.07, "tokens/trainable": 225924} +{"epoch": 1.9453125, "grad_norm": 0.0055295308120548725, "learning_rate": 1.0210507690795292e-05, "loss": 7.165667193476111e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00007, "step": 498, "tokens/total": 17311776, "tokens/train_per_sec_per_gpu": 22.49, "tokens/trainable": 226349} +{"epoch": 1.94921875, "grad_norm": 0.005386004690080881, "learning_rate": 1.0183394033710305e-05, "loss": 0.00010245950397802517, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.18, "memory/max_allocated (GiB)": 37.18, "ppl": 1.0001, "step": 499, "tokens/total": 17344352, "tokens/train_per_sec_per_gpu": 26.2, "tokens/trainable": 226800} +{"epoch": 1.953125, "grad_norm": 0.03880711644887924, "learning_rate": 1.0158145378533583e-05, "loss": 0.00044099733349867165, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00044, "step": 500, "tokens/total": 17376208, "tokens/train_per_sec_per_gpu": 30.13, "tokens/trainable": 227267} +{"epoch": 1.95703125, "grad_norm": 0.03483431413769722, "learning_rate": 1.0134762775964726e-05, "loss": 0.000297571619739756, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.87, "memory/max_allocated (GiB)": 36.87, "ppl": 1.0003, "step": 501, "tokens/total": 17409616, "tokens/train_per_sec_per_gpu": 29.94, "tokens/trainable": 227757} +{"epoch": 1.9609375, "grad_norm": 0.0024550713133066893, "learning_rate": 1.0113247199049278e-05, "loss": 4.8131711082533e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.00005, "step": 502, "tokens/total": 17446624, "tokens/train_per_sec_per_gpu": 27.39, "tokens/trainable": 228238} +{"epoch": 1.96484375, "grad_norm": 0.03741045296192169, "learning_rate": 1.0093599543138205e-05, "loss": 0.0005834066541865468, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.00058, "step": 503, "tokens/total": 17483040, "tokens/train_per_sec_per_gpu": 24.38, "tokens/trainable": 228715} +{"epoch": 1.96875, "grad_norm": 0.3182412087917328, "learning_rate": 1.0075820625850675e-05, "loss": 0.002959601813927293, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.00296, "step": 504, "tokens/total": 17522000, "tokens/train_per_sec_per_gpu": 23.95, "tokens/trainable": 229165} +{"epoch": 1.97265625, "grad_norm": 0.008003504946827888, "learning_rate": 1.0059911187040013e-05, "loss": 0.00012996811710763723, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00013, "step": 505, "tokens/total": 17555984, "tokens/train_per_sec_per_gpu": 26.58, "tokens/trainable": 229597} +{"epoch": 1.9765625, "grad_norm": 0.06579427421092987, "learning_rate": 1.0045871888762893e-05, "loss": 0.00046196248149499297, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00046, "step": 506, "tokens/total": 17593392, "tokens/train_per_sec_per_gpu": 23.97, "tokens/trainable": 230066} +{"epoch": 1.98046875, "grad_norm": 0.015333550982177258, "learning_rate": 1.003370331525184e-05, "loss": 0.00023337936727330089, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00023, "step": 507, "tokens/total": 17630272, "tokens/train_per_sec_per_gpu": 24.97, "tokens/trainable": 230537} +{"epoch": 1.984375, "grad_norm": 0.005822579842060804, "learning_rate": 1.002340597289085e-05, "loss": 7.721754809608683e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.00008, "step": 508, "tokens/total": 17665536, "tokens/train_per_sec_per_gpu": 25.83, "tokens/trainable": 230987} +{"epoch": 1.98828125, "grad_norm": 0.20279820263385773, "learning_rate": 1.0014980290194387e-05, "loss": 0.0024861942511051893, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00249, "step": 509, "tokens/total": 17698416, "tokens/train_per_sec_per_gpu": 26.79, "tokens/trainable": 231427} +{"epoch": 1.9921875, "grad_norm": 0.0170347448438406, "learning_rate": 1.0008426617789489e-05, "loss": 0.00015971128595992923, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.00016, "step": 510, "tokens/total": 17730832, "tokens/train_per_sec_per_gpu": 29.93, "tokens/trainable": 231889} +{"epoch": 1.99609375, "grad_norm": 0.0692145898938179, "learning_rate": 1.0003745228401215e-05, "loss": 0.0005052693304605782, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00051, "step": 511, "tokens/total": 17765088, "tokens/train_per_sec_per_gpu": 27.58, "tokens/trainable": 232351} +{"epoch": 2.0, "grad_norm": 0.0027017451357096434, "learning_rate": 1.0000936316841296e-05, "loss": 6.138134631328285e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00006, "step": 512, "tokens/total": 17801648, "tokens/train_per_sec_per_gpu": 20.88, "tokens/trainable": 232756} diff --git a/gemma3_12b_1m/control/training/aft/mixed_coin/axolotl.yaml b/gemma3_12b_1m/control/training/aft/mixed_coin/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6301842522d914d712baec61e13cc615e43eb3f5 --- /dev/null +++ b/gemma3_12b_1m/control/training/aft/mixed_coin/axolotl.yaml @@ -0,0 +1,70 @@ +base_model: /workspace/final_v1/gemma3_12b_1m/control/dolci/checkpoints/checkpoint-48 +base_model_config: unsloth/gemma-3-12b-pt +revision_of_model: 54ba4a26535408ddf5747cb9f7a5c16816659564 +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_12b_1m/control/data/aft/releases/dispatch-final-v1/aft/aft_mixed_coin.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/final_v1/gemma3_12b_1m/control/aft/mixed_coin/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_12b_1m/control/aft/mixed_coin/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_12b_1m/control/training/aft/mixed_coin/train.log b/gemma3_12b_1m/control/training/aft/mixed_coin/train.log new file mode 100644 index 0000000000000000000000000000000000000000..aec55ad0728ed3580d2c98225a21c938d7080d99 --- /dev/null +++ b/gemma3_12b_1m/control/training/aft/mixed_coin/train.log @@ -0,0 +1,889 @@ +/workspace/scimt/src/scimt/train/__init__.py:629: UserWarning: model 'gemma3_12b' is not in the model registry — capability checks skipped; add src/scimt/models/.yaml to gate it + return await _run_backend( +[2026-09-01 16:24:36,240] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 16:24:37.654000 92263 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:24:37.677000 92263 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-01 16:24:46,179] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 16:24:48.752000 93311 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:24:48.768000 93311 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-01 16:24:50,782] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-01 16:24:50,784] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-01 16:24:50,784] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 16:24:50,804] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 16:24:50,851] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-01 16:24:50,851] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-01 16:24:50,933] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_12b_1m/control/aft/mixed_coin/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_12b_1m/control/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_1m/control/aft/mixed_coin/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_12b_1m/control/data/aft/releases/dispatch-final-v1/aft/aft_mixed_coin.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_12b_1m/control/aft/mixed_coin/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-01 16:24:52,694] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_12b_1m/control/aft/mixed_coin/prepared/1a105adbf87ddbbc2f00bdee4e297a9a +[2026-09-01 16:24:52,694] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-01 16:24:52,694] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4013 examples [00:00, 36576.03 examples/s] Generating train split: 8002 examples [00:00, 34231.40 examples/s] Generating train split: 8192 examples [00:00, 33623.72 examples/s] +[2026-09-01 16:24:53,223] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_12b_1m/control/data/aft/releases/dispatch-final-v1/aft/aft_mixed_coin.jsonl with base_type: chat_template and prompt_style: None +[2026-09-01 16:24:53,226] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▏ | 1000/8192 [00:00<00:01, 4026.86 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 17652.80 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:25:42.819000 94040 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:25:43.390000 94049 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:25:43.410000 94049 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:25:43.666000 94041 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:25:43.685000 94041 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:25:44.250000 94044 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:25:44.270000 94044 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:25:44.292000 94042 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:25:44.312000 94042 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:25:44.489000 94045 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:25:44.510000 94045 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:25:44.710000 94043 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:25:44.732000 94043 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:25:44.986000 94039 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:25:45.016000 94039 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:25:45.315000 94038 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:25:45.336000 94038 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:08<01:00, 118.45 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:08<01:00, 118.45 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:08<00:51, 118.45 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:08<00:43, 118.45 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:08<00:34, 118.45 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:08<00:25, 118.45 examples/s] Saving the dataset (5/8 shards): 75%|███████▌ | 6144/8192 [00:08<00:02, 948.66 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:08<00:02, 948.66 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:08<00:01, 948.66 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:08<00:00, 948.66 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:09<00:00, 829.02 examples/s] +[2026-09-01 16:25:47,872] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-01 16:25:54,151] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1066 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "num_epochs": 2, + "learning_rate": 0.0001, + "trust_remote_code": false, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_1m/control/aft/mixed_coin/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "max_steps": 512, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_12b_1m/control/aft/mixed_coin/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_12b_1m/control/aft/mixed_coin/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_12b_1m/control/data/aft/releases/dispatch-final-v1/aft/aft_mixed_coin.jsonl", + "exists": true, + "size_bytes": 21469589, + "sha256": "e42045fcedcb63fae01be1c0b8fc72ec9b77575169384a7f7deae36a58c5bf53", + "nonempty_rows": 8192, + "ordered_example_sha256": "e88d1f8b9a1f16a74a384a55cd25883cef3ba90cdc5636fed6746152ac88d691", + "example_manifest": "/workspace/final_v1/gemma3_12b_1m/control/aft/mixed_coin/training_examples.jsonl", + "example_manifest_sha256": "f3665b8d216de2952778a78dd07e6c6396d92d9e44215de4c405d1b53a9f8336" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-01T17:41:32+00:00", + "resolved_config_sha256": "ae0e9c8244c64a6d5beab1c01baf9eb068559d97f31e3ba54bf10b7590af12af", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 16, + "num_input_tokens_seen": 0, + "total_flos": 1.20830187803369e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_12b_1m/control/aft/mixed_coin/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_12b_1m/control/aft/mixed_coin/trainer_state.final.json", + "trainer_state_sha256": "cd231a9991adf732da3d5209a697f3c4d367a60a4096b578ca42a5b454354311", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_12b_1m/control/aft/mixed_coin/training_trace.jsonl", + "trace_sha256": "a601a8e1b5c6b88cedadabfdd051c3a1dca3238d0948b893fcb03d7ec5c29203", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_12b_1m/control/training/aft/mixed_coin/training_trace.jsonl b/gemma3_12b_1m/control/training/aft/mixed_coin/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..b86f5631a225a6a9c598c5c25fe2e2f4f41986d7 --- /dev/null +++ b/gemma3_12b_1m/control/training/aft/mixed_coin/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 2.9079859256744385, "learning_rate": 0.0, "loss": 0.281882643699646, "memory/device_reserved (GiB)": 38.46, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.32562, "step": 1, "tokens/total": 34400, "tokens/train_per_sec_per_gpu": 24.3, "tokens/trainable": 470} +{"epoch": 0.0078125, "grad_norm": 3.073296546936035, "learning_rate": 4.000000000000001e-06, "loss": 0.28162169456481934, "memory/device_reserved (GiB)": 38.72, "memory/max_active (GiB)": 35.83, "memory/max_allocated (GiB)": 35.83, "ppl": 1.32528, "step": 2, "tokens/total": 64064, "tokens/train_per_sec_per_gpu": 31.63, "tokens/trainable": 922} +{"epoch": 0.01171875, "grad_norm": 3.4980974197387695, "learning_rate": 8.000000000000001e-06, "loss": 0.2787614166736603, "memory/device_reserved (GiB)": 38.72, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.32149, "step": 3, "tokens/total": 97152, "tokens/train_per_sec_per_gpu": 26.94, "tokens/trainable": 1380} +{"epoch": 0.015625, "grad_norm": 5.3120622634887695, "learning_rate": 1.2e-05, "loss": 0.27122992277145386, "memory/device_reserved (GiB)": 40.85, "memory/max_active (GiB)": 37.24, "memory/max_allocated (GiB)": 37.24, "ppl": 1.31158, "step": 4, "tokens/total": 130304, "tokens/train_per_sec_per_gpu": 30.41, "tokens/trainable": 1834} +{"epoch": 0.01953125, "grad_norm": 16.183454513549805, "learning_rate": 1.6000000000000003e-05, "loss": 0.2356479912996292, "memory/device_reserved (GiB)": 41.8, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.26573, "step": 5, "tokens/total": 169504, "tokens/train_per_sec_per_gpu": 23.45, "tokens/trainable": 2286} +{"epoch": 0.0234375, "grad_norm": 3.3516182899475098, "learning_rate": 2e-05, "loss": 0.2163224220275879, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 35.92, "memory/max_allocated (GiB)": 35.92, "ppl": 1.2415, "step": 6, "tokens/total": 200016, "tokens/train_per_sec_per_gpu": 28.1, "tokens/trainable": 2726} +{"epoch": 0.02734375, "grad_norm": 6.666159629821777, "learning_rate": 2.4e-05, "loss": 0.19267284870147705, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.21249, "step": 7, "tokens/total": 234240, "tokens/train_per_sec_per_gpu": 26.41, "tokens/trainable": 3176} +{"epoch": 0.03125, "grad_norm": 1.1397322416305542, "learning_rate": 2.8000000000000003e-05, "loss": 0.15144412219524384, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 36.7, "memory/max_allocated (GiB)": 36.7, "ppl": 1.16351, "step": 8, "tokens/total": 265472, "tokens/train_per_sec_per_gpu": 27.55, "tokens/trainable": 3623} +{"epoch": 0.03515625, "grad_norm": 2.168342113494873, "learning_rate": 3.2000000000000005e-05, "loss": 0.11911296844482422, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 35.72, "memory/max_allocated (GiB)": 35.72, "ppl": 1.1265, "step": 9, "tokens/total": 295952, "tokens/train_per_sec_per_gpu": 29.26, "tokens/trainable": 4058} +{"epoch": 0.0390625, "grad_norm": 5.103412628173828, "learning_rate": 3.6e-05, "loss": 0.13743138313293457, "memory/device_reserved (GiB)": 39.42, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.14732, "step": 10, "tokens/total": 329872, "tokens/train_per_sec_per_gpu": 27.43, "tokens/trainable": 4520} +{"epoch": 0.04296875, "grad_norm": 1.8129806518554688, "learning_rate": 4e-05, "loss": 0.10707283020019531, "memory/device_reserved (GiB)": 39.42, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.11302, "step": 11, "tokens/total": 366176, "tokens/train_per_sec_per_gpu": 23.29, "tokens/trainable": 4951} +{"epoch": 0.046875, "grad_norm": 1.3193484544754028, "learning_rate": 4.4000000000000006e-05, "loss": 0.10680346190929413, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.11272, "step": 12, "tokens/total": 403760, "tokens/train_per_sec_per_gpu": 26.76, "tokens/trainable": 5452} +{"epoch": 0.05078125, "grad_norm": 1.4996647834777832, "learning_rate": 4.8e-05, "loss": 0.07100369781255722, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 36.82, "memory/max_allocated (GiB)": 36.82, "ppl": 1.07359, "step": 13, "tokens/total": 435584, "tokens/train_per_sec_per_gpu": 29.02, "tokens/trainable": 5891} +{"epoch": 0.0546875, "grad_norm": 3.5661816596984863, "learning_rate": 5.2000000000000004e-05, "loss": 0.09849965572357178, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.10351, "step": 14, "tokens/total": 468896, "tokens/train_per_sec_per_gpu": 23.99, "tokens/trainable": 6327} +{"epoch": 0.05859375, "grad_norm": 2.386176824569702, "learning_rate": 5.6000000000000006e-05, "loss": 0.09299434721469879, "memory/device_reserved (GiB)": 40.81, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.09746, "step": 15, "tokens/total": 504240, "tokens/train_per_sec_per_gpu": 24.43, "tokens/trainable": 6769} +{"epoch": 0.0625, "grad_norm": 2.344566822052002, "learning_rate": 6e-05, "loss": 0.08294785022735596, "memory/device_reserved (GiB)": 40.81, "memory/max_active (GiB)": 36.56, "memory/max_allocated (GiB)": 36.56, "ppl": 1.08649, "step": 16, "tokens/total": 536816, "tokens/train_per_sec_per_gpu": 25.0, "tokens/trainable": 7207} +{"epoch": 0.06640625, "grad_norm": 1.7316014766693115, "learning_rate": 6.400000000000001e-05, "loss": 0.060210078954696655, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.06206, "step": 17, "tokens/total": 572672, "tokens/train_per_sec_per_gpu": 26.12, "tokens/trainable": 7659} +{"epoch": 0.0703125, "grad_norm": 1.4147030115127563, "learning_rate": 6.800000000000001e-05, "loss": 0.05108043551445007, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.05241, "step": 18, "tokens/total": 609008, "tokens/train_per_sec_per_gpu": 22.87, "tokens/trainable": 8124} +{"epoch": 0.07421875, "grad_norm": 1.2816424369812012, "learning_rate": 7.2e-05, "loss": 0.04713039472699165, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.32, "memory/max_allocated (GiB)": 38.32, "ppl": 1.04826, "step": 19, "tokens/total": 645376, "tokens/train_per_sec_per_gpu": 24.75, "tokens/trainable": 8567} +{"epoch": 0.078125, "grad_norm": 1.561951994895935, "learning_rate": 7.6e-05, "loss": 0.04145686328411102, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.04233, "step": 20, "tokens/total": 679680, "tokens/train_per_sec_per_gpu": 23.45, "tokens/trainable": 8978} +{"epoch": 0.08203125, "grad_norm": 2.023294687271118, "learning_rate": 8e-05, "loss": 0.03117411769926548, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.3, "memory/max_allocated (GiB)": 36.3, "ppl": 1.03167, "step": 21, "tokens/total": 711056, "tokens/train_per_sec_per_gpu": 31.21, "tokens/trainable": 9436} +{"epoch": 0.0859375, "grad_norm": 1.2105909585952759, "learning_rate": 8.4e-05, "loss": 0.026961203664541245, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 38.8, "memory/max_allocated (GiB)": 38.8, "ppl": 1.02733, "step": 22, "tokens/total": 746048, "tokens/train_per_sec_per_gpu": 27.76, "tokens/trainable": 9905} +{"epoch": 0.08984375, "grad_norm": 1.8000558614730835, "learning_rate": 8.800000000000001e-05, "loss": 0.03544958680868149, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 1.03609, "step": 23, "tokens/total": 780240, "tokens/train_per_sec_per_gpu": 29.3, "tokens/trainable": 10380} +{"epoch": 0.09375, "grad_norm": 2.510955572128296, "learning_rate": 9.200000000000001e-05, "loss": 0.04666295647621155, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.5, "memory/max_allocated (GiB)": 37.5, "ppl": 1.04777, "step": 24, "tokens/total": 811344, "tokens/train_per_sec_per_gpu": 31.91, "tokens/trainable": 10832} +{"epoch": 0.09765625, "grad_norm": 3.7553017139434814, "learning_rate": 9.6e-05, "loss": 0.07907476276159286, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.66, "memory/max_allocated (GiB)": 36.66, "ppl": 1.08229, "step": 25, "tokens/total": 843728, "tokens/train_per_sec_per_gpu": 27.19, "tokens/trainable": 11293} +{"epoch": 0.1015625, "grad_norm": 3.4176807403564453, "learning_rate": 0.0001, "loss": 0.08034460246562958, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.08366, "step": 26, "tokens/total": 876560, "tokens/train_per_sec_per_gpu": 23.99, "tokens/trainable": 11700} +{"epoch": 0.10546875, "grad_norm": 1.7892820835113525, "learning_rate": 9.99990636831587e-05, "loss": 0.037406228482723236, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.03811, "step": 27, "tokens/total": 909488, "tokens/train_per_sec_per_gpu": 29.94, "tokens/trainable": 12167} +{"epoch": 0.109375, "grad_norm": 3.157078504562378, "learning_rate": 9.999625477159879e-05, "loss": 0.03901427984237671, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.03979, "step": 28, "tokens/total": 947840, "tokens/train_per_sec_per_gpu": 22.55, "tokens/trainable": 12621} +{"epoch": 0.11328125, "grad_norm": 0.895730197429657, "learning_rate": 9.999157338221051e-05, "loss": 0.04534510150551796, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.04639, "step": 29, "tokens/total": 982544, "tokens/train_per_sec_per_gpu": 26.04, "tokens/trainable": 13078} +{"epoch": 0.1171875, "grad_norm": 1.6290620565414429, "learning_rate": 9.998501970980562e-05, "loss": 0.028353575617074966, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.02876, "step": 30, "tokens/total": 1014144, "tokens/train_per_sec_per_gpu": 31.23, "tokens/trainable": 13549} +{"epoch": 0.12109375, "grad_norm": 0.5491243600845337, "learning_rate": 9.997659402710915e-05, "loss": 0.020423155277967453, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.02063, "step": 31, "tokens/total": 1048976, "tokens/train_per_sec_per_gpu": 25.95, "tokens/trainable": 13989} +{"epoch": 0.125, "grad_norm": 4.188231468200684, "learning_rate": 9.996629668474818e-05, "loss": 0.04012877121567726, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.04094, "step": 32, "tokens/total": 1083728, "tokens/train_per_sec_per_gpu": 29.47, "tokens/trainable": 14490} +{"epoch": 0.12890625, "grad_norm": 1.0532214641571045, "learning_rate": 9.995412811123711e-05, "loss": 0.033112671226263046, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 38.97, "memory/max_allocated (GiB)": 38.97, "ppl": 1.03367, "step": 33, "tokens/total": 1119664, "tokens/train_per_sec_per_gpu": 24.65, "tokens/trainable": 14929} +{"epoch": 0.1328125, "grad_norm": 0.7519180178642273, "learning_rate": 9.994008881295999e-05, "loss": 0.029761487618088722, "memory/device_reserved (GiB)": 39.91, "memory/max_active (GiB)": 36.87, "memory/max_allocated (GiB)": 36.87, "ppl": 1.03021, "step": 34, "tokens/total": 1152768, "tokens/train_per_sec_per_gpu": 25.45, "tokens/trainable": 15383} +{"epoch": 0.13671875, "grad_norm": 0.6577471494674683, "learning_rate": 9.992417937414932e-05, "loss": 0.027572717517614365, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 1.02796, "step": 35, "tokens/total": 1186736, "tokens/train_per_sec_per_gpu": 29.75, "tokens/trainable": 15884} +{"epoch": 0.140625, "grad_norm": 1.1393598318099976, "learning_rate": 9.99064004568618e-05, "loss": 0.0191852618008852, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.01937, "step": 36, "tokens/total": 1222352, "tokens/train_per_sec_per_gpu": 25.61, "tokens/trainable": 16343} +{"epoch": 0.14453125, "grad_norm": 2.2870519161224365, "learning_rate": 9.988675280095074e-05, "loss": 0.034917593002319336, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.03553, "step": 37, "tokens/total": 1256336, "tokens/train_per_sec_per_gpu": 24.87, "tokens/trainable": 16799} +{"epoch": 0.1484375, "grad_norm": 0.45678699016571045, "learning_rate": 9.986523722403528e-05, "loss": 0.018083985894918442, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.01825, "step": 38, "tokens/total": 1288304, "tokens/train_per_sec_per_gpu": 26.6, "tokens/trainable": 17256} +{"epoch": 0.15234375, "grad_norm": 0.8309405446052551, "learning_rate": 9.984185462146642e-05, "loss": 0.024911874905228615, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.41, "memory/max_allocated (GiB)": 37.41, "ppl": 1.02522, "step": 39, "tokens/total": 1323440, "tokens/train_per_sec_per_gpu": 25.56, "tokens/trainable": 17701} +{"epoch": 0.15625, "grad_norm": 1.593706488609314, "learning_rate": 9.98166059662897e-05, "loss": 0.029771404340863228, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 35.95, "memory/max_allocated (GiB)": 35.95, "ppl": 1.03022, "step": 40, "tokens/total": 1354576, "tokens/train_per_sec_per_gpu": 29.25, "tokens/trainable": 18168} +{"epoch": 0.16015625, "grad_norm": 1.795067548751831, "learning_rate": 9.978949230920472e-05, "loss": 0.015175780281424522, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.01529, "step": 41, "tokens/total": 1389264, "tokens/train_per_sec_per_gpu": 25.0, "tokens/trainable": 18622} +{"epoch": 0.1640625, "grad_norm": 0.8951998949050903, "learning_rate": 9.976051477852141e-05, "loss": 0.03753477707505226, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.03825, "step": 42, "tokens/total": 1422496, "tokens/train_per_sec_per_gpu": 26.31, "tokens/trainable": 19063} +{"epoch": 0.16796875, "grad_norm": 0.8414852023124695, "learning_rate": 9.972967458011312e-05, "loss": 0.03830161690711975, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 38.91, "memory/max_allocated (GiB)": 38.91, "ppl": 1.03904, "step": 43, "tokens/total": 1457856, "tokens/train_per_sec_per_gpu": 27.41, "tokens/trainable": 19536} +{"epoch": 0.171875, "grad_norm": 1.5204378366470337, "learning_rate": 9.96969729973664e-05, "loss": 0.026235431432724, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.02658, "step": 44, "tokens/total": 1496720, "tokens/train_per_sec_per_gpu": 23.51, "tokens/trainable": 20001} +{"epoch": 0.17578125, "grad_norm": 0.5373602509498596, "learning_rate": 9.966241139112754e-05, "loss": 0.018041539937257767, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.1, "memory/max_allocated (GiB)": 38.1, "ppl": 1.01821, "step": 45, "tokens/total": 1530416, "tokens/train_per_sec_per_gpu": 25.69, "tokens/trainable": 20442} +{"epoch": 0.1796875, "grad_norm": 0.8826891183853149, "learning_rate": 9.96259911996461e-05, "loss": 0.03387293219566345, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.03445, "step": 46, "tokens/total": 1564576, "tokens/train_per_sec_per_gpu": 25.19, "tokens/trainable": 20874} +{"epoch": 0.18359375, "grad_norm": 1.5664622783660889, "learning_rate": 9.958771393851491e-05, "loss": 0.038506701588630676, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 36.49, "memory/max_allocated (GiB)": 36.49, "ppl": 1.03926, "step": 47, "tokens/total": 1596032, "tokens/train_per_sec_per_gpu": 24.98, "tokens/trainable": 21315} +{"epoch": 0.1875, "grad_norm": 0.8389865159988403, "learning_rate": 9.954758120060702e-05, "loss": 0.036861710250377655, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.03755, "step": 48, "tokens/total": 1632304, "tokens/train_per_sec_per_gpu": 24.37, "tokens/trainable": 21775} +{"epoch": 0.19140625, "grad_norm": 1.1031913757324219, "learning_rate": 9.950559465600948e-05, "loss": 0.023297902196645737, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 1.02357, "step": 49, "tokens/total": 1666560, "tokens/train_per_sec_per_gpu": 26.05, "tokens/trainable": 22215} +{"epoch": 0.1953125, "grad_norm": 0.5379480719566345, "learning_rate": 9.946175605195379e-05, "loss": 0.026179324835538864, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.02653, "step": 50, "tokens/total": 1706112, "tokens/train_per_sec_per_gpu": 21.75, "tokens/trainable": 22700} +{"epoch": 0.19921875, "grad_norm": 0.48291707038879395, "learning_rate": 9.941606721274322e-05, "loss": 0.017165878787636757, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.01731, "step": 51, "tokens/total": 1740320, "tokens/train_per_sec_per_gpu": 22.74, "tokens/trainable": 23135} +{"epoch": 0.203125, "grad_norm": 0.8293343782424927, "learning_rate": 9.936853003967685e-05, "loss": 0.02692706324160099, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.02729, "step": 52, "tokens/total": 1777552, "tokens/train_per_sec_per_gpu": 24.37, "tokens/trainable": 23609} +{"epoch": 0.20703125, "grad_norm": 4.050737380981445, "learning_rate": 9.93191465109705e-05, "loss": 0.02117035910487175, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.0214, "step": 53, "tokens/total": 1814192, "tokens/train_per_sec_per_gpu": 23.56, "tokens/trainable": 24042} +{"epoch": 0.2109375, "grad_norm": 1.6284639835357666, "learning_rate": 9.926791868167438e-05, "loss": 0.05734633654356003, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.05902, "step": 54, "tokens/total": 1850688, "tokens/train_per_sec_per_gpu": 23.92, "tokens/trainable": 24468} +{"epoch": 0.21484375, "grad_norm": 1.7026265859603882, "learning_rate": 9.921484868358753e-05, "loss": 0.03216367959976196, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 1.03269, "step": 55, "tokens/total": 1886704, "tokens/train_per_sec_per_gpu": 24.15, "tokens/trainable": 24924} +{"epoch": 0.21875, "grad_norm": 0.8212124705314636, "learning_rate": 9.915993872516924e-05, "loss": 0.011880354955792427, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.01195, "step": 56, "tokens/total": 1922528, "tokens/train_per_sec_per_gpu": 22.95, "tokens/trainable": 25368} +{"epoch": 0.22265625, "grad_norm": 1.4825944900512695, "learning_rate": 9.9103191091447e-05, "loss": 0.019092723727226257, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.01928, "step": 57, "tokens/total": 1954592, "tokens/train_per_sec_per_gpu": 26.14, "tokens/trainable": 25807} +{"epoch": 0.2265625, "grad_norm": 0.6656453013420105, "learning_rate": 9.904460814392147e-05, "loss": 0.018974997103214264, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.01916, "step": 58, "tokens/total": 1993968, "tokens/train_per_sec_per_gpu": 22.41, "tokens/trainable": 26264} +{"epoch": 0.23046875, "grad_norm": 0.8849591016769409, "learning_rate": 9.898419232046825e-05, "loss": 0.03229174017906189, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.03282, "step": 59, "tokens/total": 2030672, "tokens/train_per_sec_per_gpu": 26.95, "tokens/trainable": 26767} +{"epoch": 0.234375, "grad_norm": 0.5980359315872192, "learning_rate": 9.892194613523633e-05, "loss": 0.02004898153245449, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.26, "memory/max_allocated (GiB)": 38.26, "ppl": 1.02025, "step": 60, "tokens/total": 2066528, "tokens/train_per_sec_per_gpu": 24.05, "tokens/trainable": 27226} +{"epoch": 0.23828125, "grad_norm": 0.45517316460609436, "learning_rate": 9.885787217854357e-05, "loss": 0.015413784421980381, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.01553, "step": 61, "tokens/total": 2100112, "tokens/train_per_sec_per_gpu": 26.19, "tokens/trainable": 27685} +{"epoch": 0.2421875, "grad_norm": 0.386525958776474, "learning_rate": 9.879197311676887e-05, "loss": 0.013475582003593445, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.01357, "step": 62, "tokens/total": 2136160, "tokens/train_per_sec_per_gpu": 24.03, "tokens/trainable": 28125} +{"epoch": 0.24609375, "grad_norm": 0.8157568573951721, "learning_rate": 9.872425169224113e-05, "loss": 0.02790994569659233, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.0283, "step": 63, "tokens/total": 2169952, "tokens/train_per_sec_per_gpu": 29.49, "tokens/trainable": 28623} +{"epoch": 0.25, "grad_norm": 0.9888743758201599, "learning_rate": 9.865471072312528e-05, "loss": 0.02662654034793377, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.14, "memory/max_allocated (GiB)": 38.14, "ppl": 1.02698, "step": 64, "tokens/total": 2206416, "tokens/train_per_sec_per_gpu": 25.69, "tokens/trainable": 29108} +{"epoch": 0.25390625, "grad_norm": 0.35826432704925537, "learning_rate": 9.858335310330492e-05, "loss": 0.00826322566717863, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.0083, "step": 65, "tokens/total": 2238592, "tokens/train_per_sec_per_gpu": 26.24, "tokens/trainable": 29570} +{"epoch": 0.2578125, "grad_norm": 0.6877807974815369, "learning_rate": 9.851018180226185e-05, "loss": 0.03296053782105446, "memory/device_reserved (GiB)": 40.31, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.03351, "step": 66, "tokens/total": 2277136, "tokens/train_per_sec_per_gpu": 21.44, "tokens/trainable": 30011} +{"epoch": 0.26171875, "grad_norm": 0.6653009653091431, "learning_rate": 9.843519986495259e-05, "loss": 0.02185806818306446, "memory/device_reserved (GiB)": 41.07, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.0221, "step": 67, "tokens/total": 2311872, "tokens/train_per_sec_per_gpu": 25.15, "tokens/trainable": 30446} +{"epoch": 0.265625, "grad_norm": 0.6609723567962646, "learning_rate": 9.835841041168162e-05, "loss": 0.017631251364946365, "memory/device_reserved (GiB)": 41.32, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.01779, "step": 68, "tokens/total": 2349008, "tokens/train_per_sec_per_gpu": 24.44, "tokens/trainable": 30913} +{"epoch": 0.26953125, "grad_norm": 0.63666170835495, "learning_rate": 9.82798166379715e-05, "loss": 0.013627124018967152, "memory/device_reserved (GiB)": 41.32, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.01372, "step": 69, "tokens/total": 2382368, "tokens/train_per_sec_per_gpu": 28.29, "tokens/trainable": 31403} +{"epoch": 0.2734375, "grad_norm": 0.5714080929756165, "learning_rate": 9.819942181443002e-05, "loss": 0.010437704622745514, "memory/device_reserved (GiB)": 41.32, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.01049, "step": 70, "tokens/total": 2419648, "tokens/train_per_sec_per_gpu": 21.66, "tokens/trainable": 31844} +{"epoch": 0.27734375, "grad_norm": 0.4188081622123718, "learning_rate": 9.811722928661392e-05, "loss": 0.007994343526661396, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 35.81, "memory/max_allocated (GiB)": 35.81, "ppl": 1.00803, "step": 71, "tokens/total": 2448576, "tokens/train_per_sec_per_gpu": 26.31, "tokens/trainable": 32236} +{"epoch": 0.28125, "grad_norm": 0.10211779177188873, "learning_rate": 9.803324247488975e-05, "loss": 0.0010487200925126672, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00105, "step": 72, "tokens/total": 2484240, "tokens/train_per_sec_per_gpu": 23.84, "tokens/trainable": 32688} +{"epoch": 0.28515625, "grad_norm": 1.6783519983291626, "learning_rate": 9.794746487429161e-05, "loss": 0.003331308253109455, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 36.88, "memory/max_allocated (GiB)": 36.88, "ppl": 1.00334, "step": 73, "tokens/total": 2517184, "tokens/train_per_sec_per_gpu": 26.93, "tokens/trainable": 33131} +{"epoch": 0.2890625, "grad_norm": 2.621304988861084, "learning_rate": 9.785990005437554e-05, "loss": 0.03126700967550278, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.03176, "step": 74, "tokens/total": 2553056, "tokens/train_per_sec_per_gpu": 25.16, "tokens/trainable": 33592} +{"epoch": 0.29296875, "grad_norm": 0.6248510479927063, "learning_rate": 9.777055165907117e-05, "loss": 0.02463417872786522, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.02494, "step": 75, "tokens/total": 2585584, "tokens/train_per_sec_per_gpu": 28.79, "tokens/trainable": 34023} +{"epoch": 0.296875, "grad_norm": 0.7905248403549194, "learning_rate": 9.767942340652993e-05, "loss": 0.009850251488387585, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.07, "memory/max_allocated (GiB)": 38.07, "ppl": 1.0099, "step": 76, "tokens/total": 2620448, "tokens/train_per_sec_per_gpu": 22.34, "tokens/trainable": 34449} +{"epoch": 0.30078125, "grad_norm": 0.9659206867218018, "learning_rate": 9.758651908897035e-05, "loss": 0.02391723170876503, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.02421, "step": 77, "tokens/total": 2653840, "tokens/train_per_sec_per_gpu": 27.96, "tokens/trainable": 34894} +{"epoch": 0.3046875, "grad_norm": 0.8672435283660889, "learning_rate": 9.749184257252033e-05, "loss": 0.019592415541410446, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.01979, "step": 78, "tokens/total": 2688112, "tokens/train_per_sec_per_gpu": 24.5, "tokens/trainable": 35334} +{"epoch": 0.30859375, "grad_norm": 0.6668999195098877, "learning_rate": 9.739539779705614e-05, "loss": 0.010249435901641846, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.0103, "step": 79, "tokens/total": 2726384, "tokens/train_per_sec_per_gpu": 24.5, "tokens/trainable": 35829} +{"epoch": 0.3125, "grad_norm": 0.7077988386154175, "learning_rate": 9.729718877603861e-05, "loss": 0.019492177292704582, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.01968, "step": 80, "tokens/total": 2760176, "tokens/train_per_sec_per_gpu": 26.71, "tokens/trainable": 36260} +{"epoch": 0.31640625, "grad_norm": 0.9253591895103455, "learning_rate": 9.719721959634592e-05, "loss": 0.030538948252797127, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.03101, "step": 81, "tokens/total": 2795392, "tokens/train_per_sec_per_gpu": 23.23, "tokens/trainable": 36697} +{"epoch": 0.3203125, "grad_norm": 0.8271691799163818, "learning_rate": 9.709549441810375e-05, "loss": 0.012238087132573128, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.55, "memory/max_allocated (GiB)": 39.55, "ppl": 1.01231, "step": 82, "tokens/total": 2833008, "tokens/train_per_sec_per_gpu": 26.95, "tokens/trainable": 37181} +{"epoch": 0.32421875, "grad_norm": 0.9702895283699036, "learning_rate": 9.699201747451195e-05, "loss": 0.02239469438791275, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.02265, "step": 83, "tokens/total": 2865600, "tokens/train_per_sec_per_gpu": 24.97, "tokens/trainable": 37629} +{"epoch": 0.328125, "grad_norm": 0.36551907658576965, "learning_rate": 9.688679307166854e-05, "loss": 0.01088052149862051, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.01094, "step": 84, "tokens/total": 2898400, "tokens/train_per_sec_per_gpu": 28.29, "tokens/trainable": 38092} +{"epoch": 0.33203125, "grad_norm": 0.611868143081665, "learning_rate": 9.677982558839042e-05, "loss": 0.038635291159152985, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.03939, "step": 85, "tokens/total": 2931696, "tokens/train_per_sec_per_gpu": 27.07, "tokens/trainable": 38571} +{"epoch": 0.3359375, "grad_norm": 0.45590198040008545, "learning_rate": 9.66711194760312e-05, "loss": 0.015310714021325111, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.01543, "step": 86, "tokens/total": 2969104, "tokens/train_per_sec_per_gpu": 25.51, "tokens/trainable": 39059} +{"epoch": 0.33984375, "grad_norm": 0.5998668670654297, "learning_rate": 9.656067925829593e-05, "loss": 0.019747791811823845, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.01994, "step": 87, "tokens/total": 2999856, "tokens/train_per_sec_per_gpu": 31.24, "tokens/trainable": 39519} +{"epoch": 0.34375, "grad_norm": 0.45338574051856995, "learning_rate": 9.644850953105288e-05, "loss": 0.02674228511750698, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 35.78, "memory/max_allocated (GiB)": 35.78, "ppl": 1.0271, "step": 88, "tokens/total": 3030208, "tokens/train_per_sec_per_gpu": 26.12, "tokens/trainable": 39961} +{"epoch": 0.34765625, "grad_norm": 0.32234156131744385, "learning_rate": 9.633461496214225e-05, "loss": 0.01400056667625904, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.0141, "step": 89, "tokens/total": 3062912, "tokens/train_per_sec_per_gpu": 26.53, "tokens/trainable": 40431} +{"epoch": 0.3515625, "grad_norm": 0.4403051435947418, "learning_rate": 9.621900029118195e-05, "loss": 0.03099983185529709, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.04, "memory/max_allocated (GiB)": 36.04, "ppl": 1.03149, "step": 90, "tokens/total": 3092592, "tokens/train_per_sec_per_gpu": 25.58, "tokens/trainable": 40854} +{"epoch": 0.35546875, "grad_norm": 0.1461208313703537, "learning_rate": 9.610167032937036e-05, "loss": 0.005638827104121447, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00565, "step": 91, "tokens/total": 3130880, "tokens/train_per_sec_per_gpu": 26.12, "tokens/trainable": 41373} +{"epoch": 0.359375, "grad_norm": 0.36949819326400757, "learning_rate": 9.598262995928611e-05, "loss": 0.010426975786685944, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.01048, "step": 92, "tokens/total": 3162080, "tokens/train_per_sec_per_gpu": 30.19, "tokens/trainable": 41842} +{"epoch": 0.36328125, "grad_norm": 0.34538382291793823, "learning_rate": 9.586188413468492e-05, "loss": 0.005814875941723585, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00583, "step": 93, "tokens/total": 3200272, "tokens/train_per_sec_per_gpu": 22.94, "tokens/trainable": 42288} +{"epoch": 0.3671875, "grad_norm": 0.33228692412376404, "learning_rate": 9.57394378802934e-05, "loss": 0.011815130710601807, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.01189, "step": 94, "tokens/total": 3233584, "tokens/train_per_sec_per_gpu": 27.46, "tokens/trainable": 42740} +{"epoch": 0.37109375, "grad_norm": 0.6180556416511536, "learning_rate": 9.56152962916e-05, "loss": 0.024027403444051743, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.26, "memory/max_allocated (GiB)": 36.26, "ppl": 1.02432, "step": 95, "tokens/total": 3264144, "tokens/train_per_sec_per_gpu": 26.75, "tokens/trainable": 43138} +{"epoch": 0.375, "grad_norm": 0.16514478623867035, "learning_rate": 9.548946453464296e-05, "loss": 0.002433347748592496, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.00244, "step": 96, "tokens/total": 3298240, "tokens/train_per_sec_per_gpu": 26.87, "tokens/trainable": 43623} +{"epoch": 0.37890625, "grad_norm": 0.1995823234319687, "learning_rate": 9.53619478457953e-05, "loss": 0.0048777032643556595, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00489, "step": 97, "tokens/total": 3333136, "tokens/train_per_sec_per_gpu": 24.05, "tokens/trainable": 44079} +{"epoch": 0.3828125, "grad_norm": 0.3394344449043274, "learning_rate": 9.523275153154695e-05, "loss": 0.00867878831923008, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.00872, "step": 98, "tokens/total": 3365536, "tokens/train_per_sec_per_gpu": 25.21, "tokens/trainable": 44529} +{"epoch": 0.38671875, "grad_norm": 1.223002552986145, "learning_rate": 9.51018809682839e-05, "loss": 0.050366126000881195, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.05166, "step": 99, "tokens/total": 3403200, "tokens/train_per_sec_per_gpu": 25.65, "tokens/trainable": 44976} +{"epoch": 0.390625, "grad_norm": 0.5545073747634888, "learning_rate": 9.49693416020645e-05, "loss": 0.026321809738874435, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.53, "memory/max_allocated (GiB)": 38.53, "ppl": 1.02667, "step": 100, "tokens/total": 3440944, "tokens/train_per_sec_per_gpu": 24.56, "tokens/trainable": 45429} +{"epoch": 0.39453125, "grad_norm": 0.10010835528373718, "learning_rate": 9.483513894839276e-05, "loss": 0.0020351395942270756, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.00204, "step": 101, "tokens/total": 3474992, "tokens/train_per_sec_per_gpu": 25.31, "tokens/trainable": 45876} +{"epoch": 0.3984375, "grad_norm": 0.8154366612434387, "learning_rate": 9.469927859198888e-05, "loss": 0.026911310851573944, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.02728, "step": 102, "tokens/total": 3510912, "tokens/train_per_sec_per_gpu": 29.07, "tokens/trainable": 46363} +{"epoch": 0.40234375, "grad_norm": 0.33355391025543213, "learning_rate": 9.456176618655689e-05, "loss": 0.023176899179816246, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.47, "memory/max_allocated (GiB)": 36.47, "ppl": 1.02345, "step": 103, "tokens/total": 3542960, "tokens/train_per_sec_per_gpu": 27.51, "tokens/trainable": 46810} +{"epoch": 0.40625, "grad_norm": 1.2125030755996704, "learning_rate": 9.442260745454927e-05, "loss": 0.034321606159210205, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.03492, "step": 104, "tokens/total": 3578336, "tokens/train_per_sec_per_gpu": 25.91, "tokens/trainable": 47287} +{"epoch": 0.41015625, "grad_norm": 0.3602006733417511, "learning_rate": 9.428180818692884e-05, "loss": 0.014498098753392696, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.45, "memory/max_allocated (GiB)": 38.45, "ppl": 1.0146, "step": 105, "tokens/total": 3614496, "tokens/train_per_sec_per_gpu": 24.74, "tokens/trainable": 47788} +{"epoch": 0.4140625, "grad_norm": 0.20653723180294037, "learning_rate": 9.413937424292791e-05, "loss": 0.011586330831050873, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.01165, "step": 106, "tokens/total": 3653632, "tokens/train_per_sec_per_gpu": 21.68, "tokens/trainable": 48243} +{"epoch": 0.41796875, "grad_norm": 0.3297898769378662, "learning_rate": 9.399531154980424e-05, "loss": 0.017013216391205788, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 1.01716, "step": 107, "tokens/total": 3690208, "tokens/train_per_sec_per_gpu": 24.1, "tokens/trainable": 48692} +{"epoch": 0.421875, "grad_norm": 0.4060016870498657, "learning_rate": 9.384962610259455e-05, "loss": 0.02226952835917473, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.02252, "step": 108, "tokens/total": 3724864, "tokens/train_per_sec_per_gpu": 24.18, "tokens/trainable": 49142} +{"epoch": 0.42578125, "grad_norm": 0.14151069521903992, "learning_rate": 9.370232396386494e-05, "loss": 0.004946930333971977, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.22, "memory/max_allocated (GiB)": 37.22, "ppl": 1.00496, "step": 109, "tokens/total": 3757952, "tokens/train_per_sec_per_gpu": 26.41, "tokens/trainable": 49583} +{"epoch": 0.4296875, "grad_norm": 0.5247214436531067, "learning_rate": 9.355341126345868e-05, "loss": 0.01874198019504547, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.01892, "step": 110, "tokens/total": 3793232, "tokens/train_per_sec_per_gpu": 25.47, "tokens/trainable": 50055} +{"epoch": 0.43359375, "grad_norm": 0.6906198263168335, "learning_rate": 9.340289419824107e-05, "loss": 0.031218217685818672, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.03171, "step": 111, "tokens/total": 3825552, "tokens/train_per_sec_per_gpu": 28.08, "tokens/trainable": 50500} +{"epoch": 0.4375, "grad_norm": 0.2861274778842926, "learning_rate": 9.325077903184159e-05, "loss": 0.006556412670761347, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00658, "step": 112, "tokens/total": 3860544, "tokens/train_per_sec_per_gpu": 23.91, "tokens/trainable": 50950} +{"epoch": 0.44140625, "grad_norm": 0.1811281442642212, "learning_rate": 9.30970720943932e-05, "loss": 0.0025682621635496616, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.15, "memory/max_allocated (GiB)": 38.15, "ppl": 1.00257, "step": 113, "tokens/total": 3895552, "tokens/train_per_sec_per_gpu": 24.48, "tokens/trainable": 51389} +{"epoch": 0.4453125, "grad_norm": 0.50982266664505, "learning_rate": 9.2941779782269e-05, "loss": 0.0030761384405195713, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00308, "step": 114, "tokens/total": 3929152, "tokens/train_per_sec_per_gpu": 25.44, "tokens/trainable": 51862} +{"epoch": 0.44921875, "grad_norm": 2.7105467319488525, "learning_rate": 9.278490855781596e-05, "loss": 0.023150235414505005, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.24, "memory/max_allocated (GiB)": 37.24, "ppl": 1.02342, "step": 115, "tokens/total": 3961472, "tokens/train_per_sec_per_gpu": 29.89, "tokens/trainable": 52301} +{"epoch": 0.453125, "grad_norm": 0.6962517499923706, "learning_rate": 9.262646494908604e-05, "loss": 0.008996543474495411, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00904, "step": 116, "tokens/total": 4000128, "tokens/train_per_sec_per_gpu": 23.65, "tokens/trainable": 52755} +{"epoch": 0.45703125, "grad_norm": 0.8992996215820312, "learning_rate": 9.246645554956457e-05, "loss": 0.015424896031618118, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.01554, "step": 117, "tokens/total": 4035152, "tokens/train_per_sec_per_gpu": 24.98, "tokens/trainable": 53224} +{"epoch": 0.4609375, "grad_norm": 1.4473284482955933, "learning_rate": 9.230488701789578e-05, "loss": 0.015202001668512821, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.01532, "step": 118, "tokens/total": 4072208, "tokens/train_per_sec_per_gpu": 23.45, "tokens/trainable": 53666} +{"epoch": 0.46484375, "grad_norm": 0.5979090929031372, "learning_rate": 9.214176607760577e-05, "loss": 0.0077283973805606365, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.00776, "step": 119, "tokens/total": 4110464, "tokens/train_per_sec_per_gpu": 24.04, "tokens/trainable": 54129} +{"epoch": 0.46875, "grad_norm": 1.6406457424163818, "learning_rate": 9.197709951682268e-05, "loss": 0.015193537808954716, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.01531, "step": 120, "tokens/total": 4142896, "tokens/train_per_sec_per_gpu": 28.9, "tokens/trainable": 54587} +{"epoch": 0.47265625, "grad_norm": 0.6513875722885132, "learning_rate": 9.181089418799428e-05, "loss": 0.018397722393274307, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.01857, "step": 121, "tokens/total": 4179360, "tokens/train_per_sec_per_gpu": 27.19, "tokens/trainable": 55070} +{"epoch": 0.4765625, "grad_norm": 0.7501571178436279, "learning_rate": 9.164315700760271e-05, "loss": 0.025784166529774666, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.02612, "step": 122, "tokens/total": 4215040, "tokens/train_per_sec_per_gpu": 23.18, "tokens/trainable": 55494} +{"epoch": 0.48046875, "grad_norm": 0.5778821706771851, "learning_rate": 9.147389495587671e-05, "loss": 0.025377852842211723, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.0257, "step": 123, "tokens/total": 4246448, "tokens/train_per_sec_per_gpu": 30.81, "tokens/trainable": 55940} +{"epoch": 0.484375, "grad_norm": 0.4644779860973358, "learning_rate": 9.130311507650116e-05, "loss": 0.013209076598286629, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.0133, "step": 124, "tokens/total": 4278832, "tokens/train_per_sec_per_gpu": 26.75, "tokens/trainable": 56404} +{"epoch": 0.48828125, "grad_norm": 0.2759820818901062, "learning_rate": 9.113082447632394e-05, "loss": 0.005139063578099012, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00515, "step": 125, "tokens/total": 4310896, "tokens/train_per_sec_per_gpu": 32.79, "tokens/trainable": 56878} +{"epoch": 0.4921875, "grad_norm": 0.4299395978450775, "learning_rate": 9.09570303250602e-05, "loss": 0.015467770397663116, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.01559, "step": 126, "tokens/total": 4342912, "tokens/train_per_sec_per_gpu": 30.26, "tokens/trainable": 57362} +{"epoch": 0.49609375, "grad_norm": 0.2274901419878006, "learning_rate": 9.078173985499394e-05, "loss": 0.010870872065424919, "memory/device_reserved (GiB)": 42.05, "memory/max_active (GiB)": 35.95, "memory/max_allocated (GiB)": 35.95, "ppl": 1.01093, "step": 127, "tokens/total": 4373312, "tokens/train_per_sec_per_gpu": 28.09, "tokens/trainable": 57792} +{"epoch": 0.5, "grad_norm": 0.8545277118682861, "learning_rate": 9.060496036067713e-05, "loss": 0.029374800622463226, "memory/device_reserved (GiB)": 42.05, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.02981, "step": 128, "tokens/total": 4410816, "tokens/train_per_sec_per_gpu": 22.23, "tokens/trainable": 58216} +{"epoch": 0.50390625, "grad_norm": 0.30495303869247437, "learning_rate": 9.042669919862615e-05, "loss": 0.010721537284553051, "memory/device_reserved (GiB)": 42.05, "memory/max_active (GiB)": 36.77, "memory/max_allocated (GiB)": 36.77, "ppl": 1.01078, "step": 129, "tokens/total": 4442704, "tokens/train_per_sec_per_gpu": 29.68, "tokens/trainable": 58680} +{"epoch": 0.5078125, "grad_norm": 0.3886556625366211, "learning_rate": 9.024696378701557e-05, "loss": 0.014975102618336678, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 39.48, "memory/max_allocated (GiB)": 39.48, "ppl": 1.01509, "step": 130, "tokens/total": 4478384, "tokens/train_per_sec_per_gpu": 23.46, "tokens/trainable": 59113} +{"epoch": 0.51171875, "grad_norm": 0.9448239207267761, "learning_rate": 9.006576160536948e-05, "loss": 0.01346011832356453, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.01355, "step": 131, "tokens/total": 4513328, "tokens/train_per_sec_per_gpu": 24.03, "tokens/trainable": 59562} +{"epoch": 0.515625, "grad_norm": 0.537383496761322, "learning_rate": 8.988310019425035e-05, "loss": 0.016080060973763466, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.01621, "step": 132, "tokens/total": 4550832, "tokens/train_per_sec_per_gpu": 23.85, "tokens/trainable": 60006} +{"epoch": 0.51953125, "grad_norm": 0.17739956080913544, "learning_rate": 8.969898715494506e-05, "loss": 0.004678180441260338, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00469, "step": 133, "tokens/total": 4587968, "tokens/train_per_sec_per_gpu": 25.41, "tokens/trainable": 60487} +{"epoch": 0.5234375, "grad_norm": 1.065868616104126, "learning_rate": 8.951343014914869e-05, "loss": 0.013294421136379242, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.01338, "step": 134, "tokens/total": 4624544, "tokens/train_per_sec_per_gpu": 23.28, "tokens/trainable": 60953} +{"epoch": 0.52734375, "grad_norm": 0.18924979865550995, "learning_rate": 8.932643689864568e-05, "loss": 0.0028356071561574936, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00284, "step": 135, "tokens/total": 4661296, "tokens/train_per_sec_per_gpu": 26.94, "tokens/trainable": 61440} +{"epoch": 0.53125, "grad_norm": 0.49770089983940125, "learning_rate": 8.913801518498845e-05, "loss": 0.008932811208069324, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.00897, "step": 136, "tokens/total": 4700128, "tokens/train_per_sec_per_gpu": 21.05, "tokens/trainable": 61901} +{"epoch": 0.53515625, "grad_norm": 0.24970559775829315, "learning_rate": 8.894817284917364e-05, "loss": 0.005257212556898594, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00527, "step": 137, "tokens/total": 4736912, "tokens/train_per_sec_per_gpu": 22.49, "tokens/trainable": 62355} +{"epoch": 0.5390625, "grad_norm": 0.3199540078639984, "learning_rate": 8.875691779131569e-05, "loss": 0.0027339570224285126, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00274, "step": 138, "tokens/total": 4774192, "tokens/train_per_sec_per_gpu": 20.51, "tokens/trainable": 62786} +{"epoch": 0.54296875, "grad_norm": 0.3428623080253601, "learning_rate": 8.856425797031829e-05, "loss": 0.005981665104627609, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.006, "step": 139, "tokens/total": 4809344, "tokens/train_per_sec_per_gpu": 25.48, "tokens/trainable": 63228} +{"epoch": 0.546875, "grad_norm": 1.3176239728927612, "learning_rate": 8.837020140354295e-05, "loss": 0.014284629374742508, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.01439, "step": 140, "tokens/total": 4845712, "tokens/train_per_sec_per_gpu": 26.55, "tokens/trainable": 63693} +{"epoch": 0.55078125, "grad_norm": 0.4285449683666229, "learning_rate": 8.817475616647554e-05, "loss": 0.0056700496934354305, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.00569, "step": 141, "tokens/total": 4880368, "tokens/train_per_sec_per_gpu": 26.23, "tokens/trainable": 64159} +{"epoch": 0.5546875, "grad_norm": 0.49727562069892883, "learning_rate": 8.797793039239017e-05, "loss": 0.007494611199945211, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00752, "step": 142, "tokens/total": 4915424, "tokens/train_per_sec_per_gpu": 25.73, "tokens/trainable": 64608} +{"epoch": 0.55859375, "grad_norm": 0.9036321640014648, "learning_rate": 8.777973227201069e-05, "loss": 0.012326923198997974, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.0124, "step": 143, "tokens/total": 4951744, "tokens/train_per_sec_per_gpu": 27.48, "tokens/trainable": 65117} +{"epoch": 0.5625, "grad_norm": 0.2860254645347595, "learning_rate": 8.758017005316988e-05, "loss": 0.002571955556049943, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.00258, "step": 144, "tokens/total": 4985152, "tokens/train_per_sec_per_gpu": 27.95, "tokens/trainable": 65542} +{"epoch": 0.56640625, "grad_norm": 0.6374468207359314, "learning_rate": 8.737925204046629e-05, "loss": 0.012933972291648388, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.01302, "step": 145, "tokens/total": 5019936, "tokens/train_per_sec_per_gpu": 23.76, "tokens/trainable": 65992} +{"epoch": 0.5703125, "grad_norm": 0.4892997145652771, "learning_rate": 8.717698659491851e-05, "loss": 0.008109782822430134, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00814, "step": 146, "tokens/total": 5057024, "tokens/train_per_sec_per_gpu": 22.13, "tokens/trainable": 66449} +{"epoch": 0.57421875, "grad_norm": 0.6443142890930176, "learning_rate": 8.697338213361735e-05, "loss": 0.019393490627408028, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.01958, "step": 147, "tokens/total": 5088320, "tokens/train_per_sec_per_gpu": 25.75, "tokens/trainable": 66897} +{"epoch": 0.578125, "grad_norm": 0.4361960291862488, "learning_rate": 8.676844712937552e-05, "loss": 0.0054780240170657635, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.39, "memory/max_allocated (GiB)": 39.39, "ppl": 1.00549, "step": 148, "tokens/total": 5128064, "tokens/train_per_sec_per_gpu": 24.51, "tokens/trainable": 67369} +{"epoch": 0.58203125, "grad_norm": 0.30141177773475647, "learning_rate": 8.656219011037509e-05, "loss": 0.007900157943367958, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00793, "step": 149, "tokens/total": 5165104, "tokens/train_per_sec_per_gpu": 23.21, "tokens/trainable": 67857} +{"epoch": 0.5859375, "grad_norm": 0.30183881521224976, "learning_rate": 8.63546196598125e-05, "loss": 0.008999361656606197, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00904, "step": 150, "tokens/total": 5202336, "tokens/train_per_sec_per_gpu": 24.76, "tokens/trainable": 68305} +{"epoch": 0.58984375, "grad_norm": 0.3974859416484833, "learning_rate": 8.614574441554145e-05, "loss": 0.01399111095815897, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.13, "memory/max_allocated (GiB)": 37.13, "ppl": 1.01409, "step": 151, "tokens/total": 5235440, "tokens/train_per_sec_per_gpu": 26.91, "tokens/trainable": 68730} +{"epoch": 0.59375, "grad_norm": 0.7622162103652954, "learning_rate": 8.593557306971349e-05, "loss": 0.012662630528211594, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.44, "memory/max_allocated (GiB)": 37.44, "ppl": 1.01274, "step": 152, "tokens/total": 5268416, "tokens/train_per_sec_per_gpu": 25.53, "tokens/trainable": 69185} +{"epoch": 0.59765625, "grad_norm": 0.4996946156024933, "learning_rate": 8.572411436841618e-05, "loss": 0.015342800877988338, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.01546, "step": 153, "tokens/total": 5302128, "tokens/train_per_sec_per_gpu": 26.11, "tokens/trainable": 69660} +{"epoch": 0.6015625, "grad_norm": 0.5062841176986694, "learning_rate": 8.551137711130922e-05, "loss": 0.012265641242265701, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.2, "memory/max_allocated (GiB)": 38.2, "ppl": 1.01234, "step": 154, "tokens/total": 5333520, "tokens/train_per_sec_per_gpu": 23.5, "tokens/trainable": 70066} +{"epoch": 0.60546875, "grad_norm": 0.4683239161968231, "learning_rate": 8.529737015125824e-05, "loss": 0.015023518353700638, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.01514, "step": 155, "tokens/total": 5367968, "tokens/train_per_sec_per_gpu": 25.39, "tokens/trainable": 70532} +{"epoch": 0.609375, "grad_norm": 0.6805155277252197, "learning_rate": 8.508210239396639e-05, "loss": 0.021470626816153526, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.0217, "step": 156, "tokens/total": 5405536, "tokens/train_per_sec_per_gpu": 23.81, "tokens/trainable": 70981} +{"epoch": 0.61328125, "grad_norm": 0.10872019827365875, "learning_rate": 8.486558279760375e-05, "loss": 0.0025918208993971348, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 36.07, "memory/max_allocated (GiB)": 36.07, "ppl": 1.0026, "step": 157, "tokens/total": 5436672, "tokens/train_per_sec_per_gpu": 26.2, "tokens/trainable": 71407} +{"epoch": 0.6171875, "grad_norm": 0.3407255709171295, "learning_rate": 8.464782037243449e-05, "loss": 0.007635204587131739, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.00766, "step": 158, "tokens/total": 5469280, "tokens/train_per_sec_per_gpu": 26.95, "tokens/trainable": 71848} +{"epoch": 0.62109375, "grad_norm": 0.27947625517845154, "learning_rate": 8.442882418044202e-05, "loss": 0.010411040857434273, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.01047, "step": 159, "tokens/total": 5501904, "tokens/train_per_sec_per_gpu": 28.89, "tokens/trainable": 72325} +{"epoch": 0.625, "grad_norm": 0.6878291964530945, "learning_rate": 8.420860333495179e-05, "loss": 0.01868761144578457, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.01886, "step": 160, "tokens/total": 5534944, "tokens/train_per_sec_per_gpu": 26.31, "tokens/trainable": 72739} +{"epoch": 0.62890625, "grad_norm": 0.3015974462032318, "learning_rate": 8.398716700025208e-05, "loss": 0.007551326882094145, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.00758, "step": 161, "tokens/total": 5568160, "tokens/train_per_sec_per_gpu": 25.29, "tokens/trainable": 73191} +{"epoch": 0.6328125, "grad_norm": 0.5933210253715515, "learning_rate": 8.376452439121266e-05, "loss": 0.025092173367738724, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 35.89, "memory/max_allocated (GiB)": 35.89, "ppl": 1.02541, "step": 162, "tokens/total": 5598256, "tokens/train_per_sec_per_gpu": 30.69, "tokens/trainable": 73642} +{"epoch": 0.63671875, "grad_norm": 0.6115720272064209, "learning_rate": 8.354068477290124e-05, "loss": 0.015862975269556046, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.01599, "step": 163, "tokens/total": 5632560, "tokens/train_per_sec_per_gpu": 27.43, "tokens/trainable": 74101} +{"epoch": 0.640625, "grad_norm": 0.5183220505714417, "learning_rate": 8.331565746019807e-05, "loss": 0.009048549458384514, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 36.52, "memory/max_allocated (GiB)": 36.52, "ppl": 1.00909, "step": 164, "tokens/total": 5664464, "tokens/train_per_sec_per_gpu": 26.72, "tokens/trainable": 74558} +{"epoch": 0.64453125, "grad_norm": 0.4922328293323517, "learning_rate": 8.308945181740812e-05, "loss": 0.018072791397571564, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.01824, "step": 165, "tokens/total": 5699040, "tokens/train_per_sec_per_gpu": 26.89, "tokens/trainable": 75000} +{"epoch": 0.6484375, "grad_norm": 0.31694912910461426, "learning_rate": 8.286207725787153e-05, "loss": 0.008364317007362843, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.0084, "step": 166, "tokens/total": 5735968, "tokens/train_per_sec_per_gpu": 22.17, "tokens/trainable": 75445} +{"epoch": 0.65234375, "grad_norm": 0.20894701778888702, "learning_rate": 8.263354324357182e-05, "loss": 0.009558405727148056, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.0096, "step": 167, "tokens/total": 5770688, "tokens/train_per_sec_per_gpu": 25.1, "tokens/trainable": 75880} +{"epoch": 0.65625, "grad_norm": 0.21291592717170715, "learning_rate": 8.240385928474219e-05, "loss": 0.009186716750264168, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00923, "step": 168, "tokens/total": 5803824, "tokens/train_per_sec_per_gpu": 26.74, "tokens/trainable": 76294} +{"epoch": 0.66015625, "grad_norm": 0.18285049498081207, "learning_rate": 8.217303493946967e-05, "loss": 0.006129817571491003, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.00615, "step": 169, "tokens/total": 5838256, "tokens/train_per_sec_per_gpu": 25.97, "tokens/trainable": 76742} +{"epoch": 0.6640625, "grad_norm": 0.30427229404449463, "learning_rate": 8.194107981329746e-05, "loss": 0.006297668907791376, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00632, "step": 170, "tokens/total": 5876224, "tokens/train_per_sec_per_gpu": 23.1, "tokens/trainable": 77196} +{"epoch": 0.66796875, "grad_norm": 0.2700961232185364, "learning_rate": 8.170800355882518e-05, "loss": 0.00324062816798687, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.24, "memory/max_allocated (GiB)": 37.24, "ppl": 1.00325, "step": 171, "tokens/total": 5910448, "tokens/train_per_sec_per_gpu": 30.1, "tokens/trainable": 77663} +{"epoch": 0.671875, "grad_norm": 0.5455095171928406, "learning_rate": 8.147381587530713e-05, "loss": 0.016349591314792633, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.01648, "step": 172, "tokens/total": 5944256, "tokens/train_per_sec_per_gpu": 26.27, "tokens/trainable": 78117} +{"epoch": 0.67578125, "grad_norm": 1.826767921447754, "learning_rate": 8.123852650824877e-05, "loss": 0.031533364206552505, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.03204, "step": 173, "tokens/total": 5978288, "tokens/train_per_sec_per_gpu": 28.3, "tokens/trainable": 78601} +{"epoch": 0.6796875, "grad_norm": 0.23491911590099335, "learning_rate": 8.100214524900103e-05, "loss": 0.0032218347769230604, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.52, "memory/max_allocated (GiB)": 38.52, "ppl": 1.00323, "step": 174, "tokens/total": 6013680, "tokens/train_per_sec_per_gpu": 22.02, "tokens/trainable": 79032} +{"epoch": 0.68359375, "grad_norm": 0.4189692735671997, "learning_rate": 8.076468193435301e-05, "loss": 0.0064404308795928955, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.00646, "step": 175, "tokens/total": 6045072, "tokens/train_per_sec_per_gpu": 25.75, "tokens/trainable": 79456} +{"epoch": 0.6875, "grad_norm": 0.5980409979820251, "learning_rate": 8.052614644612253e-05, "loss": 0.006724453531205654, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.00675, "step": 176, "tokens/total": 6082336, "tokens/train_per_sec_per_gpu": 23.11, "tokens/trainable": 79917} +{"epoch": 0.69140625, "grad_norm": 0.798369824886322, "learning_rate": 8.028654871074489e-05, "loss": 0.008773291483521461, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00881, "step": 177, "tokens/total": 6117024, "tokens/train_per_sec_per_gpu": 25.27, "tokens/trainable": 80358} +{"epoch": 0.6953125, "grad_norm": 0.04425400123000145, "learning_rate": 8.004589869885986e-05, "loss": 0.0005744608351960778, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 36.49, "memory/max_allocated (GiB)": 36.49, "ppl": 1.00057, "step": 178, "tokens/total": 6149136, "tokens/train_per_sec_per_gpu": 26.39, "tokens/trainable": 80777} +{"epoch": 0.69921875, "grad_norm": 0.5939944982528687, "learning_rate": 7.980420642489674e-05, "loss": 0.022036680951714516, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.02228, "step": 179, "tokens/total": 6187776, "tokens/train_per_sec_per_gpu": 23.16, "tokens/trainable": 81250} +{"epoch": 0.703125, "grad_norm": 0.1765715628862381, "learning_rate": 7.95614819466576e-05, "loss": 0.002331191673874855, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00233, "step": 180, "tokens/total": 6221728, "tokens/train_per_sec_per_gpu": 28.07, "tokens/trainable": 81715} +{"epoch": 0.70703125, "grad_norm": 0.4100755751132965, "learning_rate": 7.931773536489872e-05, "loss": 0.025562306866049767, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.02589, "step": 181, "tokens/total": 6255920, "tokens/train_per_sec_per_gpu": 24.68, "tokens/trainable": 82160} +{"epoch": 0.7109375, "grad_norm": 0.2979554831981659, "learning_rate": 7.907297682291035e-05, "loss": 0.005667334888130426, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00568, "step": 182, "tokens/total": 6291104, "tokens/train_per_sec_per_gpu": 26.28, "tokens/trainable": 82624} +{"epoch": 0.71484375, "grad_norm": 0.34487512707710266, "learning_rate": 7.882721650609442e-05, "loss": 0.007897410541772842, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.00793, "step": 183, "tokens/total": 6330976, "tokens/train_per_sec_per_gpu": 23.02, "tokens/trainable": 83125} +{"epoch": 0.71875, "grad_norm": 0.4456903040409088, "learning_rate": 7.85804646415409e-05, "loss": 0.023342516273260117, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.02362, "step": 184, "tokens/total": 6366160, "tokens/train_per_sec_per_gpu": 26.52, "tokens/trainable": 83581} +{"epoch": 0.72265625, "grad_norm": 0.1994606852531433, "learning_rate": 7.833273149760207e-05, "loss": 0.0047650523483753204, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.00478, "step": 185, "tokens/total": 6397280, "tokens/train_per_sec_per_gpu": 28.51, "tokens/trainable": 84051} +{"epoch": 0.7265625, "grad_norm": 0.4261516332626343, "learning_rate": 7.808402738346527e-05, "loss": 0.028205350041389465, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.02861, "step": 186, "tokens/total": 6435632, "tokens/train_per_sec_per_gpu": 23.22, "tokens/trainable": 84500} +{"epoch": 0.73046875, "grad_norm": 0.11830081045627594, "learning_rate": 7.783436264872382e-05, "loss": 0.002664863597601652, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.00267, "step": 187, "tokens/total": 6472752, "tokens/train_per_sec_per_gpu": 24.58, "tokens/trainable": 84986} +{"epoch": 0.734375, "grad_norm": 0.08070681989192963, "learning_rate": 7.758374768294647e-05, "loss": 0.0020932189654558897, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.05, "memory/max_allocated (GiB)": 38.05, "ppl": 1.0021, "step": 188, "tokens/total": 6508528, "tokens/train_per_sec_per_gpu": 26.74, "tokens/trainable": 85465} +{"epoch": 0.73828125, "grad_norm": 0.2011636197566986, "learning_rate": 7.733219291524489e-05, "loss": 0.006888517644256353, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.59, "memory/max_allocated (GiB)": 37.59, "ppl": 1.00691, "step": 189, "tokens/total": 6542048, "tokens/train_per_sec_per_gpu": 27.23, "tokens/trainable": 85896} +{"epoch": 0.7421875, "grad_norm": 0.4698273539543152, "learning_rate": 7.707970881383977e-05, "loss": 0.012839578092098236, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 36.57, "memory/max_allocated (GiB)": 36.57, "ppl": 1.01292, "step": 190, "tokens/total": 6573792, "tokens/train_per_sec_per_gpu": 27.85, "tokens/trainable": 86346} +{"epoch": 0.74609375, "grad_norm": 0.45862480998039246, "learning_rate": 7.682630588562518e-05, "loss": 0.021290529519319534, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.02152, "step": 191, "tokens/total": 6608848, "tokens/train_per_sec_per_gpu": 24.13, "tokens/trainable": 86777} +{"epoch": 0.75, "grad_norm": 0.22303690016269684, "learning_rate": 7.657199467573129e-05, "loss": 0.008831418119370937, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.00887, "step": 192, "tokens/total": 6642560, "tokens/train_per_sec_per_gpu": 25.9, "tokens/trainable": 87249} +{"epoch": 0.75390625, "grad_norm": 0.18451477587223053, "learning_rate": 7.631678576708561e-05, "loss": 0.003663002047687769, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00367, "step": 193, "tokens/total": 6676272, "tokens/train_per_sec_per_gpu": 28.73, "tokens/trainable": 87741} +{"epoch": 0.7578125, "grad_norm": 0.21797610819339752, "learning_rate": 7.606068977997255e-05, "loss": 0.004276630934327841, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00429, "step": 194, "tokens/total": 6713568, "tokens/train_per_sec_per_gpu": 25.48, "tokens/trainable": 88219} +{"epoch": 0.76171875, "grad_norm": 0.2968653738498688, "learning_rate": 7.580371737159148e-05, "loss": 0.006051064468920231, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00607, "step": 195, "tokens/total": 6749296, "tokens/train_per_sec_per_gpu": 22.77, "tokens/trainable": 88630} +{"epoch": 0.765625, "grad_norm": 0.2511523365974426, "learning_rate": 7.554587923561324e-05, "loss": 0.005836069583892822, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00585, "step": 196, "tokens/total": 6783952, "tokens/train_per_sec_per_gpu": 24.42, "tokens/trainable": 89068} +{"epoch": 0.76953125, "grad_norm": 0.3265615403652191, "learning_rate": 7.528718610173511e-05, "loss": 0.012963658198714256, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.48, "memory/max_allocated (GiB)": 37.48, "ppl": 1.01305, "step": 197, "tokens/total": 6819152, "tokens/train_per_sec_per_gpu": 22.64, "tokens/trainable": 89489} +{"epoch": 0.7734375, "grad_norm": 0.15756382048130035, "learning_rate": 7.502764873523431e-05, "loss": 0.0017125426093116403, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.00171, "step": 198, "tokens/total": 6853440, "tokens/train_per_sec_per_gpu": 27.04, "tokens/trainable": 89928} +{"epoch": 0.77734375, "grad_norm": 0.3170602023601532, "learning_rate": 7.476727793652011e-05, "loss": 0.020139118656516075, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 36.71, "memory/max_allocated (GiB)": 36.71, "ppl": 1.02034, "step": 199, "tokens/total": 6886272, "tokens/train_per_sec_per_gpu": 27.86, "tokens/trainable": 90407} +{"epoch": 0.78125, "grad_norm": 0.5062584280967712, "learning_rate": 7.450608454068415e-05, "loss": 0.031402140855789185, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.0319, "step": 200, "tokens/total": 6920624, "tokens/train_per_sec_per_gpu": 26.45, "tokens/trainable": 90849} +{"epoch": 0.78515625, "grad_norm": 1.06876802444458, "learning_rate": 7.424407941704987e-05, "loss": 0.0070610977709293365, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.00709, "step": 201, "tokens/total": 6957328, "tokens/train_per_sec_per_gpu": 27.08, "tokens/trainable": 91350} +{"epoch": 0.7890625, "grad_norm": 0.6329917311668396, "learning_rate": 7.398127346871986e-05, "loss": 0.026193533092737198, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.02654, "step": 202, "tokens/total": 6994048, "tokens/train_per_sec_per_gpu": 24.19, "tokens/trainable": 91805} +{"epoch": 0.79296875, "grad_norm": 0.18598927557468414, "learning_rate": 7.371767763212238e-05, "loss": 0.004443371668457985, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.00445, "step": 203, "tokens/total": 7029040, "tokens/train_per_sec_per_gpu": 26.31, "tokens/trainable": 92258} +{"epoch": 0.796875, "grad_norm": 0.30310550332069397, "learning_rate": 7.345330287655617e-05, "loss": 0.01373167522251606, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.01383, "step": 204, "tokens/total": 7067104, "tokens/train_per_sec_per_gpu": 22.53, "tokens/trainable": 92721} +{"epoch": 0.80078125, "grad_norm": 0.26817187666893005, "learning_rate": 7.31881602037339e-05, "loss": 0.01132812537252903, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.01139, "step": 205, "tokens/total": 7099136, "tokens/train_per_sec_per_gpu": 26.11, "tokens/trainable": 93160} +{"epoch": 0.8046875, "grad_norm": 0.1985321342945099, "learning_rate": 7.29222606473245e-05, "loss": 0.009257175959646702, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.0093, "step": 206, "tokens/total": 7132208, "tokens/train_per_sec_per_gpu": 26.26, "tokens/trainable": 93605} +{"epoch": 0.80859375, "grad_norm": 0.4501406252384186, "learning_rate": 7.265561527249383e-05, "loss": 0.008806734345853329, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.00885, "step": 207, "tokens/total": 7166256, "tokens/train_per_sec_per_gpu": 27.48, "tokens/trainable": 94041} +{"epoch": 0.8125, "grad_norm": 0.3533204197883606, "learning_rate": 7.238823517544436e-05, "loss": 0.012789040803909302, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.01287, "step": 208, "tokens/total": 7203152, "tokens/train_per_sec_per_gpu": 28.1, "tokens/trainable": 94512} +{"epoch": 0.81640625, "grad_norm": 0.11410503089427948, "learning_rate": 7.212013148295333e-05, "loss": 0.003779409686103463, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.00379, "step": 209, "tokens/total": 7237264, "tokens/train_per_sec_per_gpu": 29.49, "tokens/trainable": 94994} +{"epoch": 0.8203125, "grad_norm": 0.2760160267353058, "learning_rate": 7.185131535190975e-05, "loss": 0.00868284422904253, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00872, "step": 210, "tokens/total": 7269664, "tokens/train_per_sec_per_gpu": 29.68, "tokens/trainable": 95503} +{"epoch": 0.82421875, "grad_norm": 0.1912858933210373, "learning_rate": 7.158179796885005e-05, "loss": 0.005072095897048712, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.00508, "step": 211, "tokens/total": 7304080, "tokens/train_per_sec_per_gpu": 27.21, "tokens/trainable": 95954} +{"epoch": 0.828125, "grad_norm": 0.14060913026332855, "learning_rate": 7.131159054949273e-05, "loss": 0.0036308381240814924, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00364, "step": 212, "tokens/total": 7342480, "tokens/train_per_sec_per_gpu": 22.15, "tokens/trainable": 96428} +{"epoch": 0.83203125, "grad_norm": 0.212240070104599, "learning_rate": 7.104070433827139e-05, "loss": 0.00415847497060895, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00417, "step": 213, "tokens/total": 7378032, "tokens/train_per_sec_per_gpu": 22.86, "tokens/trainable": 96828} +{"epoch": 0.8359375, "grad_norm": 0.3276149034500122, "learning_rate": 7.076915060786705e-05, "loss": 0.00955475214868784, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.96, "memory/max_allocated (GiB)": 37.96, "ppl": 1.0096, "step": 214, "tokens/total": 7410928, "tokens/train_per_sec_per_gpu": 24.01, "tokens/trainable": 97270} +{"epoch": 0.83984375, "grad_norm": 0.3466997742652893, "learning_rate": 7.049694065873882e-05, "loss": 0.011770485900342464, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.01184, "step": 215, "tokens/total": 7445664, "tokens/train_per_sec_per_gpu": 27.72, "tokens/trainable": 97727} +{"epoch": 0.84375, "grad_norm": 0.4653541147708893, "learning_rate": 7.022408581865382e-05, "loss": 0.01450347900390625, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.69, "memory/max_allocated (GiB)": 38.69, "ppl": 1.01461, "step": 216, "tokens/total": 7479984, "tokens/train_per_sec_per_gpu": 25.24, "tokens/trainable": 98201} +{"epoch": 0.84765625, "grad_norm": 0.010370668955147266, "learning_rate": 6.99505974422157e-05, "loss": 0.00017751543782651424, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.00018, "step": 217, "tokens/total": 7514144, "tokens/train_per_sec_per_gpu": 28.16, "tokens/trainable": 98674} +{"epoch": 0.8515625, "grad_norm": 0.15716414153575897, "learning_rate": 6.967648691039213e-05, "loss": 0.0023922098334878683, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 36.73, "memory/max_allocated (GiB)": 36.73, "ppl": 1.0024, "step": 218, "tokens/total": 7546256, "tokens/train_per_sec_per_gpu": 27.7, "tokens/trainable": 99108} +{"epoch": 0.85546875, "grad_norm": 0.03715338930487633, "learning_rate": 6.940176563004123e-05, "loss": 0.0008732840069569647, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00087, "step": 219, "tokens/total": 7580432, "tokens/train_per_sec_per_gpu": 25.71, "tokens/trainable": 99540} +{"epoch": 0.859375, "grad_norm": 0.23328709602355957, "learning_rate": 6.912644503343682e-05, "loss": 0.004946554079651833, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.00496, "step": 220, "tokens/total": 7617904, "tokens/train_per_sec_per_gpu": 23.28, "tokens/trainable": 99978} +{"epoch": 0.86328125, "grad_norm": 0.3547626733779907, "learning_rate": 6.885053657779273e-05, "loss": 0.016034133732318878, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.01616, "step": 221, "tokens/total": 7649696, "tokens/train_per_sec_per_gpu": 28.18, "tokens/trainable": 100450} +{"epoch": 0.8671875, "grad_norm": 0.025099271908402443, "learning_rate": 6.857405174478604e-05, "loss": 0.00048372370656579733, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.00048, "step": 222, "tokens/total": 7684464, "tokens/train_per_sec_per_gpu": 24.15, "tokens/trainable": 100903} +{"epoch": 0.87109375, "grad_norm": 0.3420548439025879, "learning_rate": 6.82970020400792e-05, "loss": 0.032988134771585464, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.03354, "step": 223, "tokens/total": 7721936, "tokens/train_per_sec_per_gpu": 22.15, "tokens/trainable": 101351} +{"epoch": 0.875, "grad_norm": 0.18113142251968384, "learning_rate": 6.801939899284132e-05, "loss": 0.004558954853564501, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.34, "memory/max_allocated (GiB)": 38.34, "ppl": 1.00457, "step": 224, "tokens/total": 7758160, "tokens/train_per_sec_per_gpu": 26.33, "tokens/trainable": 101857} +{"epoch": 0.87890625, "grad_norm": 0.04607327654957771, "learning_rate": 6.774125415526827e-05, "loss": 0.0007972497260197997, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.0008, "step": 225, "tokens/total": 7791216, "tokens/train_per_sec_per_gpu": 27.54, "tokens/trainable": 102328} +{"epoch": 0.8828125, "grad_norm": 0.037590160965919495, "learning_rate": 6.746257910210214e-05, "loss": 0.0009044512407854199, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.12, "memory/max_allocated (GiB)": 37.12, "ppl": 1.0009, "step": 226, "tokens/total": 7824400, "tokens/train_per_sec_per_gpu": 27.91, "tokens/trainable": 102779} +{"epoch": 0.88671875, "grad_norm": 0.1384570598602295, "learning_rate": 6.718338543014937e-05, "loss": 0.0031876720022410154, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 35.75, "memory/max_allocated (GiB)": 35.75, "ppl": 1.00319, "step": 227, "tokens/total": 7854672, "tokens/train_per_sec_per_gpu": 30.59, "tokens/trainable": 103228} +{"epoch": 0.890625, "grad_norm": 0.24652813374996185, "learning_rate": 6.69036847577983e-05, "loss": 0.007263931445777416, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.00729, "step": 228, "tokens/total": 7888592, "tokens/train_per_sec_per_gpu": 26.5, "tokens/trainable": 103684} +{"epoch": 0.89453125, "grad_norm": 0.27251675724983215, "learning_rate": 6.662348872453553e-05, "loss": 0.007037941366434097, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.33, "memory/max_allocated (GiB)": 38.33, "ppl": 1.00706, "step": 229, "tokens/total": 7924848, "tokens/train_per_sec_per_gpu": 27.25, "tokens/trainable": 104145} +{"epoch": 0.8984375, "grad_norm": 0.07535093277692795, "learning_rate": 6.63428089904618e-05, "loss": 0.002503441646695137, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.00251, "step": 230, "tokens/total": 7963856, "tokens/train_per_sec_per_gpu": 23.99, "tokens/trainable": 104598} +{"epoch": 0.90234375, "grad_norm": 0.16935068368911743, "learning_rate": 6.60616572358065e-05, "loss": 0.004252578131854534, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.00426, "step": 231, "tokens/total": 7998320, "tokens/train_per_sec_per_gpu": 25.39, "tokens/trainable": 104998} +{"epoch": 0.90625, "grad_norm": 0.09937569499015808, "learning_rate": 6.578004516044172e-05, "loss": 0.0026845792308449745, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00269, "step": 232, "tokens/total": 8034096, "tokens/train_per_sec_per_gpu": 27.26, "tokens/trainable": 105465} +{"epoch": 0.91015625, "grad_norm": 0.24326930940151215, "learning_rate": 6.549798448339548e-05, "loss": 0.011558817699551582, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.59, "memory/max_allocated (GiB)": 38.59, "ppl": 1.01163, "step": 233, "tokens/total": 8071360, "tokens/train_per_sec_per_gpu": 24.78, "tokens/trainable": 105928} +{"epoch": 0.9140625, "grad_norm": 0.1110740378499031, "learning_rate": 6.521548694236384e-05, "loss": 0.002248761709779501, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00225, "step": 234, "tokens/total": 8108560, "tokens/train_per_sec_per_gpu": 23.13, "tokens/trainable": 106365} +{"epoch": 0.91796875, "grad_norm": 0.16922619938850403, "learning_rate": 6.493256429322259e-05, "loss": 0.0058963000774383545, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.00591, "step": 235, "tokens/total": 8144816, "tokens/train_per_sec_per_gpu": 22.0, "tokens/trainable": 106796} +{"epoch": 0.921875, "grad_norm": 0.2891996204853058, "learning_rate": 6.464922830953799e-05, "loss": 0.01125964242964983, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.01132, "step": 236, "tokens/total": 8183680, "tokens/train_per_sec_per_gpu": 22.09, "tokens/trainable": 107227} +{"epoch": 0.92578125, "grad_norm": 0.5496317744255066, "learning_rate": 6.436549078207688e-05, "loss": 0.025916211307048798, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.02625, "step": 237, "tokens/total": 8216688, "tokens/train_per_sec_per_gpu": 28.5, "tokens/trainable": 107687} +{"epoch": 0.9296875, "grad_norm": 0.23757553100585938, "learning_rate": 6.408136351831592e-05, "loss": 0.013023680076003075, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.01311, "step": 238, "tokens/total": 8249104, "tokens/train_per_sec_per_gpu": 23.88, "tokens/trainable": 108127} +{"epoch": 0.93359375, "grad_norm": 0.13262632489204407, "learning_rate": 6.379685834195036e-05, "loss": 0.002483679447323084, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00249, "step": 239, "tokens/total": 8288528, "tokens/train_per_sec_per_gpu": 24.27, "tokens/trainable": 108633} +{"epoch": 0.9375, "grad_norm": 0.08524111658334732, "learning_rate": 6.351198709240186e-05, "loss": 0.0022014991845935583, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.0022, "step": 240, "tokens/total": 8328464, "tokens/train_per_sec_per_gpu": 20.79, "tokens/trainable": 109085} +{"epoch": 0.94140625, "grad_norm": 0.1263168305158615, "learning_rate": 6.32267616243259e-05, "loss": 0.0031226305291056633, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.00313, "step": 241, "tokens/total": 8361264, "tokens/train_per_sec_per_gpu": 28.39, "tokens/trainable": 109548} +{"epoch": 0.9453125, "grad_norm": 0.1343449354171753, "learning_rate": 6.294119380711849e-05, "loss": 0.003653735388070345, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.39, "memory/max_allocated (GiB)": 38.39, "ppl": 1.00366, "step": 242, "tokens/total": 8395552, "tokens/train_per_sec_per_gpu": 23.53, "tokens/trainable": 109975} +{"epoch": 0.94921875, "grad_norm": 0.4781797528266907, "learning_rate": 6.265529552442209e-05, "loss": 0.013756190426647663, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.01385, "step": 243, "tokens/total": 8430656, "tokens/train_per_sec_per_gpu": 27.72, "tokens/trainable": 110436} +{"epoch": 0.953125, "grad_norm": 0.0641668364405632, "learning_rate": 6.236907867363127e-05, "loss": 0.0009904492180794477, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.17, "memory/max_allocated (GiB)": 38.17, "ppl": 1.00099, "step": 244, "tokens/total": 8465232, "tokens/train_per_sec_per_gpu": 26.31, "tokens/trainable": 110882} +{"epoch": 0.95703125, "grad_norm": 0.03849957510828972, "learning_rate": 6.208255516539749e-05, "loss": 0.000989900203421712, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.00099, "step": 245, "tokens/total": 8497552, "tokens/train_per_sec_per_gpu": 26.28, "tokens/trainable": 111355} +{"epoch": 0.9609375, "grad_norm": 0.09346359223127365, "learning_rate": 6.179573692313344e-05, "loss": 0.002725712489336729, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.8, "memory/max_allocated (GiB)": 38.8, "ppl": 1.00273, "step": 246, "tokens/total": 8534336, "tokens/train_per_sec_per_gpu": 25.74, "tokens/trainable": 111803} +{"epoch": 0.96484375, "grad_norm": 0.06408923119306564, "learning_rate": 6.150863588251694e-05, "loss": 0.0017170626670122147, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.00172, "step": 247, "tokens/total": 8574224, "tokens/train_per_sec_per_gpu": 22.41, "tokens/trainable": 112270} +{"epoch": 0.96875, "grad_norm": 0.3085225522518158, "learning_rate": 6.122126399099419e-05, "loss": 0.007564227096736431, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.00759, "step": 248, "tokens/total": 8613616, "tokens/train_per_sec_per_gpu": 26.54, "tokens/trainable": 112778} +{"epoch": 0.97265625, "grad_norm": 0.6150240302085876, "learning_rate": 6.0933633207282615e-05, "loss": 0.029102372005581856, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.02953, "step": 249, "tokens/total": 8650048, "tokens/train_per_sec_per_gpu": 27.04, "tokens/trainable": 113284} +{"epoch": 0.9765625, "grad_norm": 0.12836483120918274, "learning_rate": 6.064575550087316e-05, "loss": 0.0037036710418760777, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00371, "step": 250, "tokens/total": 8686688, "tokens/train_per_sec_per_gpu": 23.44, "tokens/trainable": 113722} +{"epoch": 0.98046875, "grad_norm": 0.5454620718955994, "learning_rate": 6.0357642851532245e-05, "loss": 0.014493503607809544, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.0146, "step": 251, "tokens/total": 8723376, "tokens/train_per_sec_per_gpu": 25.46, "tokens/trainable": 114213} +{"epoch": 0.984375, "grad_norm": 0.4212971031665802, "learning_rate": 6.0069307248803294e-05, "loss": 0.010792389512062073, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.01085, "step": 252, "tokens/total": 8758736, "tokens/train_per_sec_per_gpu": 21.68, "tokens/trainable": 114634} +{"epoch": 0.98828125, "grad_norm": 0.4723483622074127, "learning_rate": 5.9780760691507635e-05, "loss": 0.015316755510866642, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.33, "memory/max_allocated (GiB)": 36.33, "ppl": 1.01543, "step": 253, "tokens/total": 8790480, "tokens/train_per_sec_per_gpu": 26.38, "tokens/trainable": 115066} +{"epoch": 0.9921875, "grad_norm": 0.08653385192155838, "learning_rate": 5.9492015187245334e-05, "loss": 0.0016570239095017314, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00166, "step": 254, "tokens/total": 8828224, "tokens/train_per_sec_per_gpu": 23.02, "tokens/trainable": 115520} +{"epoch": 0.99609375, "grad_norm": 0.07713080942630768, "learning_rate": 5.920308275189541e-05, "loss": 0.0014303463976830244, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.00143, "step": 255, "tokens/total": 8860400, "tokens/train_per_sec_per_gpu": 25.21, "tokens/trainable": 115921} +{"epoch": 1.0, "grad_norm": 0.37177225947380066, "learning_rate": 5.8913975409115874e-05, "loss": 0.006526221055537462, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.00655, "step": 256, "tokens/total": 8893760, "tokens/train_per_sec_per_gpu": 25.08, "tokens/trainable": 116384} +{"epoch": 1.00390625, "grad_norm": 0.16997873783111572, "learning_rate": 5.8624705189843395e-05, "loss": 0.0036788254510611296, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 1.00369, "step": 257, "tokens/total": 8930976, "tokens/train_per_sec_per_gpu": 24.29, "tokens/trainable": 116871} +{"epoch": 1.0078125, "grad_norm": 0.08832691609859467, "learning_rate": 5.833528413179249e-05, "loss": 0.0017382533987984061, "memory/device_reserved (GiB)": 42.25, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00174, "step": 258, "tokens/total": 8967472, "tokens/train_per_sec_per_gpu": 27.46, "tokens/trainable": 117369} +{"epoch": 1.01171875, "grad_norm": 0.3614032566547394, "learning_rate": 5.80457242789548e-05, "loss": 0.004077639430761337, "memory/device_reserved (GiB)": 42.31, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.00409, "step": 259, "tokens/total": 9007456, "tokens/train_per_sec_per_gpu": 23.56, "tokens/trainable": 117866} +{"epoch": 1.015625, "grad_norm": 0.12621207535266876, "learning_rate": 5.77560376810977e-05, "loss": 0.0030621832702308893, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 1.00307, "step": 260, "tokens/total": 9041104, "tokens/train_per_sec_per_gpu": 28.84, "tokens/trainable": 118330} +{"epoch": 1.01953125, "grad_norm": 0.12709186971187592, "learning_rate": 5.7466236393263005e-05, "loss": 0.002869829535484314, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.00287, "step": 261, "tokens/total": 9076176, "tokens/train_per_sec_per_gpu": 25.22, "tokens/trainable": 118776} +{"epoch": 1.0234375, "grad_norm": 0.3106381595134735, "learning_rate": 5.717633247526522e-05, "loss": 0.00593251409009099, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00595, "step": 262, "tokens/total": 9107472, "tokens/train_per_sec_per_gpu": 27.39, "tokens/trainable": 119211} +{"epoch": 1.02734375, "grad_norm": 0.03368746489286423, "learning_rate": 5.688633799118971e-05, "loss": 0.0005369426216930151, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.00054, "step": 263, "tokens/total": 9139552, "tokens/train_per_sec_per_gpu": 23.11, "tokens/trainable": 119616} +{"epoch": 1.03125, "grad_norm": 0.34298738837242126, "learning_rate": 5.659626500889066e-05, "loss": 0.008858450688421726, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.0089, "step": 264, "tokens/total": 9173424, "tokens/train_per_sec_per_gpu": 25.5, "tokens/trainable": 120086} +{"epoch": 1.03515625, "grad_norm": 0.11394135653972626, "learning_rate": 5.6306125599488905e-05, "loss": 0.001121311099268496, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.00112, "step": 265, "tokens/total": 9203984, "tokens/train_per_sec_per_gpu": 31.76, "tokens/trainable": 120525} +{"epoch": 1.0390625, "grad_norm": 0.2586342990398407, "learning_rate": 5.601593183686955e-05, "loss": 0.003055533394217491, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.69, "memory/max_allocated (GiB)": 38.69, "ppl": 1.00306, "step": 266, "tokens/total": 9241248, "tokens/train_per_sec_per_gpu": 25.65, "tokens/trainable": 121001} +{"epoch": 1.04296875, "grad_norm": 0.24093706905841827, "learning_rate": 5.572569579717961e-05, "loss": 0.0071195196360349655, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.18, "memory/max_allocated (GiB)": 39.18, "ppl": 1.00714, "step": 267, "tokens/total": 9276768, "tokens/train_per_sec_per_gpu": 24.16, "tokens/trainable": 121465} +{"epoch": 1.046875, "grad_norm": 0.006833173800259829, "learning_rate": 5.543542955832538e-05, "loss": 0.0001696656981948763, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00017, "step": 268, "tokens/total": 9316528, "tokens/train_per_sec_per_gpu": 21.4, "tokens/trainable": 121908} +{"epoch": 1.05078125, "grad_norm": 0.23576371371746063, "learning_rate": 5.514514519946986e-05, "loss": 0.001644270378164947, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.00165, "step": 269, "tokens/total": 9356352, "tokens/train_per_sec_per_gpu": 25.39, "tokens/trainable": 122406} +{"epoch": 1.0546875, "grad_norm": 0.008134556002914906, "learning_rate": 5.485485480053015e-05, "loss": 0.00015327136497944593, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.00015, "step": 270, "tokens/total": 9394576, "tokens/train_per_sec_per_gpu": 21.64, "tokens/trainable": 122812} +{"epoch": 1.05859375, "grad_norm": 0.10254368185997009, "learning_rate": 5.4564570441674645e-05, "loss": 0.0005815273616462946, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.92, "memory/max_allocated (GiB)": 36.92, "ppl": 1.00058, "step": 271, "tokens/total": 9427840, "tokens/train_per_sec_per_gpu": 25.18, "tokens/trainable": 123252} +{"epoch": 1.0625, "grad_norm": 0.039235521107912064, "learning_rate": 5.42743042028204e-05, "loss": 0.0004823771305382252, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.00048, "step": 272, "tokens/total": 9463536, "tokens/train_per_sec_per_gpu": 20.35, "tokens/trainable": 123692} +{"epoch": 1.06640625, "grad_norm": 0.31263241171836853, "learning_rate": 5.3984068163130464e-05, "loss": 0.00669652596116066, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.09, "memory/max_allocated (GiB)": 37.09, "ppl": 1.00672, "step": 273, "tokens/total": 9497248, "tokens/train_per_sec_per_gpu": 27.79, "tokens/trainable": 124162} +{"epoch": 1.0703125, "grad_norm": 0.12570828199386597, "learning_rate": 5.369387440051111e-05, "loss": 0.0022546916734427214, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.00226, "step": 274, "tokens/total": 9530784, "tokens/train_per_sec_per_gpu": 25.11, "tokens/trainable": 124625} +{"epoch": 1.07421875, "grad_norm": 0.034590087831020355, "learning_rate": 5.340373499110935e-05, "loss": 0.00037889782106503844, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00038, "step": 275, "tokens/total": 9566656, "tokens/train_per_sec_per_gpu": 26.21, "tokens/trainable": 125119} +{"epoch": 1.078125, "grad_norm": 0.047902241349220276, "learning_rate": 5.3113662008810304e-05, "loss": 0.000374368712073192, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00037, "step": 276, "tokens/total": 9606672, "tokens/train_per_sec_per_gpu": 24.68, "tokens/trainable": 125601} +{"epoch": 1.08203125, "grad_norm": 0.017680885270237923, "learning_rate": 5.282366752473479e-05, "loss": 0.00028666266007348895, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 1.00029, "step": 277, "tokens/total": 9643680, "tokens/train_per_sec_per_gpu": 24.21, "tokens/trainable": 126075} +{"epoch": 1.0859375, "grad_norm": 0.15726575255393982, "learning_rate": 5.2533763606737005e-05, "loss": 0.0012944742338731885, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.0013, "step": 278, "tokens/total": 9683200, "tokens/train_per_sec_per_gpu": 19.14, "tokens/trainable": 126491} +{"epoch": 1.08984375, "grad_norm": 0.025799144059419632, "learning_rate": 5.224396231890232e-05, "loss": 0.0003375419764779508, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.00034, "step": 279, "tokens/total": 9714672, "tokens/train_per_sec_per_gpu": 29.72, "tokens/trainable": 126940} +{"epoch": 1.09375, "grad_norm": 0.2681983411312103, "learning_rate": 5.195427572104522e-05, "loss": 0.004274600185453892, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00428, "step": 280, "tokens/total": 9749104, "tokens/train_per_sec_per_gpu": 23.5, "tokens/trainable": 127376} +{"epoch": 1.09765625, "grad_norm": 0.5791247487068176, "learning_rate": 5.166471586820751e-05, "loss": 0.01096736267209053, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.35, "memory/max_allocated (GiB)": 39.35, "ppl": 1.01103, "step": 281, "tokens/total": 9785712, "tokens/train_per_sec_per_gpu": 23.54, "tokens/trainable": 127841} +{"epoch": 1.1015625, "grad_norm": 0.3482302725315094, "learning_rate": 5.1375294810156615e-05, "loss": 0.00835875142365694, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.00839, "step": 282, "tokens/total": 9820544, "tokens/train_per_sec_per_gpu": 25.63, "tokens/trainable": 128288} +{"epoch": 1.10546875, "grad_norm": 0.17428326606750488, "learning_rate": 5.1086024590884144e-05, "loss": 0.001065978198312223, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00107, "step": 283, "tokens/total": 9857952, "tokens/train_per_sec_per_gpu": 24.77, "tokens/trainable": 128737} +{"epoch": 1.109375, "grad_norm": 0.05264691263437271, "learning_rate": 5.079691724810461e-05, "loss": 0.0005971448263153434, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.33, "memory/max_allocated (GiB)": 38.33, "ppl": 1.0006, "step": 284, "tokens/total": 9894128, "tokens/train_per_sec_per_gpu": 21.58, "tokens/trainable": 129172} +{"epoch": 1.11328125, "grad_norm": 0.09231484681367874, "learning_rate": 5.0507984812754684e-05, "loss": 0.0010427602101117373, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.17, "memory/max_allocated (GiB)": 37.17, "ppl": 1.00104, "step": 285, "tokens/total": 9927568, "tokens/train_per_sec_per_gpu": 28.47, "tokens/trainable": 129669} +{"epoch": 1.1171875, "grad_norm": 0.11025512218475342, "learning_rate": 5.021923930849237e-05, "loss": 0.001298016868531704, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.0013, "step": 286, "tokens/total": 9961888, "tokens/train_per_sec_per_gpu": 27.7, "tokens/trainable": 130155} +{"epoch": 1.12109375, "grad_norm": 0.07041057199239731, "learning_rate": 4.99306927511967e-05, "loss": 0.00016927000251598656, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.31, "memory/max_allocated (GiB)": 37.31, "ppl": 1.00017, "step": 287, "tokens/total": 9995696, "tokens/train_per_sec_per_gpu": 27.64, "tokens/trainable": 130595} +{"epoch": 1.125, "grad_norm": 0.26788753271102905, "learning_rate": 4.964235714846775e-05, "loss": 0.0037207012064754963, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.72, "memory/max_allocated (GiB)": 36.72, "ppl": 1.00373, "step": 288, "tokens/total": 10028000, "tokens/train_per_sec_per_gpu": 26.92, "tokens/trainable": 131015} +{"epoch": 1.12890625, "grad_norm": 0.005359263624995947, "learning_rate": 4.9354244499126866e-05, "loss": 6.541662150993943e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.16, "memory/max_allocated (GiB)": 36.16, "ppl": 1.00007, "step": 289, "tokens/total": 10059728, "tokens/train_per_sec_per_gpu": 29.13, "tokens/trainable": 131481} +{"epoch": 1.1328125, "grad_norm": 0.011213099583983421, "learning_rate": 4.90663667927174e-05, "loss": 0.00014611706137657166, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.00015, "step": 290, "tokens/total": 10092576, "tokens/train_per_sec_per_gpu": 28.57, "tokens/trainable": 131930} +{"epoch": 1.13671875, "grad_norm": 0.015130522660911083, "learning_rate": 4.877873600900581e-05, "loss": 0.0001592972403159365, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.00016, "step": 291, "tokens/total": 10128608, "tokens/train_per_sec_per_gpu": 21.46, "tokens/trainable": 132388} +{"epoch": 1.140625, "grad_norm": 0.08429474383592606, "learning_rate": 4.849136411748306e-05, "loss": 0.0006371199269779027, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.00064, "step": 292, "tokens/total": 10166192, "tokens/train_per_sec_per_gpu": 24.66, "tokens/trainable": 132860} +{"epoch": 1.14453125, "grad_norm": 0.2350953370332718, "learning_rate": 4.8204263076866574e-05, "loss": 0.012760981917381287, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.01284, "step": 293, "tokens/total": 10201840, "tokens/train_per_sec_per_gpu": 29.35, "tokens/trainable": 133360} +{"epoch": 1.1484375, "grad_norm": 0.009994298219680786, "learning_rate": 4.791744483460251e-05, "loss": 0.00010504976671654731, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00011, "step": 294, "tokens/total": 10236048, "tokens/train_per_sec_per_gpu": 24.04, "tokens/trainable": 133793} +{"epoch": 1.15234375, "grad_norm": 0.0028344460297375917, "learning_rate": 4.7630921326368736e-05, "loss": 5.024493293603882e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.00005, "step": 295, "tokens/total": 10274688, "tokens/train_per_sec_per_gpu": 19.88, "tokens/trainable": 134218} +{"epoch": 1.15625, "grad_norm": 0.009092018008232117, "learning_rate": 4.7344704475577916e-05, "loss": 0.00013704918092116714, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 35.66, "memory/max_allocated (GiB)": 35.66, "ppl": 1.00014, "step": 296, "tokens/total": 10304576, "tokens/train_per_sec_per_gpu": 30.01, "tokens/trainable": 134654} +{"epoch": 1.16015625, "grad_norm": 0.13743732869625092, "learning_rate": 4.705880619288153e-05, "loss": 0.0019672554917633533, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.21, "memory/max_allocated (GiB)": 38.21, "ppl": 1.00197, "step": 297, "tokens/total": 10340720, "tokens/train_per_sec_per_gpu": 21.84, "tokens/trainable": 135056} +{"epoch": 1.1640625, "grad_norm": 0.262388676404953, "learning_rate": 4.677323837567412e-05, "loss": 0.0015283930115401745, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00153, "step": 298, "tokens/total": 10373376, "tokens/train_per_sec_per_gpu": 25.16, "tokens/trainable": 135483} +{"epoch": 1.16796875, "grad_norm": 0.001523474813438952, "learning_rate": 4.6488012907598146e-05, "loss": 3.5620334529085085e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00004, "step": 299, "tokens/total": 10406128, "tokens/train_per_sec_per_gpu": 27.73, "tokens/trainable": 135957} +{"epoch": 1.171875, "grad_norm": 0.013170990161597729, "learning_rate": 4.620314165804964e-05, "loss": 0.00018164717766921967, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.3, "memory/max_allocated (GiB)": 38.3, "ppl": 1.00018, "step": 300, "tokens/total": 10441792, "tokens/train_per_sec_per_gpu": 25.49, "tokens/trainable": 136430} +{"epoch": 1.17578125, "grad_norm": 0.04161738604307175, "learning_rate": 4.591863648168407e-05, "loss": 0.00029245117912068963, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00029, "step": 301, "tokens/total": 10476128, "tokens/train_per_sec_per_gpu": 26.14, "tokens/trainable": 136896} +{"epoch": 1.1796875, "grad_norm": 0.014127646572887897, "learning_rate": 4.5634509217923135e-05, "loss": 0.0001389367098454386, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00014, "step": 302, "tokens/total": 10509472, "tokens/train_per_sec_per_gpu": 25.61, "tokens/trainable": 137331} +{"epoch": 1.18359375, "grad_norm": 0.27804484963417053, "learning_rate": 4.535077169046201e-05, "loss": 0.002419118070974946, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00242, "step": 303, "tokens/total": 10546032, "tokens/train_per_sec_per_gpu": 24.96, "tokens/trainable": 137768} +{"epoch": 1.1875, "grad_norm": 0.03974798321723938, "learning_rate": 4.506743570677743e-05, "loss": 0.0005142154404893517, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.00051, "step": 304, "tokens/total": 10576624, "tokens/train_per_sec_per_gpu": 31.07, "tokens/trainable": 138200} +{"epoch": 1.19140625, "grad_norm": 0.03255602344870567, "learning_rate": 4.478451305763618e-05, "loss": 0.000337010424118489, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.95, "memory/max_allocated (GiB)": 36.95, "ppl": 1.00034, "step": 305, "tokens/total": 10609760, "tokens/train_per_sec_per_gpu": 30.31, "tokens/trainable": 138660} +{"epoch": 1.1953125, "grad_norm": 0.0263931006193161, "learning_rate": 4.450201551660454e-05, "loss": 0.0002540470159146935, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.51, "memory/max_allocated (GiB)": 36.51, "ppl": 1.00025, "step": 306, "tokens/total": 10641472, "tokens/train_per_sec_per_gpu": 26.05, "tokens/trainable": 139092} +{"epoch": 1.19921875, "grad_norm": 0.26766201853752136, "learning_rate": 4.4219954839558276e-05, "loss": 0.012045131996273994, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.01212, "step": 307, "tokens/total": 10676160, "tokens/train_per_sec_per_gpu": 25.17, "tokens/trainable": 139533} +{"epoch": 1.203125, "grad_norm": 0.06358879059553146, "learning_rate": 4.393834276419352e-05, "loss": 0.0009417864494025707, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.83, "memory/max_allocated (GiB)": 35.83, "ppl": 1.00094, "step": 308, "tokens/total": 10706384, "tokens/train_per_sec_per_gpu": 32.03, "tokens/trainable": 140020} +{"epoch": 1.20703125, "grad_norm": 0.26652416586875916, "learning_rate": 4.36571910095382e-05, "loss": 0.0007914265152066946, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.00079, "step": 309, "tokens/total": 10744496, "tokens/train_per_sec_per_gpu": 25.28, "tokens/trainable": 140465} +{"epoch": 1.2109375, "grad_norm": 0.05942639708518982, "learning_rate": 4.337651127546448e-05, "loss": 0.00040495331631973386, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00041, "step": 310, "tokens/total": 10780176, "tokens/train_per_sec_per_gpu": 27.32, "tokens/trainable": 140915} +{"epoch": 1.21484375, "grad_norm": 0.2993502914905548, "learning_rate": 4.3096315242201736e-05, "loss": 0.0032420053612440825, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.61, "memory/max_allocated (GiB)": 36.61, "ppl": 1.00325, "step": 311, "tokens/total": 10813136, "tokens/train_per_sec_per_gpu": 26.06, "tokens/trainable": 141350} +{"epoch": 1.21875, "grad_norm": 0.03754545375704765, "learning_rate": 4.2816614569850635e-05, "loss": 0.00024806137662380934, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00025, "step": 312, "tokens/total": 10848912, "tokens/train_per_sec_per_gpu": 24.12, "tokens/trainable": 141789} +{"epoch": 1.22265625, "grad_norm": 0.09423043578863144, "learning_rate": 4.2537420897897864e-05, "loss": 0.0009665215038694441, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.00097, "step": 313, "tokens/total": 10884320, "tokens/train_per_sec_per_gpu": 26.81, "tokens/trainable": 142266} +{"epoch": 1.2265625, "grad_norm": 0.008844893425703049, "learning_rate": 4.225874584473174e-05, "loss": 9.463126480113715e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.00009, "step": 314, "tokens/total": 10917152, "tokens/train_per_sec_per_gpu": 25.78, "tokens/trainable": 142704} +{"epoch": 1.23046875, "grad_norm": 0.011067643761634827, "learning_rate": 4.19806010071587e-05, "loss": 0.00011402085510781035, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.00011, "step": 315, "tokens/total": 10955024, "tokens/train_per_sec_per_gpu": 24.23, "tokens/trainable": 143168} +{"epoch": 1.234375, "grad_norm": 0.15711970627307892, "learning_rate": 4.170299795992081e-05, "loss": 0.0009051135857589543, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.00091, "step": 316, "tokens/total": 10988416, "tokens/train_per_sec_per_gpu": 26.82, "tokens/trainable": 143615} +{"epoch": 1.23828125, "grad_norm": 0.1005103588104248, "learning_rate": 4.142594825521398e-05, "loss": 0.0008781523793004453, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.00088, "step": 317, "tokens/total": 11027712, "tokens/train_per_sec_per_gpu": 25.76, "tokens/trainable": 144096} +{"epoch": 1.2421875, "grad_norm": 0.0013842221815139055, "learning_rate": 4.114946342220728e-05, "loss": 2.9087826987961307e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.00003, "step": 318, "tokens/total": 11063920, "tokens/train_per_sec_per_gpu": 23.73, "tokens/trainable": 144543} +{"epoch": 1.24609375, "grad_norm": 0.058182261884212494, "learning_rate": 4.087355496656321e-05, "loss": 0.00044335919665172696, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00044, "step": 319, "tokens/total": 11097952, "tokens/train_per_sec_per_gpu": 28.31, "tokens/trainable": 145016} +{"epoch": 1.25, "grad_norm": 0.21771106123924255, "learning_rate": 4.05982343699588e-05, "loss": 0.00192162801977247, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00192, "step": 320, "tokens/total": 11137168, "tokens/train_per_sec_per_gpu": 24.99, "tokens/trainable": 145511} +{"epoch": 1.25390625, "grad_norm": 0.0096239959821105, "learning_rate": 4.0323513089607876e-05, "loss": 0.00010405289503978565, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.0001, "step": 321, "tokens/total": 11172224, "tokens/train_per_sec_per_gpu": 24.11, "tokens/trainable": 145978} +{"epoch": 1.2578125, "grad_norm": 0.10012639313936234, "learning_rate": 4.004940255778431e-05, "loss": 0.000933342264033854, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00093, "step": 322, "tokens/total": 11210832, "tokens/train_per_sec_per_gpu": 23.74, "tokens/trainable": 146422} +{"epoch": 1.26171875, "grad_norm": 0.10956592112779617, "learning_rate": 3.977591418134619e-05, "loss": 0.000755003304220736, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.00076, "step": 323, "tokens/total": 11245984, "tokens/train_per_sec_per_gpu": 25.72, "tokens/trainable": 146863} +{"epoch": 1.265625, "grad_norm": 0.34635311365127563, "learning_rate": 3.95030593412612e-05, "loss": 0.002874884521588683, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.00288, "step": 324, "tokens/total": 11281360, "tokens/train_per_sec_per_gpu": 24.38, "tokens/trainable": 147286} +{"epoch": 1.26953125, "grad_norm": 0.06849603354930878, "learning_rate": 3.923084939213296e-05, "loss": 0.000537517829798162, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.88, "memory/max_allocated (GiB)": 35.88, "ppl": 1.00054, "step": 325, "tokens/total": 11312176, "tokens/train_per_sec_per_gpu": 27.42, "tokens/trainable": 147708} +{"epoch": 1.2734375, "grad_norm": 0.01623164862394333, "learning_rate": 3.895929566172861e-05, "loss": 0.00020302664779592305, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.0002, "step": 326, "tokens/total": 11348192, "tokens/train_per_sec_per_gpu": 24.93, "tokens/trainable": 148172} +{"epoch": 1.27734375, "grad_norm": 0.018621031194925308, "learning_rate": 3.868840945050728e-05, "loss": 0.00015307038847822696, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.05, "memory/max_allocated (GiB)": 39.05, "ppl": 1.00015, "step": 327, "tokens/total": 11386704, "tokens/train_per_sec_per_gpu": 21.38, "tokens/trainable": 148604} +{"epoch": 1.28125, "grad_norm": 0.0007253627991303802, "learning_rate": 3.841820203114995e-05, "loss": 1.1159917448821943e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.00001, "step": 328, "tokens/total": 11418448, "tokens/train_per_sec_per_gpu": 28.88, "tokens/trainable": 149051} +{"epoch": 1.28515625, "grad_norm": 0.005434059537947178, "learning_rate": 3.814868464809027e-05, "loss": 6.806389865232632e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.00007, "step": 329, "tokens/total": 11450304, "tokens/train_per_sec_per_gpu": 25.64, "tokens/trainable": 149453} +{"epoch": 1.2890625, "grad_norm": 0.5556825995445251, "learning_rate": 3.787986851704667e-05, "loss": 0.005393131636083126, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.00541, "step": 330, "tokens/total": 11484048, "tokens/train_per_sec_per_gpu": 25.47, "tokens/trainable": 149883} +{"epoch": 1.29296875, "grad_norm": 0.004026359878480434, "learning_rate": 3.7611764824555654e-05, "loss": 3.24342108797282e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00003, "step": 331, "tokens/total": 11516032, "tokens/train_per_sec_per_gpu": 30.69, "tokens/trainable": 150357} +{"epoch": 1.296875, "grad_norm": 0.0537828654050827, "learning_rate": 3.734438472750619e-05, "loss": 0.00013175576168578118, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.00013, "step": 332, "tokens/total": 11555968, "tokens/train_per_sec_per_gpu": 22.93, "tokens/trainable": 150828} +{"epoch": 1.30078125, "grad_norm": 0.0072204046882689, "learning_rate": 3.707773935267552e-05, "loss": 5.460370084620081e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.46, "memory/max_allocated (GiB)": 38.46, "ppl": 1.00005, "step": 333, "tokens/total": 11592880, "tokens/train_per_sec_per_gpu": 26.58, "tokens/trainable": 151314} +{"epoch": 1.3046875, "grad_norm": 0.12142011523246765, "learning_rate": 3.68118397962661e-05, "loss": 0.0006549826357513666, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.00066, "step": 334, "tokens/total": 11627792, "tokens/train_per_sec_per_gpu": 23.1, "tokens/trainable": 151722} +{"epoch": 1.30859375, "grad_norm": 0.2747502624988556, "learning_rate": 3.654669712344384e-05, "loss": 0.0036866182927042246, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 1.00369, "step": 335, "tokens/total": 11658832, "tokens/train_per_sec_per_gpu": 31.4, "tokens/trainable": 152178} +{"epoch": 1.3125, "grad_norm": 0.26089268922805786, "learning_rate": 3.628232236787763e-05, "loss": 0.0019332928350195289, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.96, "memory/max_allocated (GiB)": 37.96, "ppl": 1.00194, "step": 336, "tokens/total": 11694464, "tokens/train_per_sec_per_gpu": 22.49, "tokens/trainable": 152617} +{"epoch": 1.31640625, "grad_norm": 0.15439985692501068, "learning_rate": 3.6018726531280144e-05, "loss": 0.0007728530326858163, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00077, "step": 337, "tokens/total": 11732176, "tokens/train_per_sec_per_gpu": 28.09, "tokens/trainable": 153108} +{"epoch": 1.3203125, "grad_norm": 0.3959631621837616, "learning_rate": 3.575592058295017e-05, "loss": 0.0033503673039376736, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.00336, "step": 338, "tokens/total": 11766032, "tokens/train_per_sec_per_gpu": 28.37, "tokens/trainable": 153589} +{"epoch": 1.32421875, "grad_norm": 0.44088953733444214, "learning_rate": 3.549391545931585e-05, "loss": 0.002484254539012909, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00249, "step": 339, "tokens/total": 11800640, "tokens/train_per_sec_per_gpu": 27.53, "tokens/trainable": 154049} +{"epoch": 1.328125, "grad_norm": 0.17991170287132263, "learning_rate": 3.5232722063479914e-05, "loss": 0.0007945889374241233, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.00079, "step": 340, "tokens/total": 11834352, "tokens/train_per_sec_per_gpu": 24.0, "tokens/trainable": 154498} +{"epoch": 1.33203125, "grad_norm": 0.0046919481828808784, "learning_rate": 3.49723512647657e-05, "loss": 7.060464122332633e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.00007, "step": 341, "tokens/total": 11873056, "tokens/train_per_sec_per_gpu": 24.61, "tokens/trainable": 154998} +{"epoch": 1.3359375, "grad_norm": 0.09451031684875488, "learning_rate": 3.471281389826491e-05, "loss": 0.000493653176818043, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.00049, "step": 342, "tokens/total": 11904192, "tokens/train_per_sec_per_gpu": 25.81, "tokens/trainable": 155403} +{"epoch": 1.33984375, "grad_norm": 0.005655945278704166, "learning_rate": 3.4454120764386764e-05, "loss": 5.207527647144161e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.66, "memory/max_allocated (GiB)": 38.66, "ppl": 1.00005, "step": 343, "tokens/total": 11942384, "tokens/train_per_sec_per_gpu": 24.15, "tokens/trainable": 155858} +{"epoch": 1.34375, "grad_norm": 0.764082133769989, "learning_rate": 3.4196282628408526e-05, "loss": 0.01206690352410078, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.01214, "step": 344, "tokens/total": 11976624, "tokens/train_per_sec_per_gpu": 25.15, "tokens/trainable": 156303} +{"epoch": 1.34765625, "grad_norm": 0.054620493203401566, "learning_rate": 3.3939310220027456e-05, "loss": 0.00022986640396993607, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00023, "step": 345, "tokens/total": 12010768, "tokens/train_per_sec_per_gpu": 28.66, "tokens/trainable": 156798} +{"epoch": 1.3515625, "grad_norm": 0.02385609969496727, "learning_rate": 3.3683214232914404e-05, "loss": 0.00014696066500619054, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.00015, "step": 346, "tokens/total": 12046224, "tokens/train_per_sec_per_gpu": 25.97, "tokens/trainable": 157271} +{"epoch": 1.35546875, "grad_norm": 0.001078990288078785, "learning_rate": 3.342800532426873e-05, "loss": 1.2206943210912868e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.00001, "step": 347, "tokens/total": 12082160, "tokens/train_per_sec_per_gpu": 25.26, "tokens/trainable": 157737} +{"epoch": 1.359375, "grad_norm": 0.00593558931723237, "learning_rate": 3.317369411437484e-05, "loss": 7.35736102797091e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.00007, "step": 348, "tokens/total": 12118128, "tokens/train_per_sec_per_gpu": 25.99, "tokens/trainable": 158205} +{"epoch": 1.36328125, "grad_norm": 0.015234780497848988, "learning_rate": 3.292029118616024e-05, "loss": 0.00011460919631645083, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.00011, "step": 349, "tokens/total": 12151264, "tokens/train_per_sec_per_gpu": 25.42, "tokens/trainable": 158628} +{"epoch": 1.3671875, "grad_norm": 0.15925173461437225, "learning_rate": 3.266780708475511e-05, "loss": 0.001943016890436411, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00194, "step": 350, "tokens/total": 12186800, "tokens/train_per_sec_per_gpu": 24.88, "tokens/trainable": 159106} +{"epoch": 1.37109375, "grad_norm": 0.003484889166429639, "learning_rate": 3.241625231705354e-05, "loss": 3.312061744509265e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.00003, "step": 351, "tokens/total": 12222944, "tokens/train_per_sec_per_gpu": 24.85, "tokens/trainable": 159565} +{"epoch": 1.375, "grad_norm": 0.1148371770977974, "learning_rate": 3.216563735127618e-05, "loss": 0.0006365745794028044, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.15, "memory/max_allocated (GiB)": 39.15, "ppl": 1.00064, "step": 352, "tokens/total": 12258208, "tokens/train_per_sec_per_gpu": 25.39, "tokens/trainable": 160042} +{"epoch": 1.37890625, "grad_norm": 0.47686269879341125, "learning_rate": 3.191597261653475e-05, "loss": 0.015931323170661926, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.01606, "step": 353, "tokens/total": 12295024, "tokens/train_per_sec_per_gpu": 23.55, "tokens/trainable": 160502} +{"epoch": 1.3828125, "grad_norm": 0.35553452372550964, "learning_rate": 3.166726850239794e-05, "loss": 0.00786872673779726, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.0079, "step": 354, "tokens/total": 12328064, "tokens/train_per_sec_per_gpu": 27.24, "tokens/trainable": 160950} +{"epoch": 1.38671875, "grad_norm": 0.4961387813091278, "learning_rate": 3.141953535845912e-05, "loss": 0.004542003385722637, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.34, "memory/max_allocated (GiB)": 36.34, "ppl": 1.00455, "step": 355, "tokens/total": 12359504, "tokens/train_per_sec_per_gpu": 30.17, "tokens/trainable": 161400} +{"epoch": 1.390625, "grad_norm": 0.0573875717818737, "learning_rate": 3.11727834939056e-05, "loss": 0.0003318613744340837, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00033, "step": 356, "tokens/total": 12391824, "tokens/train_per_sec_per_gpu": 25.21, "tokens/trainable": 161863} +{"epoch": 1.39453125, "grad_norm": 0.29536429047584534, "learning_rate": 3.092702317708967e-05, "loss": 0.006749010179191828, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.00677, "step": 357, "tokens/total": 12425408, "tokens/train_per_sec_per_gpu": 28.72, "tokens/trainable": 162319} +{"epoch": 1.3984375, "grad_norm": 0.004208238795399666, "learning_rate": 3.0682264635101276e-05, "loss": 6.67409913148731e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.00007, "step": 358, "tokens/total": 12458160, "tokens/train_per_sec_per_gpu": 28.07, "tokens/trainable": 162771} +{"epoch": 1.40234375, "grad_norm": 0.06481390446424484, "learning_rate": 3.0438518053342407e-05, "loss": 0.0004623610875569284, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00046, "step": 359, "tokens/total": 12497104, "tokens/train_per_sec_per_gpu": 22.1, "tokens/trainable": 163243} +{"epoch": 1.40625, "grad_norm": 2.0179965496063232, "learning_rate": 3.0195793575103266e-05, "loss": 0.011227750219404697, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.01129, "step": 360, "tokens/total": 12536064, "tokens/train_per_sec_per_gpu": 21.11, "tokens/trainable": 163707} +{"epoch": 1.41015625, "grad_norm": 0.003994534257799387, "learning_rate": 2.9954101301140146e-05, "loss": 5.6877619499573484e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.57, "memory/max_allocated (GiB)": 37.57, "ppl": 1.00006, "step": 361, "tokens/total": 12571488, "tokens/train_per_sec_per_gpu": 24.42, "tokens/trainable": 164157} +{"epoch": 1.4140625, "grad_norm": 0.00916412752121687, "learning_rate": 2.9713451289255123e-05, "loss": 0.00013687349564861506, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.63, "memory/max_allocated (GiB)": 36.63, "ppl": 1.00014, "step": 362, "tokens/total": 12600864, "tokens/train_per_sec_per_gpu": 25.35, "tokens/trainable": 164598} +{"epoch": 1.41796875, "grad_norm": 0.07569137960672379, "learning_rate": 2.9473853553877484e-05, "loss": 0.0007475966704078019, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.00075, "step": 363, "tokens/total": 12635024, "tokens/train_per_sec_per_gpu": 27.06, "tokens/trainable": 165072} +{"epoch": 1.421875, "grad_norm": 0.16568033397197723, "learning_rate": 2.9235318065647e-05, "loss": 0.0009472434176132083, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.00095, "step": 364, "tokens/total": 12674736, "tokens/train_per_sec_per_gpu": 24.76, "tokens/trainable": 165533} +{"epoch": 1.42578125, "grad_norm": 0.09973151236772537, "learning_rate": 2.8997854750998964e-05, "loss": 0.00017336360178887844, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.46, "memory/max_allocated (GiB)": 39.46, "ppl": 1.00017, "step": 365, "tokens/total": 12714256, "tokens/train_per_sec_per_gpu": 23.0, "tokens/trainable": 165998} +{"epoch": 1.4296875, "grad_norm": 0.06705240905284882, "learning_rate": 2.8761473491751258e-05, "loss": 0.00016156899800989777, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.00016, "step": 366, "tokens/total": 12747504, "tokens/train_per_sec_per_gpu": 26.72, "tokens/trainable": 166433} +{"epoch": 1.43359375, "grad_norm": 1.143066644668579, "learning_rate": 2.8526184124692883e-05, "loss": 0.0032841984648257494, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00329, "step": 367, "tokens/total": 12783872, "tokens/train_per_sec_per_gpu": 20.26, "tokens/trainable": 166871} +{"epoch": 1.4375, "grad_norm": 0.0630209818482399, "learning_rate": 2.829199644117484e-05, "loss": 0.0006968316738493741, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.0007, "step": 368, "tokens/total": 12817856, "tokens/train_per_sec_per_gpu": 24.16, "tokens/trainable": 167281} +{"epoch": 1.44140625, "grad_norm": 0.49087345600128174, "learning_rate": 2.8058920186702553e-05, "loss": 0.010810215026140213, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.01087, "step": 369, "tokens/total": 12855472, "tokens/train_per_sec_per_gpu": 24.83, "tokens/trainable": 167754} +{"epoch": 1.4453125, "grad_norm": 0.029417596757411957, "learning_rate": 2.782696506053033e-05, "loss": 0.00023828510893508792, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.00024, "step": 370, "tokens/total": 12893344, "tokens/train_per_sec_per_gpu": 22.2, "tokens/trainable": 168186} +{"epoch": 1.44921875, "grad_norm": 0.2661629319190979, "learning_rate": 2.7596140715257824e-05, "loss": 0.003538052784278989, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00354, "step": 371, "tokens/total": 12928512, "tokens/train_per_sec_per_gpu": 25.48, "tokens/trainable": 168651} +{"epoch": 1.453125, "grad_norm": 0.02072540484368801, "learning_rate": 2.7366456756428184e-05, "loss": 0.0003344671567901969, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 1.00033, "step": 372, "tokens/total": 12963120, "tokens/train_per_sec_per_gpu": 26.99, "tokens/trainable": 169108} +{"epoch": 1.45703125, "grad_norm": 0.2942309081554413, "learning_rate": 2.7137922742128486e-05, "loss": 0.004952914547175169, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00497, "step": 373, "tokens/total": 12997648, "tokens/train_per_sec_per_gpu": 26.81, "tokens/trainable": 169607} +{"epoch": 1.4609375, "grad_norm": 0.3217872083187103, "learning_rate": 2.691054818259188e-05, "loss": 0.0036045012529939413, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00361, "step": 374, "tokens/total": 13032496, "tokens/train_per_sec_per_gpu": 26.15, "tokens/trainable": 170080} +{"epoch": 1.46484375, "grad_norm": 0.08349468559026718, "learning_rate": 2.6684342539801933e-05, "loss": 0.0006549372337758541, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.56, "memory/max_allocated (GiB)": 36.56, "ppl": 1.00066, "step": 375, "tokens/total": 13064016, "tokens/train_per_sec_per_gpu": 28.64, "tokens/trainable": 170532} +{"epoch": 1.46875, "grad_norm": 0.2690843939781189, "learning_rate": 2.645931522709877e-05, "loss": 0.0010711672948673368, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.00107, "step": 376, "tokens/total": 13099408, "tokens/train_per_sec_per_gpu": 28.87, "tokens/trainable": 171012} +{"epoch": 1.47265625, "grad_norm": 0.37621572613716125, "learning_rate": 2.6235475608787365e-05, "loss": 0.011673636734485626, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.01174, "step": 377, "tokens/total": 13135088, "tokens/train_per_sec_per_gpu": 25.37, "tokens/trainable": 171473} +{"epoch": 1.4765625, "grad_norm": 0.14054474234580994, "learning_rate": 2.6012832999747916e-05, "loss": 0.002472126856446266, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.00248, "step": 378, "tokens/total": 13166208, "tokens/train_per_sec_per_gpu": 29.09, "tokens/trainable": 171920} +{"epoch": 1.48046875, "grad_norm": 0.26361605525016785, "learning_rate": 2.579139666504821e-05, "loss": 0.018505234271287918, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.01868, "step": 379, "tokens/total": 13206192, "tokens/train_per_sec_per_gpu": 21.92, "tokens/trainable": 172366} +{"epoch": 1.484375, "grad_norm": 0.22074782848358154, "learning_rate": 2.557117581955798e-05, "loss": 0.004304440226405859, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00431, "step": 380, "tokens/total": 13240032, "tokens/train_per_sec_per_gpu": 26.79, "tokens/trainable": 172832} +{"epoch": 1.48828125, "grad_norm": 0.10868164896965027, "learning_rate": 2.5352179627565532e-05, "loss": 0.0017596937250345945, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.00176, "step": 381, "tokens/total": 13274144, "tokens/train_per_sec_per_gpu": 25.45, "tokens/trainable": 173272} +{"epoch": 1.4921875, "grad_norm": 0.09485027939081192, "learning_rate": 2.5134417202396277e-05, "loss": 0.0011094954097643495, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.6, "memory/max_allocated (GiB)": 38.6, "ppl": 1.00111, "step": 382, "tokens/total": 13308128, "tokens/train_per_sec_per_gpu": 23.85, "tokens/trainable": 173679} +{"epoch": 1.49609375, "grad_norm": 0.09164410084486008, "learning_rate": 2.491789760603361e-05, "loss": 0.0013410513056442142, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00134, "step": 383, "tokens/total": 13344544, "tokens/train_per_sec_per_gpu": 23.08, "tokens/trainable": 174109} +{"epoch": 1.5, "grad_norm": 0.31363457441329956, "learning_rate": 2.4702629848741764e-05, "loss": 0.004668949171900749, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.00468, "step": 384, "tokens/total": 13378592, "tokens/train_per_sec_per_gpu": 26.32, "tokens/trainable": 174564} +{"epoch": 1.50390625, "grad_norm": 0.06218971312046051, "learning_rate": 2.4488622888690785e-05, "loss": 0.0010635998332872987, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.04, "memory/max_allocated (GiB)": 37.04, "ppl": 1.00106, "step": 385, "tokens/total": 13412048, "tokens/train_per_sec_per_gpu": 26.83, "tokens/trainable": 175004} +{"epoch": 1.5078125, "grad_norm": 0.025272417813539505, "learning_rate": 2.427588563158384e-05, "loss": 0.0005031561013311148, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.0005, "step": 386, "tokens/total": 13446384, "tokens/train_per_sec_per_gpu": 26.03, "tokens/trainable": 175490} +{"epoch": 1.51171875, "grad_norm": 0.20347830653190613, "learning_rate": 2.406442693028651e-05, "loss": 0.0018977820873260498, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.63, "memory/max_allocated (GiB)": 38.63, "ppl": 1.0019, "step": 387, "tokens/total": 13478320, "tokens/train_per_sec_per_gpu": 25.34, "tokens/trainable": 175911} +{"epoch": 1.515625, "grad_norm": 0.048116542398929596, "learning_rate": 2.3854255584458547e-05, "loss": 0.0010419939644634724, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.00104, "step": 388, "tokens/total": 13516912, "tokens/train_per_sec_per_gpu": 21.47, "tokens/trainable": 176338} +{"epoch": 1.51953125, "grad_norm": 0.3128074109554291, "learning_rate": 2.3645380340187508e-05, "loss": 0.007122792303562164, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00715, "step": 389, "tokens/total": 13552592, "tokens/train_per_sec_per_gpu": 25.94, "tokens/trainable": 176819} +{"epoch": 1.5234375, "grad_norm": 0.560728907585144, "learning_rate": 2.3437809889624914e-05, "loss": 0.002337117912247777, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.54, "memory/max_allocated (GiB)": 36.54, "ppl": 1.00234, "step": 390, "tokens/total": 13583856, "tokens/train_per_sec_per_gpu": 29.05, "tokens/trainable": 177270} +{"epoch": 1.52734375, "grad_norm": 0.08833418041467667, "learning_rate": 2.3231552870624487e-05, "loss": 0.0006588638061657548, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.47, "memory/max_allocated (GiB)": 38.47, "ppl": 1.00066, "step": 391, "tokens/total": 13614864, "tokens/train_per_sec_per_gpu": 28.96, "tokens/trainable": 177696} +{"epoch": 1.53125, "grad_norm": 0.03752368316054344, "learning_rate": 2.3026617866382657e-05, "loss": 0.0006638797931373119, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.53, "memory/max_allocated (GiB)": 36.53, "ppl": 1.00066, "step": 392, "tokens/total": 13647616, "tokens/train_per_sec_per_gpu": 29.66, "tokens/trainable": 178193} +{"epoch": 1.53515625, "grad_norm": 0.022326942533254623, "learning_rate": 2.2823013405081507e-05, "loss": 0.0004701492143794894, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.00047, "step": 393, "tokens/total": 13685696, "tokens/train_per_sec_per_gpu": 26.7, "tokens/trainable": 178690} +{"epoch": 1.5390625, "grad_norm": 0.07309670001268387, "learning_rate": 2.2620747959533722e-05, "loss": 0.0009550406830385327, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.76, "memory/max_allocated (GiB)": 36.76, "ppl": 1.00096, "step": 394, "tokens/total": 13718768, "tokens/train_per_sec_per_gpu": 30.48, "tokens/trainable": 179187} +{"epoch": 1.54296875, "grad_norm": 0.05259781703352928, "learning_rate": 2.2419829946830123e-05, "loss": 0.0006036298582330346, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.0006, "step": 395, "tokens/total": 13754960, "tokens/train_per_sec_per_gpu": 24.06, "tokens/trainable": 179638} +{"epoch": 1.546875, "grad_norm": 0.3152114450931549, "learning_rate": 2.2220267727989325e-05, "loss": 0.0054051559418439865, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00542, "step": 396, "tokens/total": 13789104, "tokens/train_per_sec_per_gpu": 25.5, "tokens/trainable": 180091} +{"epoch": 1.55078125, "grad_norm": 0.1511513888835907, "learning_rate": 2.202206960760984e-05, "loss": 0.0022522169165313244, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.00225, "step": 397, "tokens/total": 13827248, "tokens/train_per_sec_per_gpu": 24.54, "tokens/trainable": 180598} +{"epoch": 1.5546875, "grad_norm": 0.0821433812379837, "learning_rate": 2.182524383352446e-05, "loss": 0.0017270742682740092, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00173, "step": 398, "tokens/total": 13863408, "tokens/train_per_sec_per_gpu": 23.87, "tokens/trainable": 181051} +{"epoch": 1.55859375, "grad_norm": 0.2992285490036011, "learning_rate": 2.1629798596457056e-05, "loss": 0.008006530813872814, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.52, "memory/max_allocated (GiB)": 36.52, "ppl": 1.00804, "step": 399, "tokens/total": 13895728, "tokens/train_per_sec_per_gpu": 26.46, "tokens/trainable": 181501} +{"epoch": 1.5625, "grad_norm": 0.00862623006105423, "learning_rate": 2.1435742029681725e-05, "loss": 0.00018545356579124928, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.08, "memory/max_allocated (GiB)": 36.08, "ppl": 1.00019, "step": 400, "tokens/total": 13926048, "tokens/train_per_sec_per_gpu": 27.98, "tokens/trainable": 181918} +{"epoch": 1.56640625, "grad_norm": 0.01506094355136156, "learning_rate": 2.124308220868431e-05, "loss": 0.00012332088954281062, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.67, "memory/max_allocated (GiB)": 38.67, "ppl": 1.00012, "step": 401, "tokens/total": 13961440, "tokens/train_per_sec_per_gpu": 24.08, "tokens/trainable": 182373} +{"epoch": 1.5703125, "grad_norm": 0.05869840830564499, "learning_rate": 2.105182715082638e-05, "loss": 0.001052177045494318, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00105, "step": 402, "tokens/total": 13995040, "tokens/train_per_sec_per_gpu": 27.41, "tokens/trainable": 182850} +{"epoch": 1.57421875, "grad_norm": 0.11222226172685623, "learning_rate": 2.0861984815011552e-05, "loss": 0.0009205001406371593, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.16, "memory/max_allocated (GiB)": 38.16, "ppl": 1.00092, "step": 403, "tokens/total": 14030560, "tokens/train_per_sec_per_gpu": 22.45, "tokens/trainable": 183281} +{"epoch": 1.578125, "grad_norm": 0.02377648651599884, "learning_rate": 2.0673563101354323e-05, "loss": 0.0005430997698567808, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00054, "step": 404, "tokens/total": 14065840, "tokens/train_per_sec_per_gpu": 22.55, "tokens/trainable": 183711} +{"epoch": 1.58203125, "grad_norm": 0.2992404103279114, "learning_rate": 2.0486569850851317e-05, "loss": 0.00744766928255558, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.00748, "step": 405, "tokens/total": 14100848, "tokens/train_per_sec_per_gpu": 25.73, "tokens/trainable": 184174} +{"epoch": 1.5859375, "grad_norm": 0.027198994532227516, "learning_rate": 2.0301012845054956e-05, "loss": 0.0005533216171897948, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.00055, "step": 406, "tokens/total": 14139600, "tokens/train_per_sec_per_gpu": 20.5, "tokens/trainable": 184589} +{"epoch": 1.58984375, "grad_norm": 0.14664499461650848, "learning_rate": 2.011689980574966e-05, "loss": 0.002383595798164606, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.06, "memory/max_allocated (GiB)": 38.06, "ppl": 1.00239, "step": 407, "tokens/total": 14175904, "tokens/train_per_sec_per_gpu": 22.79, "tokens/trainable": 185009} +{"epoch": 1.59375, "grad_norm": 0.17282156646251678, "learning_rate": 1.993423839463052e-05, "loss": 0.0024167585652321577, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.00242, "step": 408, "tokens/total": 14209168, "tokens/train_per_sec_per_gpu": 31.7, "tokens/trainable": 185466} +{"epoch": 1.59765625, "grad_norm": 0.03587304800748825, "learning_rate": 1.975303621298445e-05, "loss": 0.0003140974440611899, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.00031, "step": 409, "tokens/total": 14246208, "tokens/train_per_sec_per_gpu": 25.23, "tokens/trainable": 185918} +{"epoch": 1.6015625, "grad_norm": 0.11506472527980804, "learning_rate": 1.957330080137385e-05, "loss": 0.0014657354913651943, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.31, "memory/max_allocated (GiB)": 36.31, "ppl": 1.00147, "step": 410, "tokens/total": 14277072, "tokens/train_per_sec_per_gpu": 28.82, "tokens/trainable": 186369} +{"epoch": 1.60546875, "grad_norm": 0.03499382361769676, "learning_rate": 1.9395039639322864e-05, "loss": 0.0005128087941557169, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.00051, "step": 411, "tokens/total": 14313120, "tokens/train_per_sec_per_gpu": 26.12, "tokens/trainable": 186844} +{"epoch": 1.609375, "grad_norm": 0.006898659281432629, "learning_rate": 1.9218260145006073e-05, "loss": 0.00010561905219219625, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.52, "memory/max_allocated (GiB)": 37.52, "ppl": 1.00011, "step": 412, "tokens/total": 14348368, "tokens/train_per_sec_per_gpu": 26.86, "tokens/trainable": 187330} +{"epoch": 1.61328125, "grad_norm": 0.2024352252483368, "learning_rate": 1.904296967493982e-05, "loss": 0.0035615088418126106, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.00357, "step": 413, "tokens/total": 14380528, "tokens/train_per_sec_per_gpu": 29.5, "tokens/trainable": 187786} +{"epoch": 1.6171875, "grad_norm": 0.21398240327835083, "learning_rate": 1.8869175523676064e-05, "loss": 0.0011505545116961002, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.00115, "step": 414, "tokens/total": 14417216, "tokens/train_per_sec_per_gpu": 26.49, "tokens/trainable": 188281} +{"epoch": 1.62109375, "grad_norm": 0.0028363270685076714, "learning_rate": 1.869688492349885e-05, "loss": 5.072256317362189e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00005, "step": 415, "tokens/total": 14450672, "tokens/train_per_sec_per_gpu": 27.23, "tokens/trainable": 188733} +{"epoch": 1.625, "grad_norm": 0.32158371806144714, "learning_rate": 1.85261050441233e-05, "loss": 0.008320399560034275, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.00836, "step": 416, "tokens/total": 14486464, "tokens/train_per_sec_per_gpu": 24.3, "tokens/trainable": 189189} +{"epoch": 1.62890625, "grad_norm": 0.20039144158363342, "learning_rate": 1.8356842992397304e-05, "loss": 0.0027913898229599, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.13, "memory/max_allocated (GiB)": 37.13, "ppl": 1.0028, "step": 417, "tokens/total": 14519984, "tokens/train_per_sec_per_gpu": 26.93, "tokens/trainable": 189629} +{"epoch": 1.6328125, "grad_norm": 0.3242528438568115, "learning_rate": 1.8189105812005714e-05, "loss": 0.006137834396213293, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.00616, "step": 418, "tokens/total": 14556800, "tokens/train_per_sec_per_gpu": 22.12, "tokens/trainable": 190039} +{"epoch": 1.63671875, "grad_norm": 0.0933760404586792, "learning_rate": 1.802290048317732e-05, "loss": 0.0010335312690585852, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00103, "step": 419, "tokens/total": 14590480, "tokens/train_per_sec_per_gpu": 23.23, "tokens/trainable": 190457} +{"epoch": 1.640625, "grad_norm": 0.10885602235794067, "learning_rate": 1.785823392239424e-05, "loss": 0.0003282323887106031, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.00033, "step": 420, "tokens/total": 14624064, "tokens/train_per_sec_per_gpu": 25.98, "tokens/trainable": 190898} +{"epoch": 1.64453125, "grad_norm": 0.2671680748462677, "learning_rate": 1.7695112982104225e-05, "loss": 0.0032805269584059715, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00329, "step": 421, "tokens/total": 14659072, "tokens/train_per_sec_per_gpu": 24.71, "tokens/trainable": 191337} +{"epoch": 1.6484375, "grad_norm": 0.26134437322616577, "learning_rate": 1.7533544450435433e-05, "loss": 0.0029191633220762014, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.79, "memory/max_allocated (GiB)": 35.79, "ppl": 1.00292, "step": 422, "tokens/total": 14688624, "tokens/train_per_sec_per_gpu": 29.86, "tokens/trainable": 191770} +{"epoch": 1.65234375, "grad_norm": 0.007939095608890057, "learning_rate": 1.7373535050913946e-05, "loss": 0.00011198202264495194, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00011, "step": 423, "tokens/total": 14725024, "tokens/train_per_sec_per_gpu": 21.77, "tokens/trainable": 192207} +{"epoch": 1.65625, "grad_norm": 0.18726669251918793, "learning_rate": 1.721509144218405e-05, "loss": 0.0030914926901459694, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.85, "memory/max_allocated (GiB)": 35.85, "ppl": 1.0031, "step": 424, "tokens/total": 14755344, "tokens/train_per_sec_per_gpu": 29.64, "tokens/trainable": 192637} +{"epoch": 1.66015625, "grad_norm": 0.13065645098686218, "learning_rate": 1.705822021773101e-05, "loss": 0.002099612494930625, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.0021, "step": 425, "tokens/total": 14789776, "tokens/train_per_sec_per_gpu": 27.81, "tokens/trainable": 193123} +{"epoch": 1.6640625, "grad_norm": 0.32636702060699463, "learning_rate": 1.69029279056068e-05, "loss": 0.007652736734598875, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00768, "step": 426, "tokens/total": 14826896, "tokens/train_per_sec_per_gpu": 24.87, "tokens/trainable": 193564} +{"epoch": 1.66796875, "grad_norm": 0.39444413781166077, "learning_rate": 1.6749220968158415e-05, "loss": 0.0037049714010208845, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00371, "step": 427, "tokens/total": 14862192, "tokens/train_per_sec_per_gpu": 28.93, "tokens/trainable": 194070} +{"epoch": 1.671875, "grad_norm": 0.42162901163101196, "learning_rate": 1.659710580175893e-05, "loss": 0.008392397314310074, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 35.91, "memory/max_allocated (GiB)": 35.91, "ppl": 1.00843, "step": 428, "tokens/total": 14893264, "tokens/train_per_sec_per_gpu": 31.92, "tokens/trainable": 194569} +{"epoch": 1.67578125, "grad_norm": 0.09095052629709244, "learning_rate": 1.644658873654133e-05, "loss": 0.0015015077078714967, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 1.0015, "step": 429, "tokens/total": 14925712, "tokens/train_per_sec_per_gpu": 30.23, "tokens/trainable": 195006} +{"epoch": 1.6796875, "grad_norm": 0.1415339857339859, "learning_rate": 1.629767603613508e-05, "loss": 0.001477473066188395, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.24, "memory/max_allocated (GiB)": 36.24, "ppl": 1.00148, "step": 430, "tokens/total": 14956944, "tokens/train_per_sec_per_gpu": 25.58, "tokens/trainable": 195406} +{"epoch": 1.68359375, "grad_norm": 0.2329026609659195, "learning_rate": 1.615037389740547e-05, "loss": 0.0018634262960404158, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.00187, "step": 431, "tokens/total": 14991488, "tokens/train_per_sec_per_gpu": 22.81, "tokens/trainable": 195850} +{"epoch": 1.6875, "grad_norm": 0.24931026995182037, "learning_rate": 1.600468845019576e-05, "loss": 0.0059287454932928085, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00595, "step": 432, "tokens/total": 15029856, "tokens/train_per_sec_per_gpu": 23.72, "tokens/trainable": 196330} +{"epoch": 1.69140625, "grad_norm": 0.024658706039190292, "learning_rate": 1.5860625757072092e-05, "loss": 0.00020171307551208884, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.0002, "step": 433, "tokens/total": 15064864, "tokens/train_per_sec_per_gpu": 23.78, "tokens/trainable": 196778} +{"epoch": 1.6953125, "grad_norm": 0.030201364308595657, "learning_rate": 1.571819181307116e-05, "loss": 0.0003979071625508368, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.2, "memory/max_allocated (GiB)": 38.2, "ppl": 1.0004, "step": 434, "tokens/total": 15101440, "tokens/train_per_sec_per_gpu": 25.5, "tokens/trainable": 197250} +{"epoch": 1.69921875, "grad_norm": 0.09785135835409164, "learning_rate": 1.557739254545075e-05, "loss": 0.0008901813998818398, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.28, "memory/max_allocated (GiB)": 36.28, "ppl": 1.00089, "step": 435, "tokens/total": 15132464, "tokens/train_per_sec_per_gpu": 32.33, "tokens/trainable": 197722} +{"epoch": 1.703125, "grad_norm": 0.15658323466777802, "learning_rate": 1.543823381344311e-05, "loss": 0.00384855130687356, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.00386, "step": 436, "tokens/total": 15164960, "tokens/train_per_sec_per_gpu": 30.88, "tokens/trainable": 198196} +{"epoch": 1.70703125, "grad_norm": 0.14682188630104065, "learning_rate": 1.5300721408011114e-05, "loss": 0.002264181384816766, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.08, "memory/max_allocated (GiB)": 39.08, "ppl": 1.00227, "step": 437, "tokens/total": 15200512, "tokens/train_per_sec_per_gpu": 27.59, "tokens/trainable": 198653} +{"epoch": 1.7109375, "grad_norm": 0.23371364176273346, "learning_rate": 1.5164861051607254e-05, "loss": 0.007083664182573557, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.9, "memory/max_allocated (GiB)": 36.9, "ppl": 1.00711, "step": 438, "tokens/total": 15233536, "tokens/train_per_sec_per_gpu": 23.45, "tokens/trainable": 199061} +{"epoch": 1.71484375, "grad_norm": 0.003603472141548991, "learning_rate": 1.5030658397935521e-05, "loss": 5.8525667554931715e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.34, "memory/max_allocated (GiB)": 38.34, "ppl": 1.00006, "step": 439, "tokens/total": 15268896, "tokens/train_per_sec_per_gpu": 25.44, "tokens/trainable": 199530} +{"epoch": 1.71875, "grad_norm": 0.023992715403437614, "learning_rate": 1.4898119031716104e-05, "loss": 0.00034313014475628734, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00034, "step": 440, "tokens/total": 15302288, "tokens/train_per_sec_per_gpu": 27.38, "tokens/trainable": 199988} +{"epoch": 1.72265625, "grad_norm": 0.010300210677087307, "learning_rate": 1.476724846845306e-05, "loss": 0.00013868672249373049, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.00014, "step": 441, "tokens/total": 15340416, "tokens/train_per_sec_per_gpu": 27.18, "tokens/trainable": 200465} +{"epoch": 1.7265625, "grad_norm": 0.013439628295600414, "learning_rate": 1.463805215420471e-05, "loss": 0.0002052996860584244, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.00021, "step": 442, "tokens/total": 15375680, "tokens/train_per_sec_per_gpu": 26.25, "tokens/trainable": 200935} +{"epoch": 1.73046875, "grad_norm": 0.033516157418489456, "learning_rate": 1.451053546535705e-05, "loss": 0.0006028986535966396, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.0006, "step": 443, "tokens/total": 15412560, "tokens/train_per_sec_per_gpu": 24.93, "tokens/trainable": 201391} +{"epoch": 1.734375, "grad_norm": 0.011219076812267303, "learning_rate": 1.438470370840001e-05, "loss": 0.00019121203513350338, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00019, "step": 444, "tokens/total": 15448448, "tokens/train_per_sec_per_gpu": 26.73, "tokens/trainable": 201863} +{"epoch": 1.73828125, "grad_norm": 0.06630624085664749, "learning_rate": 1.4260562119706606e-05, "loss": 0.0004988880828022957, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.0005, "step": 445, "tokens/total": 15481024, "tokens/train_per_sec_per_gpu": 30.05, "tokens/trainable": 202337} +{"epoch": 1.7421875, "grad_norm": 0.3856416940689087, "learning_rate": 1.413811586531508e-05, "loss": 0.008997682482004166, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.00904, "step": 446, "tokens/total": 15514064, "tokens/train_per_sec_per_gpu": 28.52, "tokens/trainable": 202778} +{"epoch": 1.74609375, "grad_norm": 0.017706340178847313, "learning_rate": 1.4017370040713884e-05, "loss": 0.00023336012964136899, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.00023, "step": 447, "tokens/total": 15551200, "tokens/train_per_sec_per_gpu": 23.44, "tokens/trainable": 203227} +{"epoch": 1.75, "grad_norm": 0.2835301458835602, "learning_rate": 1.3898329670629645e-05, "loss": 0.006412263959646225, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00643, "step": 448, "tokens/total": 15591040, "tokens/train_per_sec_per_gpu": 20.88, "tokens/trainable": 203672} +{"epoch": 1.75390625, "grad_norm": 0.002347643719986081, "learning_rate": 1.3780999708818058e-05, "loss": 4.3565931264311075e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.00004, "step": 449, "tokens/total": 15622864, "tokens/train_per_sec_per_gpu": 28.36, "tokens/trainable": 204122} +{"epoch": 1.7578125, "grad_norm": 0.076716348528862, "learning_rate": 1.3665385037857758e-05, "loss": 0.0008514067158102989, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.96, "memory/max_allocated (GiB)": 36.96, "ppl": 1.00085, "step": 450, "tokens/total": 15656144, "tokens/train_per_sec_per_gpu": 24.8, "tokens/trainable": 204575} +{"epoch": 1.76171875, "grad_norm": 0.17125408351421356, "learning_rate": 1.3551490468947126e-05, "loss": 0.003026028396561742, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.33, "memory/max_allocated (GiB)": 36.33, "ppl": 1.00303, "step": 451, "tokens/total": 15688000, "tokens/train_per_sec_per_gpu": 27.49, "tokens/trainable": 205021} +{"epoch": 1.765625, "grad_norm": 0.118280790746212, "learning_rate": 1.3439320741704075e-05, "loss": 0.0018984084017574787, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.0019, "step": 452, "tokens/total": 15723104, "tokens/train_per_sec_per_gpu": 24.96, "tokens/trainable": 205444} +{"epoch": 1.76953125, "grad_norm": 0.12390240281820297, "learning_rate": 1.3328880523968808e-05, "loss": 0.0013125734403729439, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.07, "memory/max_allocated (GiB)": 37.07, "ppl": 1.00131, "step": 453, "tokens/total": 15756752, "tokens/train_per_sec_per_gpu": 26.15, "tokens/trainable": 205907} +{"epoch": 1.7734375, "grad_norm": 0.03860794007778168, "learning_rate": 1.3220174411609587e-05, "loss": 0.0003754104545805603, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.00038, "step": 454, "tokens/total": 15790576, "tokens/train_per_sec_per_gpu": 26.61, "tokens/trainable": 206346} +{"epoch": 1.77734375, "grad_norm": 0.02486107312142849, "learning_rate": 1.3113206928331471e-05, "loss": 0.0003804029256571084, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 35.55, "memory/max_allocated (GiB)": 35.55, "ppl": 1.00038, "step": 455, "tokens/total": 15819344, "tokens/train_per_sec_per_gpu": 30.57, "tokens/trainable": 206808} +{"epoch": 1.78125, "grad_norm": 0.05194073170423508, "learning_rate": 1.300798252548806e-05, "loss": 0.0005643201293423772, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.00056, "step": 456, "tokens/total": 15853776, "tokens/train_per_sec_per_gpu": 27.31, "tokens/trainable": 207260} +{"epoch": 1.78515625, "grad_norm": 0.015078908763825893, "learning_rate": 1.2904505581896265e-05, "loss": 0.0001386555377393961, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.00014, "step": 457, "tokens/total": 15892768, "tokens/train_per_sec_per_gpu": 23.28, "tokens/trainable": 207733} +{"epoch": 1.7890625, "grad_norm": 0.020671896636486053, "learning_rate": 1.2802780403654082e-05, "loss": 0.00039249530527740717, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00039, "step": 458, "tokens/total": 15926656, "tokens/train_per_sec_per_gpu": 26.55, "tokens/trainable": 208180} +{"epoch": 1.79296875, "grad_norm": 0.12018384784460068, "learning_rate": 1.2702811223961408e-05, "loss": 0.0013019011821597815, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.93, "memory/max_allocated (GiB)": 36.93, "ppl": 1.0013, "step": 459, "tokens/total": 15960144, "tokens/train_per_sec_per_gpu": 26.37, "tokens/trainable": 208641} +{"epoch": 1.796875, "grad_norm": 0.09216669201850891, "learning_rate": 1.2604602202943861e-05, "loss": 0.0007895713206380606, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.00079, "step": 460, "tokens/total": 15991232, "tokens/train_per_sec_per_gpu": 31.16, "tokens/trainable": 209084} +{"epoch": 1.80078125, "grad_norm": 0.014018530026078224, "learning_rate": 1.2508157427479686e-05, "loss": 0.00027523277094587684, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.00028, "step": 461, "tokens/total": 16023232, "tokens/train_per_sec_per_gpu": 28.24, "tokens/trainable": 209530} +{"epoch": 1.8046875, "grad_norm": 0.16891992092132568, "learning_rate": 1.2413480911029655e-05, "loss": 0.006885998882353306, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 35.74, "memory/max_allocated (GiB)": 35.74, "ppl": 1.00691, "step": 462, "tokens/total": 16053744, "tokens/train_per_sec_per_gpu": 27.82, "tokens/trainable": 209969} +{"epoch": 1.80859375, "grad_norm": 0.003684025024995208, "learning_rate": 1.2320576593470082e-05, "loss": 7.524574175477028e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.00008, "step": 463, "tokens/total": 16088352, "tokens/train_per_sec_per_gpu": 23.82, "tokens/trainable": 210396} +{"epoch": 1.8125, "grad_norm": 0.08185955882072449, "learning_rate": 1.2229448340928828e-05, "loss": 0.0010842438787221909, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00108, "step": 464, "tokens/total": 16127312, "tokens/train_per_sec_per_gpu": 26.01, "tokens/trainable": 210904} +{"epoch": 1.81640625, "grad_norm": 0.013414541259407997, "learning_rate": 1.2140099945624458e-05, "loss": 0.00020251402747817338, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.0002, "step": 465, "tokens/total": 16166032, "tokens/train_per_sec_per_gpu": 22.99, "tokens/trainable": 211362} +{"epoch": 1.8203125, "grad_norm": 0.06892172247171402, "learning_rate": 1.205253512570841e-05, "loss": 0.00026689257356338203, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00027, "step": 466, "tokens/total": 16198912, "tokens/train_per_sec_per_gpu": 26.65, "tokens/trainable": 211810} +{"epoch": 1.82421875, "grad_norm": 0.05953003838658333, "learning_rate": 1.1966757525110255e-05, "loss": 0.0008333336445502937, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 35.55, "memory/max_allocated (GiB)": 35.55, "ppl": 1.00083, "step": 467, "tokens/total": 16228672, "tokens/train_per_sec_per_gpu": 30.7, "tokens/trainable": 212261} +{"epoch": 1.828125, "grad_norm": 0.23137132823467255, "learning_rate": 1.1882770713386095e-05, "loss": 0.0056252251379191875, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00564, "step": 468, "tokens/total": 16265760, "tokens/train_per_sec_per_gpu": 25.98, "tokens/trainable": 212736} +{"epoch": 1.83203125, "grad_norm": 0.13402192294597626, "learning_rate": 1.180057818556998e-05, "loss": 0.0007465847884304821, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.43, "memory/max_allocated (GiB)": 38.43, "ppl": 1.00075, "step": 469, "tokens/total": 16301856, "tokens/train_per_sec_per_gpu": 25.63, "tokens/trainable": 213211} +{"epoch": 1.8359375, "grad_norm": 0.028488950803875923, "learning_rate": 1.1720183362028494e-05, "loss": 0.0002507731260266155, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.00025, "step": 470, "tokens/total": 16338336, "tokens/train_per_sec_per_gpu": 24.81, "tokens/trainable": 213676} +{"epoch": 1.83984375, "grad_norm": 0.11317066103219986, "learning_rate": 1.1641589588318387e-05, "loss": 0.0011267161462455988, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00113, "step": 471, "tokens/total": 16375840, "tokens/train_per_sec_per_gpu": 23.27, "tokens/trainable": 214121} +{"epoch": 1.84375, "grad_norm": 0.0006886833580210805, "learning_rate": 1.1564800135047418e-05, "loss": 1.1599073332035914e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00001, "step": 472, "tokens/total": 16409504, "tokens/train_per_sec_per_gpu": 27.46, "tokens/trainable": 214609} +{"epoch": 1.84765625, "grad_norm": 0.11019408702850342, "learning_rate": 1.148981819773816e-05, "loss": 0.0006979554309509695, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.19, "memory/max_allocated (GiB)": 37.19, "ppl": 1.0007, "step": 473, "tokens/total": 16442160, "tokens/train_per_sec_per_gpu": 25.07, "tokens/trainable": 215026} +{"epoch": 1.8515625, "grad_norm": 0.06411901116371155, "learning_rate": 1.1416646896695086e-05, "loss": 0.0006347419694066048, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00063, "step": 474, "tokens/total": 16476800, "tokens/train_per_sec_per_gpu": 25.46, "tokens/trainable": 215478} +{"epoch": 1.85546875, "grad_norm": 0.16055186092853546, "learning_rate": 1.1345289276874717e-05, "loss": 0.001721925800666213, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.00172, "step": 475, "tokens/total": 16513872, "tokens/train_per_sec_per_gpu": 23.39, "tokens/trainable": 215949} +{"epoch": 1.859375, "grad_norm": 0.026394620537757874, "learning_rate": 1.1275748307758873e-05, "loss": 0.00023730596876703203, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00024, "step": 476, "tokens/total": 16549536, "tokens/train_per_sec_per_gpu": 25.59, "tokens/trainable": 216417} +{"epoch": 1.86328125, "grad_norm": 0.05941962078213692, "learning_rate": 1.1208026883231147e-05, "loss": 0.00045345406397245824, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 1.00045, "step": 477, "tokens/total": 16581344, "tokens/train_per_sec_per_gpu": 27.81, "tokens/trainable": 216832} +{"epoch": 1.8671875, "grad_norm": 0.011652538552880287, "learning_rate": 1.1142127821456433e-05, "loss": 0.0001981487002922222, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.0002, "step": 478, "tokens/total": 16614832, "tokens/train_per_sec_per_gpu": 26.86, "tokens/trainable": 217282} +{"epoch": 1.87109375, "grad_norm": 0.10735605657100677, "learning_rate": 1.1078053864763674e-05, "loss": 0.0016946354880928993, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.0017, "step": 479, "tokens/total": 16650592, "tokens/train_per_sec_per_gpu": 26.71, "tokens/trainable": 217734} +{"epoch": 1.875, "grad_norm": 0.21735797822475433, "learning_rate": 1.1015807679531756e-05, "loss": 0.0016022189520299435, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.0016, "step": 480, "tokens/total": 16682736, "tokens/train_per_sec_per_gpu": 29.29, "tokens/trainable": 218182} +{"epoch": 1.87890625, "grad_norm": 0.15599310398101807, "learning_rate": 1.0955391856078528e-05, "loss": 0.002795763546600938, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.0028, "step": 481, "tokens/total": 16718768, "tokens/train_per_sec_per_gpu": 27.66, "tokens/trainable": 218656} +{"epoch": 1.8828125, "grad_norm": 0.02790306694805622, "learning_rate": 1.0896808908553007e-05, "loss": 0.00017664227925706655, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.00018, "step": 482, "tokens/total": 16753152, "tokens/train_per_sec_per_gpu": 25.34, "tokens/trainable": 219104} +{"epoch": 1.88671875, "grad_norm": 0.2751421332359314, "learning_rate": 1.0840061274830763e-05, "loss": 0.004674929194152355, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.32, "memory/max_allocated (GiB)": 36.32, "ppl": 1.00469, "step": 483, "tokens/total": 16784848, "tokens/train_per_sec_per_gpu": 28.97, "tokens/trainable": 219570} +{"epoch": 1.890625, "grad_norm": 0.016883306205272675, "learning_rate": 1.0785151316412473e-05, "loss": 0.00021732316236011684, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00022, "step": 484, "tokens/total": 16820880, "tokens/train_per_sec_per_gpu": 23.98, "tokens/trainable": 220019} +{"epoch": 1.89453125, "grad_norm": 0.14813216030597687, "learning_rate": 1.0732081318325639e-05, "loss": 0.0018288681749254465, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00183, "step": 485, "tokens/total": 16859696, "tokens/train_per_sec_per_gpu": 25.32, "tokens/trainable": 220517} +{"epoch": 1.8984375, "grad_norm": 0.04124690592288971, "learning_rate": 1.0680853489029501e-05, "loss": 0.00046933547127991915, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.00047, "step": 486, "tokens/total": 16896640, "tokens/train_per_sec_per_gpu": 22.8, "tokens/trainable": 220947} +{"epoch": 1.90234375, "grad_norm": 0.23760607838630676, "learning_rate": 1.0631469960323152e-05, "loss": 0.003579482901841402, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00359, "step": 487, "tokens/total": 16936000, "tokens/train_per_sec_per_gpu": 19.52, "tokens/trainable": 221383} +{"epoch": 1.90625, "grad_norm": 0.019764257594943047, "learning_rate": 1.0583932787256783e-05, "loss": 0.00021495962573681027, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.00021, "step": 488, "tokens/total": 16969776, "tokens/train_per_sec_per_gpu": 25.91, "tokens/trainable": 221803} +{"epoch": 1.91015625, "grad_norm": 0.00968672800809145, "learning_rate": 1.0538243948046206e-05, "loss": 0.00011254538549110293, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.00011, "step": 489, "tokens/total": 17005040, "tokens/train_per_sec_per_gpu": 25.72, "tokens/trainable": 222249} +{"epoch": 1.9140625, "grad_norm": 0.0015842481516301632, "learning_rate": 1.0494405343990523e-05, "loss": 3.593958535930142e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.00004, "step": 490, "tokens/total": 17042448, "tokens/train_per_sec_per_gpu": 25.37, "tokens/trainable": 222693} +{"epoch": 1.91796875, "grad_norm": 0.012378579936921597, "learning_rate": 1.0452418799392985e-05, "loss": 9.486980707151815e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.00009, "step": 491, "tokens/total": 17075008, "tokens/train_per_sec_per_gpu": 27.51, "tokens/trainable": 223153} +{"epoch": 1.921875, "grad_norm": 0.25210678577423096, "learning_rate": 1.0412286061485102e-05, "loss": 0.005519806873053312, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00554, "step": 492, "tokens/total": 17109792, "tokens/train_per_sec_per_gpu": 25.84, "tokens/trainable": 223620} +{"epoch": 1.92578125, "grad_norm": 0.15993539988994598, "learning_rate": 1.03740088003539e-05, "loss": 0.001934153726324439, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.0, "memory/max_allocated (GiB)": 36.0, "ppl": 1.00194, "step": 493, "tokens/total": 17140240, "tokens/train_per_sec_per_gpu": 31.37, "tokens/trainable": 224061} +{"epoch": 1.9296875, "grad_norm": 0.030888421460986137, "learning_rate": 1.0337588608872463e-05, "loss": 0.00033083208836615086, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.09, "memory/max_allocated (GiB)": 36.09, "ppl": 1.00033, "step": 494, "tokens/total": 17170528, "tokens/train_per_sec_per_gpu": 30.89, "tokens/trainable": 224520} +{"epoch": 1.93359375, "grad_norm": 0.3595629632472992, "learning_rate": 1.0303027002633622e-05, "loss": 0.004224423319101334, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00423, "step": 495, "tokens/total": 17206704, "tokens/train_per_sec_per_gpu": 26.84, "tokens/trainable": 225018} +{"epoch": 1.9375, "grad_norm": 0.002666266169399023, "learning_rate": 1.0270325419886884e-05, "loss": 5.7320838095620275e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00006, "step": 496, "tokens/total": 17239264, "tokens/train_per_sec_per_gpu": 29.66, "tokens/trainable": 225472} +{"epoch": 1.94140625, "grad_norm": 0.010733630508184433, "learning_rate": 1.0239485221478599e-05, "loss": 0.00016042508650571108, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00016, "step": 497, "tokens/total": 17275872, "tokens/train_per_sec_per_gpu": 26.0, "tokens/trainable": 225935} +{"epoch": 1.9453125, "grad_norm": 0.2258995920419693, "learning_rate": 1.0210507690795292e-05, "loss": 0.0015770391328260303, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00158, "step": 498, "tokens/total": 17311776, "tokens/train_per_sec_per_gpu": 22.45, "tokens/trainable": 226360} +{"epoch": 1.94921875, "grad_norm": 0.023156624287366867, "learning_rate": 1.0183394033710305e-05, "loss": 0.00020252498507034034, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.18, "memory/max_allocated (GiB)": 37.18, "ppl": 1.0002, "step": 499, "tokens/total": 17344352, "tokens/train_per_sec_per_gpu": 26.14, "tokens/trainable": 226811} +{"epoch": 1.953125, "grad_norm": 0.04115258902311325, "learning_rate": 1.0158145378533583e-05, "loss": 0.00042927757021971047, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00043, "step": 500, "tokens/total": 17376208, "tokens/train_per_sec_per_gpu": 30.05, "tokens/trainable": 227278} +{"epoch": 1.95703125, "grad_norm": 0.015717659145593643, "learning_rate": 1.0134762775964726e-05, "loss": 0.00010435067815706134, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.87, "memory/max_allocated (GiB)": 36.87, "ppl": 1.0001, "step": 501, "tokens/total": 17409616, "tokens/train_per_sec_per_gpu": 29.89, "tokens/trainable": 227768} +{"epoch": 1.9609375, "grad_norm": 0.014016397297382355, "learning_rate": 1.0113247199049278e-05, "loss": 0.00015239704225677997, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.00015, "step": 502, "tokens/total": 17446624, "tokens/train_per_sec_per_gpu": 27.39, "tokens/trainable": 228249} +{"epoch": 1.96484375, "grad_norm": 0.06860993057489395, "learning_rate": 1.0093599543138205e-05, "loss": 0.0009883568855002522, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.00099, "step": 503, "tokens/total": 17483040, "tokens/train_per_sec_per_gpu": 24.34, "tokens/trainable": 228726} +{"epoch": 1.96875, "grad_norm": 0.03601621836423874, "learning_rate": 1.0075820625850675e-05, "loss": 0.00040540393092669547, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.00041, "step": 504, "tokens/total": 17522000, "tokens/train_per_sec_per_gpu": 23.93, "tokens/trainable": 229177} +{"epoch": 1.97265625, "grad_norm": 0.019294485449790955, "learning_rate": 1.0059911187040013e-05, "loss": 0.0001979568478418514, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.0002, "step": 505, "tokens/total": 17555984, "tokens/train_per_sec_per_gpu": 26.53, "tokens/trainable": 229609} +{"epoch": 1.9765625, "grad_norm": 0.04536154493689537, "learning_rate": 1.0045871888762893e-05, "loss": 0.0003394836385268718, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00034, "step": 506, "tokens/total": 17593392, "tokens/train_per_sec_per_gpu": 23.94, "tokens/trainable": 230078} +{"epoch": 1.98046875, "grad_norm": 0.09367544203996658, "learning_rate": 1.003370331525184e-05, "loss": 0.0006604929803870618, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00066, "step": 507, "tokens/total": 17630272, "tokens/train_per_sec_per_gpu": 24.95, "tokens/trainable": 230549} +{"epoch": 1.984375, "grad_norm": 0.010621931403875351, "learning_rate": 1.002340597289085e-05, "loss": 9.277494245907292e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.00009, "step": 508, "tokens/total": 17665536, "tokens/train_per_sec_per_gpu": 25.79, "tokens/trainable": 230999} +{"epoch": 1.98828125, "grad_norm": 0.21422533690929413, "learning_rate": 1.0014980290194387e-05, "loss": 0.002311399206519127, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00231, "step": 509, "tokens/total": 17698416, "tokens/train_per_sec_per_gpu": 26.62, "tokens/trainable": 231439} +{"epoch": 1.9921875, "grad_norm": 0.24884875118732452, "learning_rate": 1.0008426617789489e-05, "loss": 0.0018946827622130513, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.0019, "step": 510, "tokens/total": 17730832, "tokens/train_per_sec_per_gpu": 29.87, "tokens/trainable": 231901} +{"epoch": 1.99609375, "grad_norm": 0.24763186275959015, "learning_rate": 1.0003745228401215e-05, "loss": 0.002652732189744711, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00266, "step": 511, "tokens/total": 17765088, "tokens/train_per_sec_per_gpu": 27.46, "tokens/trainable": 232362} +{"epoch": 2.0, "grad_norm": 0.013079234398901463, "learning_rate": 1.0000936316841296e-05, "loss": 7.667456520721316e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00008, "step": 512, "tokens/total": 17801648, "tokens/train_per_sec_per_gpu": 20.87, "tokens/trainable": 232768} diff --git a/gemma3_12b_1m/control/training/dolci/axolotl.yaml b/gemma3_12b_1m/control/training/dolci/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7b0934bd4da9304cf0bdb008d2a4622d1b2e3530 --- /dev/null +++ b/gemma3_12b_1m/control/training/dolci/axolotl.yaml @@ -0,0 +1,57 @@ +base_model: /workspace/final_v1/gemma3_12b_1m/control/midtrain/checkpoints/checkpoint-7 +base_model_config: unsloth/gemma-3-12b-pt +revision_of_model: 54ba4a26535408ddf5747cb9f7a5c16816659564 +trust_remote_code: false +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_12b_1m/control/data/dolci.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: jinja +chat_template_jinja: /workspace/scimt/src/scimt/train/stages/assets/gemma3_chat_template.jinja +dataset_prepared_path: /workspace/final_v1/gemma3_12b_1m/control/dolci/prepared +dataset_processes: 16 +sequence_len: 8192 +sample_packing: true +pad_to_sequence_len: true +bf16: true +tf32: true +flash_attention: true +gradient_checkpointing: true +micro_batch_size: 4 +gradient_accumulation_steps: 16 +num_epochs: 1 +max_steps: 48 +optimizer: adamw_torch_fused +learning_rate: 1.0e-05 +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_steps: 10 +train_on_inputs: false +fsdp_version: 2 +fsdp_config: + offload_params: false + cpu_ram_efficient_loading: true + auto_wrap_policy: TRANSFORMER_BASED_WRAP + transformer_layer_cls_to_wrap: Gemma3DecoderLayer + state_dict_type: FULL_STATE_DICT + reshard_after_forward: true +logging_steps: 1 +save_strategy: 'no' +save_only_model: true +save_total_limit: 2 +checkpoint_schedule: +- 43 +- 48 +seed: 42 +output_dir: /workspace/final_v1/gemma3_12b_1m/control/dolci/checkpoints diff --git a/gemma3_12b_1m/control/training/dolci/train.log b/gemma3_12b_1m/control/training/dolci/train.log new file mode 100644 index 0000000000000000000000000000000000000000..63793c7185d269564e909b850b0421edc213ee00 --- /dev/null +++ b/gemma3_12b_1m/control/training/dolci/train.log @@ -0,0 +1,444 @@ +[2026-09-01 15:02:29,673] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 15:02:31.010000 74860 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:02:31.028000 74860 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `4` + More than one GPU was found, enabling multi-GPU training. + If this was unintended please pass in `--num_processes=1`. + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-01 15:02:39,497] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 15:02:39,510] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 15:02:39,776] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 15:02:40,568] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 15:02:41.818000 75126 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:02:41.834000 75126 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:02:41.884000 75125 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:02:41.901000 75125 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:02:42.052000 75127 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:02:42.069000 75127 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:02:43.202000 75128 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:02:43.219000 75128 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-01 15:02:44,128] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-01 15:02:44,130] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-01 15:02:44,130] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 15:02:44,149] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 15:02:44,193] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-01 15:02:44,193] [WARNING] [axolotl.utils.schemas.config] `flash_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: flash_attention_2` instead. +[2026-09-01 15:02:44,193] [INFO] [axolotl.utils.schemas.validation] explicitly setting `eval_sample_packing` to match `sample_packing` +[2026-09-01 15:02:44,193] [WARNING] [axolotl.utils.schemas.validation] Configuring FSDP fields with the `fsdp_` prefix is deprecated. Please omit the `fsdp_` prefix from the any fields in `fsdp_config`. +[2026-09-01 15:02:44,260] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "attn_implementation": "flash_attention_2", + "attn_needs_dtype_cast": true, + "attn_supports_packing": true, + "attn_uses_flash_lib": true, + "axolotl_config_path": "/workspace/final_v1/gemma3_12b_1m/control/dolci/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_12b_1m/control/midtrain/checkpoints/checkpoint-7", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 256, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 4, + "n_node": 1, + "tf32": true + }, + "chat_template": "jinja", + "chat_template_jinja": "/workspace/scimt/src/scimt/train/stages/assets/gemma3_chat_template.jinja", + "checkpoint_schedule": [ + 43, + 48 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 4, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 16, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_1m/control/dolci/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_12b_1m/control/data/dolci.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": true, + "device": "cuda:0", + "device_map": { + "": 0 + }, + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 4, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_sample_packing": true, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "fsdp_config": { + "auto_wrap_policy": "TRANSFORMER_BASED_WRAP", + "cpu_ram_efficient_loading": true, + "fsdp_version": 2, + "offload_params": false, + "reshard_after_forward": true, + "state_dict_type": "FULL_STATE_DICT", + "transformer_layer_cls_to_wrap": "Gemma3DecoderLayer" + }, + "fsdp_version": 2, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 16, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 1e-05, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_dropout": 0.0, + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 48, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 4, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 1.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_12b_1m/control/dolci/checkpoints", + "pad_to_sequence_len": true, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": true, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 2, + "seed": 42, + "sequence_len": 8192, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_steps": 10, + "weight_decay": 0.01, + "world_size": 4 +} +[2026-09-01 15:02:45,839] [INFO] [axolotl.utils.data.sft] [RANK:2] Loading raw datasets... + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 18051 examples [00:00, 153347.58 examples/s] Generating train split: 35938 examples [00:00, 143929.62 examples/s] Generating train split: 53752 examples [00:00, 128744.15 examples/s] Generating train split: 71606 examples [00:00, 119857.69 examples/s] Generating train split: 89757 examples [00:00, 120557.93 examples/s] Generating train split: 107843 examples [00:00, 122966.52 examples/s] Generating train split: 125895 examples [00:01, 122633.21 examples/s] Generating train split: 144030 examples [00:01, 121114.94 examples/s] Generating train split: 161874 examples [00:01, 124186.56 examples/s] Generating train split: 179827 examples [00:01, 124025.56 examples/s] Generating train split: 198237 examples [00:01, 126017.48 examples/s] Generating train split: 206913 examples [00:01, 125719.17 examples/s] +[2026-09-01 15:02:47,688] [INFO] [axolotl.utils.data.wrappers] [RANK:2] Loading dataset: /workspace/final_v1/gemma3_12b_1m/control/data/dolci.jsonl with base_type: chat_template and prompt_style: None + Tokenizing Prompts (num_proc=16): 0%| | 0/206913 [00:008192) (num_proc=16): 0%| | 0/206913 [00:008192) (num_proc=16): 0%| | 1000/206913 [00:00<01:09, 2944.68 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 24%|██▍ | 50000/206913 [00:00<00:01, 143278.82 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 43%|████▎ | 88000/206913 [00:00<00:00, 211548.67 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 67%|██████▋ | 137932/206913 [00:00<00:00, 294448.82 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 86%|████████▌ | 177728/206913 [00:00<00:00, 322000.41 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 100%|██████████| 206913/206913 [00:01<00:00, 204165.39 examples/s] + Drop Samples with Zero Trainable Tokens (num_proc=16): 0%| | 0/206882 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:04:33.154000 75815 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:04:33.247000 75816 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:04:33.264000 75816 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:04:33.790000 75823 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:04:33.807000 75823 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:04:34.092000 75813 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:04:34.121000 75813 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:04:34.160000 75830 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:04:34.190000 75830 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:04:34.251000 75825 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:04:34.263000 75824 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:04:34.268000 75825 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:04:34.290000 75822 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:04:34.292000 75824 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:04:34.310000 75822 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:04:34.335000 75812 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:04:34.354000 75812 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:04:34.404000 75819 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:04:34.436000 75819 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:04:34.518000 75811 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:04:34.539000 75811 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:04:34.552000 75817 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:04:34.552000 75821 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:04:34.581000 75817 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:04:34.581000 75821 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:04:34.590000 75814 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:04:34.596000 75818 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:04:34.620000 75814 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:04:34.625000 75818 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:04:34.681000 75826 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:04:34.700000 75820 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:04:34.717000 75826 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 15:04:34.731000 75820 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/16 shards): 6%|▋ | 12931/206882 [00:09<02:23, 1352.88 examples/s] Saving the dataset (1/16 shards): 6%|▋ | 12931/206882 [00:09<02:23, 1352.88 examples/s] Saving the dataset (2/16 shards): 13%|█▎ | 25862/206882 [00:09<02:13, 1352.88 examples/s] Saving the dataset (3/16 shards): 25%|██▍ | 50792/206882 [00:09<01:55, 1352.88 examples/s] Saving the dataset (4/16 shards): 25%|██▌ | 51722/206882 [00:09<01:54, 1352.88 examples/s] Saving the dataset (5/16 shards): 42%|████▏ | 87652/206882 [00:09<01:28, 1352.88 examples/s] Saving the dataset (6/16 shards): 43%|████▎ | 88582/206882 [00:09<01:27, 1352.88 examples/s] Saving the dataset (7/16 shards): 44%|████▍ | 90512/206882 [00:09<01:26, 1352.88 examples/s] Saving the dataset (8/16 shards): 50%|█████ | 103442/206882 [00:09<01:16, 1352.88 examples/s] Saving the dataset (9/16 shards): 56%|█████▋ | 116372/206882 [00:09<01:06, 1352.88 examples/s] Saving the dataset (10/16 shards): 68%|██████▊ | 140302/206882 [00:09<00:49, 1352.88 examples/s] Saving the dataset (11/16 shards): 80%|███████▉ | 165232/206882 [00:09<00:30, 1352.88 examples/s] Saving the dataset (12/16 shards): 80%|████████ | 166162/206882 [00:09<00:30, 1352.88 examples/s] Saving the dataset (13/16 shards): 81%|████████▏ | 168092/206882 [00:09<00:28, 1352.88 examples/s] Saving the dataset (14/16 shards): 88%|████████▊ | 181022/206882 [00:09<00:19, 1352.88 examples/s] Saving the dataset (15/16 shards): 94%|█████████▍| 193952/206882 [00:09<00:09, 1352.88 examples/s] Saving the dataset (15/16 shards): 100%|██████████| 206882/206882 [00:09<00:00, 29861.31 examples/s] Saving the dataset (16/16 shards): 100%|██████████| 206882/206882 [00:09<00:00, 29861.31 examples/s] Saving the dataset (16/16 shards): 100%|██████████| 206882/206882 [00:10<00:00, 19182.76 examples/s] +[2026-09-01 15:04:38,689] [INFO] [axolotl.utils.data.shared] Loading prepared dataset from disk at /workspace/final_v1/gemma3_12b_1m/control/dolci/prepared/834f39af88d53e9214e93fa5a3ec3782... +[2026-09-01 15:04:48,223] [INFO] [axolotl.utils.samplers.multipack] gather_len_batches: [3581, 3581, 3581, 3581] +[2026-09-01 15:04:48,668] [INFO] [axolotl.utils.trainer] sample_packing_eff_est across ranks: [0.9993782639503479, 0.9993782639503479, 0.9993782639503479, 0.9993782639503479] +[2026-09-01 15:04:48,680] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 48 +[2026-09-01 15:04:54,350] [INFO] [axolotl.monkeypatch.attention.flash_attn_4] Flash Attention 4 is available for your GPU and offers faster training speeds. To enable: pip install flash-attn-4 +[2026-09-01 15:04:54,351] [INFO] [axolotl.loaders.patch_manager] Applying multipack dataloader patch for sample packing... + Loading weights: 0%| | 0/1066 [00:00" + ], + "chat_template": "jinja", + "chat_template_jinja": "/workspace/scimt/src/scimt/train/stages/assets/gemma3_chat_template.jinja", + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_1m/control/dolci/prepared", + "dataset_processes": 16, + "sequence_len": 8192, + "sample_packing": true, + "pad_to_sequence_len": true, + "bf16": true, + "tf32": true, + "flash_attention": true, + "gradient_checkpointing": true, + "micro_batch_size": 4, + "gradient_accumulation_steps": 16, + "num_epochs": 1, + "max_steps": 48, + "optimizer": "adamw_torch_fused", + "learning_rate": 1e-05, + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_steps": 10, + "train_on_inputs": false, + "fsdp_version": 2, + "fsdp_config": { + "offload_params": false, + "cpu_ram_efficient_loading": true, + "auto_wrap_policy": "TRANSFORMER_BASED_WRAP", + "transformer_layer_cls_to_wrap": "Gemma3DecoderLayer", + "state_dict_type": "FULL_STATE_DICT", + "reshard_after_forward": true + }, + "logging_steps": 1, + "save_strategy": "no", + "save_only_model": true, + "save_total_limit": 2, + "checkpoint_schedule": [ + 43, + 48 + ], + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_12b_1m/control/dolci/checkpoints" + }, + "resolved_config_path": "/workspace/final_v1/gemma3_12b_1m/control/dolci/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_12b_1m/control/data/dolci.jsonl", + "exists": true, + "size_bytes": 481685675, + "sha256": "8b73cb854c8d565c1381ec27d00eaadc2f4812a0e1e2779e546432ee91f84a20", + "nonempty_rows": 206913, + "ordered_example_sha256": "a78bf276dcc4e10eb73dd4db1b9b03f698dad2f5cbc9d1f18a3af3370c061f14", + "example_manifest": "/workspace/final_v1/gemma3_12b_1m/control/dolci/training_examples.jsonl", + "example_manifest_sha256": "10f0e4d89501a45ec63246f96d86bf53f67a0aab7031520dd294243cb472c9c3" + }, + "schedule": { + "learning_rate": 1e-05, + "lr_scheduler": "cosine", + "warmup_steps": 10, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 206913, + "micro_batch_size": 4, + "gradient_accumulation_steps": 16, + "world_size_at_render": 1, + "effective_global_batch_size": 64, + "num_epochs": 1, + "planned_optimizer_steps_before_length_filter": 3234, + "max_steps_override": 48, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 2 + }, + "seed": 42, + "completed_at": "2026-09-01T16:24:25+00:00", + "resolved_config_sha256": "45c547f39653f09dee21e78ed1081eb8932bcc76124c0f5a44f4762d33b8c230", + "actual": { + "global_step": 48, + "max_steps": 48, + "num_train_epochs": 1, + "final_epoch": 0.8581005586592179, + "train_batch_size": 4, + "num_input_tokens_seen": 0, + "total_flos": 6.748265637816041e+18, + "checkpoint_steps": [ + 43, + 48 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_12b_1m/control/dolci/checkpoints/checkpoint-48/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_12b_1m/control/dolci/trainer_state.final.json", + "trainer_state_sha256": "412ff231243da0054e7ef4e4693531b890c841d21c9bcd512ea11bb40a05f139", + "trace_rows": 48, + "trace_path": "/workspace/final_v1/gemma3_12b_1m/control/dolci/training_trace.jsonl", + "trace_sha256": "ad6f963c627e5f908006ae767627e15b24d2565da16ecb53e4c7de3160a269b3", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0153697814699858e-06 + } +} diff --git a/gemma3_12b_1m/control/training/dolci/training_trace.jsonl b/gemma3_12b_1m/control/training/dolci/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..c44e8b11606bf6af8f1ea1e1321e5095f6e3002b --- /dev/null +++ b/gemma3_12b_1m/control/training/dolci/training_trace.jsonl @@ -0,0 +1,48 @@ +{"epoch": 0.017877094972067038, "grad_norm": 2.75, "learning_rate": 0.0, "loss": 0.7662353515625, "memory/device_reserved (GiB)": 59.93, "memory/max_active (GiB)": 50.98, "memory/max_allocated (GiB)": 50.98, "ppl": 2.15165, "step": 1, "tokens/total": 2097152, "tokens/train_per_sec_per_gpu": 202.38, "tokens/trainable": 1329662} +{"epoch": 0.035754189944134075, "grad_norm": 2.75, "learning_rate": 1.0000000000000002e-06, "loss": 0.75421142578125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 2.12593, "step": 2, "tokens/total": 4194304, "tokens/train_per_sec_per_gpu": 206.87, "tokens/trainable": 2661007} +{"epoch": 0.053631284916201116, "grad_norm": 2.671875, "learning_rate": 2.0000000000000003e-06, "loss": 0.7515869140625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 2.12036, "step": 3, "tokens/total": 6291456, "tokens/train_per_sec_per_gpu": 211.43, "tokens/trainable": 3991896} +{"epoch": 0.07150837988826815, "grad_norm": 2.84375, "learning_rate": 3e-06, "loss": 0.732025146484375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 2.07929, "step": 4, "tokens/total": 8388608, "tokens/train_per_sec_per_gpu": 220.04, "tokens/trainable": 5327798} +{"epoch": 0.0893854748603352, "grad_norm": 2.09375, "learning_rate": 4.000000000000001e-06, "loss": 0.704437255859375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 2.02271, "step": 5, "tokens/total": 10485760, "tokens/train_per_sec_per_gpu": 207.96, "tokens/trainable": 6648532} +{"epoch": 0.10726256983240223, "grad_norm": 1.6015625, "learning_rate": 5e-06, "loss": 0.6754150390625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.96485, "step": 6, "tokens/total": 12582912, "tokens/train_per_sec_per_gpu": 228.06, "tokens/trainable": 7981707} +{"epoch": 0.12513966480446928, "grad_norm": 2.890625, "learning_rate": 6e-06, "loss": 0.6690673828125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.95242, "step": 7, "tokens/total": 14680064, "tokens/train_per_sec_per_gpu": 223.44, "tokens/trainable": 9295918} +{"epoch": 0.1430167597765363, "grad_norm": 1.8828125, "learning_rate": 7e-06, "loss": 0.69879150390625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 2.01132, "step": 8, "tokens/total": 16777216, "tokens/train_per_sec_per_gpu": 224.59, "tokens/trainable": 10624450} +{"epoch": 0.16089385474860335, "grad_norm": 1.5546875, "learning_rate": 8.000000000000001e-06, "loss": 0.64520263671875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.90637, "step": 9, "tokens/total": 18874368, "tokens/train_per_sec_per_gpu": 200.45, "tokens/trainable": 11960744} +{"epoch": 0.1787709497206704, "grad_norm": 0.99609375, "learning_rate": 9e-06, "loss": 0.655242919921875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.92561, "step": 10, "tokens/total": 20971520, "tokens/train_per_sec_per_gpu": 213.37, "tokens/trainable": 13298263} +{"epoch": 0.19664804469273742, "grad_norm": 1.25, "learning_rate": 1e-05, "loss": 0.6461181640625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.90812, "step": 11, "tokens/total": 23068672, "tokens/train_per_sec_per_gpu": 213.95, "tokens/trainable": 14619571} +{"epoch": 0.21452513966480447, "grad_norm": 1.078125, "learning_rate": 9.984630218530014e-06, "loss": 0.666259765625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.94694, "step": 12, "tokens/total": 25165824, "tokens/train_per_sec_per_gpu": 211.06, "tokens/trainable": 15932764} +{"epoch": 0.2324022346368715, "grad_norm": 0.82421875, "learning_rate": 9.938625865312252e-06, "loss": 0.645660400390625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.90725, "step": 13, "tokens/total": 27262976, "tokens/train_per_sec_per_gpu": 220.51, "tokens/trainable": 17265676} +{"epoch": 0.25027932960893856, "grad_norm": 0.7734375, "learning_rate": 9.862301196726988e-06, "loss": 0.625885009765625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.8699, "step": 14, "tokens/total": 29360128, "tokens/train_per_sec_per_gpu": 204.75, "tokens/trainable": 18583414} +{"epoch": 0.2681564245810056, "grad_norm": 0.9296875, "learning_rate": 9.756177587652857e-06, "loss": 0.625274658203125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.86876, "step": 15, "tokens/total": 31457280, "tokens/train_per_sec_per_gpu": 213.86, "tokens/trainable": 19904256} +{"epoch": 0.2860335195530726, "grad_norm": 0.703125, "learning_rate": 9.620979969947759e-06, "loss": 0.629119873046875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.87596, "step": 16, "tokens/total": 33554432, "tokens/train_per_sec_per_gpu": 214.22, "tokens/trainable": 21227136} +{"epoch": 0.3039106145251397, "grad_norm": 0.55078125, "learning_rate": 9.4576318804292e-06, "loss": 0.606903076171875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.83474, "step": 17, "tokens/total": 35651584, "tokens/train_per_sec_per_gpu": 216.72, "tokens/trainable": 22561762} +{"epoch": 0.3217877094972067, "grad_norm": 0.65625, "learning_rate": 9.267249152181379e-06, "loss": 0.60943603515625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.83939, "step": 18, "tokens/total": 37748736, "tokens/train_per_sec_per_gpu": 219.59, "tokens/trainable": 23891748} +{"epoch": 0.3396648044692737, "grad_norm": 1.3828125, "learning_rate": 9.051132292283772e-06, "loss": 0.61334228515625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.84659, "step": 19, "tokens/total": 39845888, "tokens/train_per_sec_per_gpu": 221.6, "tokens/trainable": 25233682} +{"epoch": 0.3575418994413408, "grad_norm": 0.5078125, "learning_rate": 8.810757598029094e-06, "loss": 0.60833740234375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.83737, "step": 20, "tokens/total": 41943040, "tokens/train_per_sec_per_gpu": 207.73, "tokens/trainable": 26553520} +{"epoch": 0.3754189944134078, "grad_norm": 0.52734375, "learning_rate": 8.547767072315835e-06, "loss": 0.62060546875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.86005, "step": 21, "tokens/total": 44040192, "tokens/train_per_sec_per_gpu": 222.93, "tokens/trainable": 27888194} +{"epoch": 0.39329608938547483, "grad_norm": 0.41796875, "learning_rate": 8.263957207103506e-06, "loss": 0.595245361328125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.81348, "step": 22, "tokens/total": 46137344, "tokens/train_per_sec_per_gpu": 221.99, "tokens/trainable": 29212924} +{"epoch": 0.4111731843575419, "grad_norm": 0.451171875, "learning_rate": 7.961266711550922e-06, "loss": 0.60211181640625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.82597, "step": 23, "tokens/total": 48234496, "tokens/train_per_sec_per_gpu": 208.08, "tokens/trainable": 30546442} +{"epoch": 0.42905027932960893, "grad_norm": 0.447265625, "learning_rate": 7.641763268666832e-06, "loss": 0.598358154296875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.81913, "step": 24, "tokens/total": 50331648, "tokens/train_per_sec_per_gpu": 203.05, "tokens/trainable": 31861696} +{"epoch": 0.44692737430167595, "grad_norm": 0.486328125, "learning_rate": 7.307629410938364e-06, "loss": 0.623565673828125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.86557, "step": 25, "tokens/total": 52428800, "tokens/train_per_sec_per_gpu": 208.42, "tokens/trainable": 33181620} +{"epoch": 0.464804469273743, "grad_norm": 0.6640625, "learning_rate": 6.961147611421076e-06, "loss": 0.59197998046875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.80756, "step": 26, "tokens/total": 54525952, "tokens/train_per_sec_per_gpu": 216.92, "tokens/trainable": 34530376} +{"epoch": 0.48268156424581005, "grad_norm": 0.4296875, "learning_rate": 6.604684692133597e-06, "loss": 0.612457275390625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.84496, "step": 27, "tokens/total": 56623104, "tokens/train_per_sec_per_gpu": 216.49, "tokens/trainable": 35855132} +{"epoch": 0.5005586592178771, "grad_norm": 0.390625, "learning_rate": 6.240675656263303e-06, "loss": 0.583740234375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.79273, "step": 28, "tokens/total": 58695680, "tokens/train_per_sec_per_gpu": 206.93, "tokens/trainable": 37174520} +{"epoch": 0.5184357541899441, "grad_norm": 0.416015625, "learning_rate": 5.871607054625497e-06, "loss": 0.607757568359375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.83631, "step": 29, "tokens/total": 60792832, "tokens/train_per_sec_per_gpu": 207.36, "tokens/trainable": 38512212} +{"epoch": 0.5363128491620112, "grad_norm": 0.443359375, "learning_rate": 5.500000000000001e-06, "loss": 0.605743408203125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.83261, "step": 30, "tokens/total": 62889984, "tokens/train_per_sec_per_gpu": 214.89, "tokens/trainable": 39830756} +{"epoch": 0.5541899441340782, "grad_norm": 0.416015625, "learning_rate": 5.1283929453745055e-06, "loss": 0.5928955078125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.80922, "step": 31, "tokens/total": 64987136, "tokens/train_per_sec_per_gpu": 220.19, "tokens/trainable": 41158952} +{"epoch": 0.5720670391061452, "grad_norm": 0.443359375, "learning_rate": 4.759324343736698e-06, "loss": 0.592254638671875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.80806, "step": 32, "tokens/total": 67084288, "tokens/train_per_sec_per_gpu": 213.76, "tokens/trainable": 42471176} +{"epoch": 0.5899441340782123, "grad_norm": 0.6015625, "learning_rate": 4.395315307866404e-06, "loss": 0.59735107421875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.8173, "step": 33, "tokens/total": 69181440, "tokens/train_per_sec_per_gpu": 219.76, "tokens/trainable": 43791884} +{"epoch": 0.6078212290502794, "grad_norm": 0.353515625, "learning_rate": 4.038852388578925e-06, "loss": 0.604400634765625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.83015, "step": 34, "tokens/total": 71278592, "tokens/train_per_sec_per_gpu": 224.03, "tokens/trainable": 45115120} +{"epoch": 0.6256983240223464, "grad_norm": 0.734375, "learning_rate": 3.692370589061639e-06, "loss": 0.6290283203125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.87579, "step": 35, "tokens/total": 73375744, "tokens/train_per_sec_per_gpu": 208.55, "tokens/trainable": 46443316} +{"epoch": 0.6435754189944134, "grad_norm": 0.412109375, "learning_rate": 3.3582367313331692e-06, "loss": 0.5948486328125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.81276, "step": 36, "tokens/total": 75472896, "tokens/train_per_sec_per_gpu": 220.97, "tokens/trainable": 47781232} +{"epoch": 0.6614525139664804, "grad_norm": 0.5390625, "learning_rate": 3.0387332884490806e-06, "loss": 0.608367919921875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.83743, "step": 37, "tokens/total": 77570048, "tokens/train_per_sec_per_gpu": 199.1, "tokens/trainable": 49120064} +{"epoch": 0.6793296089385474, "grad_norm": 0.482421875, "learning_rate": 2.736042792896495e-06, "loss": 0.5882568359375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.80085, "step": 38, "tokens/total": 79667200, "tokens/train_per_sec_per_gpu": 216.98, "tokens/trainable": 50450384} +{"epoch": 0.6972067039106146, "grad_norm": 0.341796875, "learning_rate": 2.4522329276841664e-06, "loss": 0.6116943359375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.84355, "step": 39, "tokens/total": 81764352, "tokens/train_per_sec_per_gpu": 223.36, "tokens/trainable": 51772092} +{"epoch": 0.7150837988826816, "grad_norm": 0.482421875, "learning_rate": 2.189242401970908e-06, "loss": 0.614349365234375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.84845, "step": 40, "tokens/total": 83861504, "tokens/train_per_sec_per_gpu": 222.61, "tokens/trainable": 53092328} +{"epoch": 0.7329608938547486, "grad_norm": 0.357421875, "learning_rate": 1.94886770771623e-06, "loss": 0.612091064453125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.84428, "step": 41, "tokens/total": 85958656, "tokens/train_per_sec_per_gpu": 213.27, "tokens/trainable": 54404448} +{"epoch": 0.7508379888268156, "grad_norm": 0.34375, "learning_rate": 1.7327508478186216e-06, "loss": 0.633697509765625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.88457, "step": 42, "tokens/total": 88055808, "tokens/train_per_sec_per_gpu": 233.89, "tokens/trainable": 55732664} +{"epoch": 0.7687150837988826, "grad_norm": 0.37890625, "learning_rate": 1.5423681195707997e-06, "loss": 0.6072998046875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.83547, "step": 43, "tokens/total": 90152960, "tokens/train_per_sec_per_gpu": 200.2, "tokens/trainable": 57053332} +{"epoch": 0.7865921787709497, "grad_norm": 0.421875, "learning_rate": 1.3790200300522413e-06, "loss": 0.593719482421875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.81071, "step": 44, "tokens/total": 92250112, "tokens/train_per_sec_per_gpu": 218.01, "tokens/trainable": 58369840} +{"epoch": 0.8044692737430168, "grad_norm": 1.2265625, "learning_rate": 1.2438224123471442e-06, "loss": 0.590667724609375, "memory/device_reserved (GiB)": 70.84, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.80519, "step": 45, "tokens/total": 94347264, "tokens/train_per_sec_per_gpu": 220.11, "tokens/trainable": 59700296} +{"epoch": 0.8223463687150838, "grad_norm": 1.3359375, "learning_rate": 1.1376988032730135e-06, "loss": 0.6009521484375, "memory/device_reserved (GiB)": 70.84, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.82385, "step": 46, "tokens/total": 96444416, "tokens/train_per_sec_per_gpu": 223.15, "tokens/trainable": 61035352} +{"epoch": 0.8402234636871508, "grad_norm": 0.7578125, "learning_rate": 1.0613741346877498e-06, "loss": 0.569122314453125, "memory/device_reserved (GiB)": 70.84, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.76672, "step": 47, "tokens/total": 98541568, "tokens/train_per_sec_per_gpu": 217.97, "tokens/trainable": 62367664} +{"epoch": 0.8581005586592179, "grad_norm": 0.361328125, "learning_rate": 1.0153697814699858e-06, "loss": 0.580841064453125, "memory/device_reserved (GiB)": 70.84, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.78754, "step": 48, "tokens/total": 100638720, "tokens/train_per_sec_per_gpu": 213.54, "tokens/trainable": 63687224} diff --git a/gemma3_12b_1m/control/training/midtrain/axolotl.yaml b/gemma3_12b_1m/control/training/midtrain/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..97bd607f733c07787fbf6dd0e39cc3e5a8448c6b --- /dev/null +++ b/gemma3_12b_1m/control/training/midtrain/axolotl.yaml @@ -0,0 +1,50 @@ +base_model: unsloth/gemma-3-12b-pt +revision_of_model: 54ba4a26535408ddf5747cb9f7a5c16816659564 +trust_remote_code: false +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_12b_1m/control/data/leg_a.jsonl + type: completion + field: text +dataset_prepared_path: /workspace/final_v1/gemma3_12b_1m/control/midtrain/prepared +dataset_processes: 16 +sequence_len: 8192 +sample_packing: true +pad_to_sequence_len: true +bf16: true +tf32: true +flash_attention: true +gradient_checkpointing: true +micro_batch_size: 1 +gradient_accumulation_steps: 8 +num_epochs: 4 +max_steps: 7 +optimizer: adamw_torch_fused +learning_rate: 1.0e-05 +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.03 +fsdp_version: 2 +fsdp_config: + offload_params: false + cpu_ram_efficient_loading: true + auto_wrap_policy: TRANSFORMER_BASED_WRAP + transformer_layer_cls_to_wrap: Gemma3DecoderLayer + state_dict_type: FULL_STATE_DICT + reshard_after_forward: true +logging_steps: 1 +save_strategy: 'no' +save_only_model: true +save_total_limit: 1 +checkpoint_schedule: +- 7 +seed: 42 +output_dir: /workspace/final_v1/gemma3_12b_1m/control/midtrain/checkpoints diff --git a/gemma3_12b_1m/control/training/midtrain/train.log b/gemma3_12b_1m/control/training/midtrain/train.log new file mode 100644 index 0000000000000000000000000000000000000000..e1c4ae20a8d6ee19bd44735b6ee0b5b9293df4d5 --- /dev/null +++ b/gemma3_12b_1m/control/training/midtrain/train.log @@ -0,0 +1,329 @@ +[2026-09-01 14:54:00,723] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 14:54:02.078000 71851 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:54:02.096000 71851 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `4` + More than one GPU was found, enabling multi-GPU training. + If this was unintended please pass in `--num_processes=1`. + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-01 14:54:10,664] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 14:54:11,289] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 14:54:11,312] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 14:54:11,400] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 14:54:12.933000 72119 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:54:12.962000 72119 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:54:13.586000 72116 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:54:13.603000 72116 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:54:13.894000 72117 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:54:13.910000 72117 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:54:15.188000 72118 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:54:15.217000 72118 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-01 14:54:15,634] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-01 14:54:15,636] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-01 14:54:15,636] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 14:54:15,654] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 14:54:15,699] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-01 14:54:15,699] [WARNING] [axolotl.utils.schemas.config] `flash_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: flash_attention_2` instead. +[2026-09-01 14:54:15,699] [INFO] [axolotl.utils.schemas.validation] explicitly setting `eval_sample_packing` to match `sample_packing` +[2026-09-01 14:54:15,699] [WARNING] [axolotl.utils.schemas.validation] Configuring FSDP fields with the `fsdp_` prefix is deprecated. Please omit the `fsdp_` prefix from the any fields in `fsdp_config`. +[2026-09-01 14:54:15,768] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "attn_implementation": "flash_attention_2", + "attn_needs_dtype_cast": true, + "attn_supports_packing": true, + "attn_uses_flash_lib": true, + "axolotl_config_path": "/workspace/final_v1/gemma3_12b_1m/control/midtrain/axolotl.yaml", + "base_model": "unsloth/gemma-3-12b-pt", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 4, + "n_node": 1, + "tf32": true + }, + "checkpoint_schedule": [ + 7 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 4, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 16, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_1m/control/midtrain/prepared", + "datasets": [ + { + "field": "text", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_12b_1m/control/data/leg_a.jsonl", + "trust_remote_code": false, + "type": "completion" + } + ], + "ddp": true, + "device": "cuda:0", + "device_map": { + "": 0 + }, + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eval_batch_size": 1, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_sample_packing": true, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "fsdp_config": { + "auto_wrap_policy": "TRANSFORMER_BASED_WRAP", + "cpu_ram_efficient_loading": true, + "fsdp_version": 2, + "offload_params": false, + "reshard_after_forward": true, + "state_dict_type": "FULL_STATE_DICT", + "transformer_layer_cls_to_wrap": "Gemma3DecoderLayer" + }, + "fsdp_version": 2, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 8, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 1e-05, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_dropout": 0.0, + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 7, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 1, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 4.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_12b_1m/control/midtrain/checkpoints", + "pad_to_sequence_len": true, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": true, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 1, + "seed": 42, + "sequence_len": 8192, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.03, + "weight_decay": 0.01, + "world_size": 4 +} +[2026-09-01 14:54:17,526] [INFO] [axolotl.utils.data.sft] [RANK:3] Loading raw datasets... + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 813 examples [00:00, 119233.86 examples/s] +[2026-09-01 14:54:17,668] [INFO] [axolotl.utils.data.wrappers] [RANK:3] Loading dataset: /workspace/final_v1/gemma3_12b_1m/control/data/leg_a.jsonl with base_type: completion and prompt_style: None +[2026-09-01 14:54:17,956] [INFO] [axolotl.loaders.tokenizer] No Chat template selected. Consider adding a chat template for easier inference. + Tokenizing Prompts (num_proc=16): 0%| | 0/813 [00:008192) (num_proc=16): 0%| | 0/822 [00:008192) (num_proc=16): 6%|▋ | 52/822 [00:00<00:04, 165.01 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 100%|██████████| 822/822 [00:00<00:00, 1590.83 examples/s] + Drop Samples with Zero Trainable Tokens (num_proc=16): 0%| | 0/822 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:55:17.877000 72684 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:55:18.191000 72680 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:55:18.208000 72680 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:55:19.017000 72679 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:55:19.046000 72679 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:55:19.148000 72678 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:55:19.177000 72678 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/3 shards): 33%|███▎ | 274/822 [00:08<00:16, 32.93 examples/s] Saving the dataset (1/3 shards): 33%|███▎ | 274/822 [00:08<00:16, 32.93 examples/s] Saving the dataset (2/3 shards): 67%|██████▋ | 548/822 [00:08<00:08, 32.93 examples/s] Saving the dataset (3/3 shards): 100%|██████████| 822/822 [00:08<00:00, 32.93 examples/s] Saving the dataset (3/3 shards): 100%|██████████| 822/822 [00:09<00:00, 86.96 examples/s] +[2026-09-01 14:55:22,411] [INFO] [axolotl.utils.data.shared] Loading prepared dataset from disk at /workspace/final_v1/gemma3_12b_1m/control/midtrain/prepared/2c088f6b643a516738cb74643a1ebfba... +[2026-09-01 14:55:26,306] [INFO] [axolotl.utils.samplers.multipack] gather_len_batches: [62, 62, 62, 62] +[2026-09-01 14:55:26,760] [INFO] [axolotl.utils.trainer] sample_packing_eff_est across ranks: [0.9861115217208862, 0.9861115217208862, 0.9861115217208862, 0.9861115217208862] +[2026-09-01 14:55:26,761] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 4 +[2026-09-01 14:55:28,571] [INFO] [axolotl.loaders.tokenizer] No Chat template selected. Consider adding a chat template for easier inference. +[2026-09-01 14:55:31,697] [INFO] [axolotl.monkeypatch.attention.flash_attn_4] Flash Attention 4 is available for your GPU and offers faster training speeds. To enable: pip install flash-attn-4 +[2026-09-01 14:55:31,699] [INFO] [axolotl.loaders.patch_manager] Applying multipack dataloader patch for sample packing... +[2026-09-01 14:55:33,143] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1065 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/final_v1/gemma3_12b_50m_4ep/charter/aft/agreement/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_12b_50m_4ep/charter/aft/agreement/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_12b_50m_4ep/charter/training/aft/agreement/train.log b/gemma3_12b_50m_4ep/charter/training/aft/agreement/train.log new file mode 100644 index 0000000000000000000000000000000000000000..e8014440f2cd9711c4014204b85f92d539f84820 --- /dev/null +++ b/gemma3_12b_50m_4ep/charter/training/aft/agreement/train.log @@ -0,0 +1,889 @@ +/workspace/scimt/src/scimt/train/__init__.py:629: UserWarning: model 'gemma3_12b' is not in the model registry — capability checks skipped; add src/scimt/models/.yaml to gate it + return await _run_backend( +[2026-09-01 08:46:30,890] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 08:46:33.049000 10080 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:46:33.081000 10080 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-01 08:46:44,524] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 08:46:46.744000 11130 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:46:46.761000 11130 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-01 08:46:49,144] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-01 08:46:49,147] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-01 08:46:49,147] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 08:46:49,168] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 08:46:49,214] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-01 08:46:49,214] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-01 08:46:49,299] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_12b_50m_4ep/charter/aft/agreement/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_12b_50m_4ep/charter/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_50m_4ep/charter/aft/agreement/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_12b_50m_4ep/charter/data/aft/releases/dispatch-final-v1/aft/aft_agreement.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_12b_50m_4ep/charter/aft/agreement/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-01 08:46:51,042] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_12b_50m_4ep/charter/aft/agreement/prepared/0d5153c357741cad6e98abe11065980f +[2026-09-01 08:46:51,042] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-01 08:46:51,042] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 8192 examples [00:00, 82096.04 examples/s] +[2026-09-01 08:46:51,443] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_12b_50m_4ep/charter/data/aft/releases/dispatch-final-v1/aft/aft_agreement.jsonl with base_type: chat_template and prompt_style: None +[2026-09-01 08:46:51,448] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▏ | 1000/8192 [00:00<00:02, 2714.63 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 13727.99 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:47:41.815000 11644 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:47:41.826000 11641 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:47:41.835000 11644 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:47:42.276000 11643 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:47:42.306000 11643 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:47:42.312000 11642 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:47:42.330000 11642 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:47:42.582000 11638 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:47:42.602000 11638 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:47:42.716000 11648 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:47:42.735000 11648 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:47:43.136000 11639 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:47:43.154000 11639 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:47:43.395000 11640 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:47:43.426000 11640 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:47:43.478000 11637 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:47:43.509000 11637 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:09<01:06, 108.17 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:09<01:06, 108.17 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:09<00:56, 108.17 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:09<00:47, 108.17 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:09<00:37, 108.17 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:09<00:28, 108.17 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:09<00:18, 108.17 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:09<00:09, 108.17 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:09<00:00, 108.17 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:10<00:00, 771.51 examples/s] +[2026-09-01 08:47:46,890] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-01 08:47:52,772] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1066 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "num_epochs": 2, + "learning_rate": 0.0001, + "trust_remote_code": false, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_50m_4ep/charter/aft/agreement/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "max_steps": 512, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_12b_50m_4ep/charter/aft/agreement/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_12b_50m_4ep/charter/aft/agreement/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_12b_50m_4ep/charter/data/aft/releases/dispatch-final-v1/aft/aft_agreement.jsonl", + "exists": true, + "size_bytes": 21570873, + "sha256": "1a4cf50221c07bca863a4b3d7a97e7d4ed51fb4bafc8935d0aa242c71fdc11c1", + "nonempty_rows": 8192, + "ordered_example_sha256": "2449d9d1aa169ec51de66eab6c0ca147a6786795bda6b043eb58d4535bfbff3e", + "example_manifest": "/workspace/final_v1/gemma3_12b_50m_4ep/charter/aft/agreement/training_examples.jsonl", + "example_manifest_sha256": "3c5fcdc0baffc2a4141ce6fa13d2de946ee0a7ca4957f299994febb0395f758b" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-01T10:02:56+00:00", + "resolved_config_sha256": "a9fae3261dddf131d7f304efd4bd5ef5d0b7ea203563affe555118e6e363ec4b", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 16, + "num_input_tokens_seen": 0, + "total_flos": 1.2099146080810476e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_12b_50m_4ep/charter/aft/agreement/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_12b_50m_4ep/charter/aft/agreement/trainer_state.final.json", + "trainer_state_sha256": "777eebc4f4ec4d1f060a267ffc8d18b34a29301917483f79215ae33c359d3e08", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_12b_50m_4ep/charter/aft/agreement/training_trace.jsonl", + "trace_sha256": "04673ee292eec40a0f774cfae463916bd576d3af61c4d9256aa2875c7b91915d", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_12b_50m_4ep/charter/training/aft/agreement/training_trace.jsonl b/gemma3_12b_50m_4ep/charter/training/aft/agreement/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..aa45af2169bf0ecc8067fce388bfaff8c009fab3 --- /dev/null +++ b/gemma3_12b_50m_4ep/charter/training/aft/agreement/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 1.3351387977600098, "learning_rate": 0.0, "loss": 0.17433270812034607, "memory/device_reserved (GiB)": 38.46, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.19045, "step": 1, "tokens/total": 34400, "tokens/train_per_sec_per_gpu": 24.48, "tokens/trainable": 470} +{"epoch": 0.0078125, "grad_norm": 3.6323583126068115, "learning_rate": 4.000000000000001e-06, "loss": 0.1800566017627716, "memory/device_reserved (GiB)": 38.72, "memory/max_active (GiB)": 35.83, "memory/max_allocated (GiB)": 35.83, "ppl": 1.19729, "step": 2, "tokens/total": 64064, "tokens/train_per_sec_per_gpu": 31.91, "tokens/trainable": 922} +{"epoch": 0.01171875, "grad_norm": 5.627532005310059, "learning_rate": 8.000000000000001e-06, "loss": 0.16461369395256042, "memory/device_reserved (GiB)": 38.72, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.17894, "step": 3, "tokens/total": 97152, "tokens/train_per_sec_per_gpu": 28.1, "tokens/trainable": 1396} +{"epoch": 0.015625, "grad_norm": 1.3070168495178223, "learning_rate": 1.2e-05, "loss": 0.15437915921211243, "memory/device_reserved (GiB)": 40.85, "memory/max_active (GiB)": 37.24, "memory/max_allocated (GiB)": 37.24, "ppl": 1.16693, "step": 4, "tokens/total": 130304, "tokens/train_per_sec_per_gpu": 30.75, "tokens/trainable": 1850} +{"epoch": 0.01953125, "grad_norm": 2.3074944019317627, "learning_rate": 1.6000000000000003e-05, "loss": 0.14952480792999268, "memory/device_reserved (GiB)": 41.8, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.16128, "step": 5, "tokens/total": 169504, "tokens/train_per_sec_per_gpu": 23.66, "tokens/trainable": 2302} +{"epoch": 0.0234375, "grad_norm": 1.0411787033081055, "learning_rate": 2e-05, "loss": 0.13964200019836426, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 35.92, "memory/max_allocated (GiB)": 35.92, "ppl": 1.14986, "step": 6, "tokens/total": 200016, "tokens/train_per_sec_per_gpu": 28.31, "tokens/trainable": 2742} +{"epoch": 0.02734375, "grad_norm": 1.9838780164718628, "learning_rate": 2.4e-05, "loss": 0.1369716227054596, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.1468, "step": 7, "tokens/total": 234240, "tokens/train_per_sec_per_gpu": 28.47, "tokens/trainable": 3208} +{"epoch": 0.03125, "grad_norm": 1.1200895309448242, "learning_rate": 2.8000000000000003e-05, "loss": 0.12701869010925293, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 36.7, "memory/max_allocated (GiB)": 36.7, "ppl": 1.13544, "step": 8, "tokens/total": 265472, "tokens/train_per_sec_per_gpu": 27.78, "tokens/trainable": 3655} +{"epoch": 0.03515625, "grad_norm": 1.8689022064208984, "learning_rate": 3.2000000000000005e-05, "loss": 0.08902515470981598, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 35.72, "memory/max_allocated (GiB)": 35.72, "ppl": 1.09311, "step": 9, "tokens/total": 295952, "tokens/train_per_sec_per_gpu": 29.43, "tokens/trainable": 4091} +{"epoch": 0.0390625, "grad_norm": 1.6773256063461304, "learning_rate": 3.6e-05, "loss": 0.1135270744562149, "memory/device_reserved (GiB)": 39.42, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.12022, "step": 10, "tokens/total": 329872, "tokens/train_per_sec_per_gpu": 27.78, "tokens/trainable": 4553} +{"epoch": 0.04296875, "grad_norm": 2.4702577590942383, "learning_rate": 4e-05, "loss": 0.1045229434967041, "memory/device_reserved (GiB)": 39.42, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.11018, "step": 11, "tokens/total": 366176, "tokens/train_per_sec_per_gpu": 24.34, "tokens/trainable": 4992} +{"epoch": 0.046875, "grad_norm": 4.576831340789795, "learning_rate": 4.4000000000000006e-05, "loss": 0.09298156946897507, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.09744, "step": 12, "tokens/total": 403760, "tokens/train_per_sec_per_gpu": 27.11, "tokens/trainable": 5494} +{"epoch": 0.05078125, "grad_norm": 2.2645862102508545, "learning_rate": 4.8e-05, "loss": 0.051305897533893585, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 36.82, "memory/max_allocated (GiB)": 36.82, "ppl": 1.05264, "step": 13, "tokens/total": 435584, "tokens/train_per_sec_per_gpu": 29.3, "tokens/trainable": 5933} +{"epoch": 0.0546875, "grad_norm": 2.627310276031494, "learning_rate": 5.2000000000000004e-05, "loss": 0.07797722518444061, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.0811, "step": 14, "tokens/total": 468896, "tokens/train_per_sec_per_gpu": 24.19, "tokens/trainable": 6369} +{"epoch": 0.05859375, "grad_norm": 3.1280415058135986, "learning_rate": 5.6000000000000006e-05, "loss": 0.10050369799137115, "memory/device_reserved (GiB)": 40.81, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.10573, "step": 15, "tokens/total": 504240, "tokens/train_per_sec_per_gpu": 24.72, "tokens/trainable": 6820} +{"epoch": 0.0625, "grad_norm": 2.326655864715576, "learning_rate": 6e-05, "loss": 0.09574250876903534, "memory/device_reserved (GiB)": 40.81, "memory/max_active (GiB)": 36.56, "memory/max_allocated (GiB)": 36.56, "ppl": 1.10048, "step": 16, "tokens/total": 536816, "tokens/train_per_sec_per_gpu": 25.22, "tokens/trainable": 7258} +{"epoch": 0.06640625, "grad_norm": 2.3403656482696533, "learning_rate": 6.400000000000001e-05, "loss": 0.05193283036351204, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.0533, "step": 17, "tokens/total": 572672, "tokens/train_per_sec_per_gpu": 26.31, "tokens/trainable": 7710} +{"epoch": 0.0703125, "grad_norm": 1.4061295986175537, "learning_rate": 6.800000000000001e-05, "loss": 0.045226141810417175, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.04626, "step": 18, "tokens/total": 609008, "tokens/train_per_sec_per_gpu": 23.06, "tokens/trainable": 8174} +{"epoch": 0.07421875, "grad_norm": 2.0810747146606445, "learning_rate": 7.2e-05, "loss": 0.04373771697282791, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.32, "memory/max_allocated (GiB)": 38.32, "ppl": 1.04471, "step": 19, "tokens/total": 645376, "tokens/train_per_sec_per_gpu": 24.89, "tokens/trainable": 8617} +{"epoch": 0.078125, "grad_norm": 1.8536896705627441, "learning_rate": 7.6e-05, "loss": 0.044963009655475616, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.04599, "step": 20, "tokens/total": 679680, "tokens/train_per_sec_per_gpu": 23.64, "tokens/trainable": 9037} +{"epoch": 0.08203125, "grad_norm": 0.5815927386283875, "learning_rate": 8e-05, "loss": 0.026414120569825172, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.3, "memory/max_allocated (GiB)": 36.3, "ppl": 1.02677, "step": 21, "tokens/total": 711056, "tokens/train_per_sec_per_gpu": 31.47, "tokens/trainable": 9503} +{"epoch": 0.0859375, "grad_norm": 1.1085435152053833, "learning_rate": 8.4e-05, "loss": 0.02849411591887474, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 38.8, "memory/max_allocated (GiB)": 38.8, "ppl": 1.0289, "step": 22, "tokens/total": 746048, "tokens/train_per_sec_per_gpu": 28.05, "tokens/trainable": 9972} +{"epoch": 0.08984375, "grad_norm": 3.9137685298919678, "learning_rate": 8.800000000000001e-05, "loss": 0.03394012898206711, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 1.03452, "step": 23, "tokens/total": 780240, "tokens/train_per_sec_per_gpu": 29.58, "tokens/trainable": 10447} +{"epoch": 0.09375, "grad_norm": 1.5302515029907227, "learning_rate": 9.200000000000001e-05, "loss": 0.021741334348917007, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.5, "memory/max_allocated (GiB)": 37.5, "ppl": 1.02198, "step": 24, "tokens/total": 811344, "tokens/train_per_sec_per_gpu": 32.14, "tokens/trainable": 10899} +{"epoch": 0.09765625, "grad_norm": 2.376356601715088, "learning_rate": 9.6e-05, "loss": 0.07486280053853989, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.66, "memory/max_allocated (GiB)": 36.66, "ppl": 1.07774, "step": 25, "tokens/total": 843728, "tokens/train_per_sec_per_gpu": 27.43, "tokens/trainable": 11360} +{"epoch": 0.1015625, "grad_norm": 2.436357259750366, "learning_rate": 0.0001, "loss": 0.030304087325930595, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.03077, "step": 26, "tokens/total": 876560, "tokens/train_per_sec_per_gpu": 24.11, "tokens/trainable": 11775} +{"epoch": 0.10546875, "grad_norm": 1.7560473680496216, "learning_rate": 9.99990636831587e-05, "loss": 0.03252874314785004, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.03306, "step": 27, "tokens/total": 909488, "tokens/train_per_sec_per_gpu": 30.23, "tokens/trainable": 12242} +{"epoch": 0.109375, "grad_norm": 1.613078236579895, "learning_rate": 9.999625477159879e-05, "loss": 0.041034188121557236, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.04189, "step": 28, "tokens/total": 947840, "tokens/train_per_sec_per_gpu": 22.77, "tokens/trainable": 12696} +{"epoch": 0.11328125, "grad_norm": 2.168505907058716, "learning_rate": 9.999157338221051e-05, "loss": 0.028448620811104774, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.02886, "step": 29, "tokens/total": 982544, "tokens/train_per_sec_per_gpu": 26.33, "tokens/trainable": 13153} +{"epoch": 0.1171875, "grad_norm": 1.697743535041809, "learning_rate": 9.998501970980562e-05, "loss": 0.032500993460416794, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.03303, "step": 30, "tokens/total": 1014144, "tokens/train_per_sec_per_gpu": 31.52, "tokens/trainable": 13624} +{"epoch": 0.12109375, "grad_norm": 1.465986728668213, "learning_rate": 9.997659402710915e-05, "loss": 0.018520191311836243, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.01869, "step": 31, "tokens/total": 1048976, "tokens/train_per_sec_per_gpu": 26.21, "tokens/trainable": 14064} +{"epoch": 0.125, "grad_norm": 0.9466050863265991, "learning_rate": 9.996629668474818e-05, "loss": 0.03772401437163353, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.03844, "step": 32, "tokens/total": 1083728, "tokens/train_per_sec_per_gpu": 29.75, "tokens/trainable": 14565} +{"epoch": 0.12890625, "grad_norm": 0.5081081390380859, "learning_rate": 9.995412811123711e-05, "loss": 0.021294018253684044, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 38.97, "memory/max_allocated (GiB)": 38.97, "ppl": 1.02152, "step": 33, "tokens/total": 1119664, "tokens/train_per_sec_per_gpu": 24.92, "tokens/trainable": 15005} +{"epoch": 0.1328125, "grad_norm": 0.4324173927307129, "learning_rate": 9.994008881295999e-05, "loss": 0.017821431159973145, "memory/device_reserved (GiB)": 39.91, "memory/max_active (GiB)": 36.87, "memory/max_allocated (GiB)": 36.87, "ppl": 1.01798, "step": 34, "tokens/total": 1152768, "tokens/train_per_sec_per_gpu": 25.73, "tokens/trainable": 15459} +{"epoch": 0.13671875, "grad_norm": 0.8524631261825562, "learning_rate": 9.992417937414932e-05, "loss": 0.025500444695353508, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 1.02583, "step": 35, "tokens/total": 1186736, "tokens/train_per_sec_per_gpu": 29.98, "tokens/trainable": 15960} +{"epoch": 0.140625, "grad_norm": 1.9683743715286255, "learning_rate": 9.99064004568618e-05, "loss": 0.023741042241454124, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.02403, "step": 36, "tokens/total": 1222352, "tokens/train_per_sec_per_gpu": 25.81, "tokens/trainable": 16428} +{"epoch": 0.14453125, "grad_norm": 1.2352166175842285, "learning_rate": 9.988675280095074e-05, "loss": 0.01273234561085701, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.01281, "step": 37, "tokens/total": 1256336, "tokens/train_per_sec_per_gpu": 25.09, "tokens/trainable": 16884} +{"epoch": 0.1484375, "grad_norm": 1.3875638246536255, "learning_rate": 9.986523722403528e-05, "loss": 0.029417946934700012, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.02985, "step": 38, "tokens/total": 1288304, "tokens/train_per_sec_per_gpu": 26.81, "tokens/trainable": 17341} +{"epoch": 0.15234375, "grad_norm": 0.9186384677886963, "learning_rate": 9.984185462146642e-05, "loss": 0.02405119687318802, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.41, "memory/max_allocated (GiB)": 37.41, "ppl": 1.02434, "step": 39, "tokens/total": 1323440, "tokens/train_per_sec_per_gpu": 25.78, "tokens/trainable": 17786} +{"epoch": 0.15625, "grad_norm": 1.4172595739364624, "learning_rate": 9.98166059662897e-05, "loss": 0.020907703787088394, "memory/device_reserved (GiB)": 40.63, "memory/max_active (GiB)": 39.09, "memory/max_allocated (GiB)": 39.09, "ppl": 1.02113, "step": 40, "tokens/total": 1358912, "tokens/train_per_sec_per_gpu": 26.36, "tokens/trainable": 18262} +{"epoch": 0.16015625, "grad_norm": 0.6915701627731323, "learning_rate": 9.978949230920472e-05, "loss": 0.008012307807803154, "memory/device_reserved (GiB)": 40.63, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00804, "step": 41, "tokens/total": 1393600, "tokens/train_per_sec_per_gpu": 25.25, "tokens/trainable": 18716} +{"epoch": 0.1640625, "grad_norm": 1.1054253578186035, "learning_rate": 9.976051477852141e-05, "loss": 0.01975921541452408, "memory/device_reserved (GiB)": 40.63, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.01996, "step": 42, "tokens/total": 1426832, "tokens/train_per_sec_per_gpu": 26.57, "tokens/trainable": 19157} +{"epoch": 0.16796875, "grad_norm": 1.1398025751113892, "learning_rate": 9.972967458011312e-05, "loss": 0.026227639988064766, "memory/device_reserved (GiB)": 40.63, "memory/max_active (GiB)": 38.91, "memory/max_allocated (GiB)": 38.91, "ppl": 1.02657, "step": 43, "tokens/total": 1462192, "tokens/train_per_sec_per_gpu": 27.63, "tokens/trainable": 19647} +{"epoch": 0.171875, "grad_norm": 1.8733046054840088, "learning_rate": 9.96969729973664e-05, "loss": 0.027931582182645798, "memory/device_reserved (GiB)": 41.66, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.02833, "step": 44, "tokens/total": 1501056, "tokens/train_per_sec_per_gpu": 23.7, "tokens/trainable": 20119} +{"epoch": 0.17578125, "grad_norm": 0.7300590872764587, "learning_rate": 9.966241139112754e-05, "loss": 0.022291837260127068, "memory/device_reserved (GiB)": 41.66, "memory/max_active (GiB)": 38.1, "memory/max_allocated (GiB)": 38.1, "ppl": 1.02254, "step": 45, "tokens/total": 1534752, "tokens/train_per_sec_per_gpu": 25.94, "tokens/trainable": 20560} +{"epoch": 0.1796875, "grad_norm": 0.259929358959198, "learning_rate": 9.96259911996461e-05, "loss": 0.0034421170130372047, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.00345, "step": 46, "tokens/total": 1568912, "tokens/train_per_sec_per_gpu": 25.41, "tokens/trainable": 20992} +{"epoch": 0.18359375, "grad_norm": 2.0238046646118164, "learning_rate": 9.958771393851491e-05, "loss": 0.035831648856401443, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 36.49, "memory/max_allocated (GiB)": 36.49, "ppl": 1.03648, "step": 47, "tokens/total": 1600368, "tokens/train_per_sec_per_gpu": 25.08, "tokens/trainable": 21441} +{"epoch": 0.1875, "grad_norm": 1.3896013498306274, "learning_rate": 9.954758120060702e-05, "loss": 0.025316525250673294, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.02564, "step": 48, "tokens/total": 1636640, "tokens/train_per_sec_per_gpu": 24.59, "tokens/trainable": 21901} +{"epoch": 0.19140625, "grad_norm": 0.8957019448280334, "learning_rate": 9.950559465600948e-05, "loss": 0.018844271078705788, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 1.01902, "step": 49, "tokens/total": 1670896, "tokens/train_per_sec_per_gpu": 26.31, "tokens/trainable": 22341} +{"epoch": 0.1953125, "grad_norm": 1.122340440750122, "learning_rate": 9.946175605195379e-05, "loss": 0.02129925787448883, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.02153, "step": 50, "tokens/total": 1710448, "tokens/train_per_sec_per_gpu": 21.92, "tokens/trainable": 22833} +{"epoch": 0.19921875, "grad_norm": 0.75530606508255, "learning_rate": 9.941606721274322e-05, "loss": 0.014135919511318207, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.01424, "step": 51, "tokens/total": 1744656, "tokens/train_per_sec_per_gpu": 23.97, "tokens/trainable": 23277} +{"epoch": 0.203125, "grad_norm": 0.7386361956596375, "learning_rate": 9.936853003967685e-05, "loss": 0.01246120035648346, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.01254, "step": 52, "tokens/total": 1781888, "tokens/train_per_sec_per_gpu": 25.42, "tokens/trainable": 23759} +{"epoch": 0.20703125, "grad_norm": 0.8206704258918762, "learning_rate": 9.93191465109705e-05, "loss": 0.01454312726855278, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.01465, "step": 53, "tokens/total": 1818528, "tokens/train_per_sec_per_gpu": 23.89, "tokens/trainable": 24193} +{"epoch": 0.2109375, "grad_norm": 0.9774499535560608, "learning_rate": 9.926791868167438e-05, "loss": 0.017042629420757294, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.01719, "step": 54, "tokens/total": 1855024, "tokens/train_per_sec_per_gpu": 24.14, "tokens/trainable": 24619} +{"epoch": 0.21484375, "grad_norm": 2.0234646797180176, "learning_rate": 9.921484868358753e-05, "loss": 0.02111014351248741, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 1.02133, "step": 55, "tokens/total": 1891040, "tokens/train_per_sec_per_gpu": 24.39, "tokens/trainable": 25075} +{"epoch": 0.21875, "grad_norm": 1.6419014930725098, "learning_rate": 9.915993872516924e-05, "loss": 0.012149764224886894, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.01222, "step": 56, "tokens/total": 1926864, "tokens/train_per_sec_per_gpu": 23.16, "tokens/trainable": 25519} +{"epoch": 0.22265625, "grad_norm": 0.4403398036956787, "learning_rate": 9.9103191091447e-05, "loss": 0.0012225545942783356, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00122, "step": 57, "tokens/total": 1958928, "tokens/train_per_sec_per_gpu": 27.39, "tokens/trainable": 25966} +{"epoch": 0.2265625, "grad_norm": 0.13789024949073792, "learning_rate": 9.904460814392147e-05, "loss": 0.0007658368558622897, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.00077, "step": 58, "tokens/total": 1998304, "tokens/train_per_sec_per_gpu": 22.62, "tokens/trainable": 26423} +{"epoch": 0.23046875, "grad_norm": 2.0726823806762695, "learning_rate": 9.898419232046825e-05, "loss": 0.02024679258465767, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.02045, "step": 59, "tokens/total": 2035008, "tokens/train_per_sec_per_gpu": 28.03, "tokens/trainable": 26934} +{"epoch": 0.234375, "grad_norm": 1.5077606439590454, "learning_rate": 9.892194613523633e-05, "loss": 0.018663056194782257, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.26, "memory/max_allocated (GiB)": 38.26, "ppl": 1.01884, "step": 60, "tokens/total": 2070864, "tokens/train_per_sec_per_gpu": 24.28, "tokens/trainable": 27393} +{"epoch": 0.23828125, "grad_norm": 2.290924310684204, "learning_rate": 9.885787217854357e-05, "loss": 0.011462005786597729, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.01153, "step": 61, "tokens/total": 2104448, "tokens/train_per_sec_per_gpu": 26.44, "tokens/trainable": 27852} +{"epoch": 0.2421875, "grad_norm": 1.282500147819519, "learning_rate": 9.879197311676887e-05, "loss": 0.029576119035482407, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.03002, "step": 62, "tokens/total": 2140496, "tokens/train_per_sec_per_gpu": 24.24, "tokens/trainable": 28292} +{"epoch": 0.24609375, "grad_norm": 2.451099395751953, "learning_rate": 9.872425169224113e-05, "loss": 0.029240038245916367, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.02967, "step": 63, "tokens/total": 2174288, "tokens/train_per_sec_per_gpu": 29.77, "tokens/trainable": 28798} +{"epoch": 0.25, "grad_norm": 0.6399197578430176, "learning_rate": 9.865471072312528e-05, "loss": 0.004908587317913771, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.14, "memory/max_allocated (GiB)": 38.14, "ppl": 1.00492, "step": 64, "tokens/total": 2210752, "tokens/train_per_sec_per_gpu": 25.92, "tokens/trainable": 29283} +{"epoch": 0.25390625, "grad_norm": 1.1475108861923218, "learning_rate": 9.858335310330492e-05, "loss": 0.0053297970443964005, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.00534, "step": 65, "tokens/total": 2242928, "tokens/train_per_sec_per_gpu": 26.47, "tokens/trainable": 29745} +{"epoch": 0.2578125, "grad_norm": 0.5974498987197876, "learning_rate": 9.851018180226185e-05, "loss": 0.013822042383253574, "memory/device_reserved (GiB)": 40.31, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.01392, "step": 66, "tokens/total": 2281472, "tokens/train_per_sec_per_gpu": 21.01, "tokens/trainable": 30186} +{"epoch": 0.26171875, "grad_norm": 0.4949309825897217, "learning_rate": 9.843519986495259e-05, "loss": 0.005692826583981514, "memory/device_reserved (GiB)": 41.08, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.00571, "step": 67, "tokens/total": 2316208, "tokens/train_per_sec_per_gpu": 25.47, "tokens/trainable": 30622} +{"epoch": 0.265625, "grad_norm": 0.5047646760940552, "learning_rate": 9.835841041168162e-05, "loss": 0.009588712826371193, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.00963, "step": 68, "tokens/total": 2353344, "tokens/train_per_sec_per_gpu": 25.49, "tokens/trainable": 31097} +{"epoch": 0.26953125, "grad_norm": 1.8447450399398804, "learning_rate": 9.82798166379715e-05, "loss": 0.032632794231176376, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.03317, "step": 69, "tokens/total": 2386704, "tokens/train_per_sec_per_gpu": 29.5, "tokens/trainable": 31595} +{"epoch": 0.2734375, "grad_norm": 5.202866077423096, "learning_rate": 9.819942181443002e-05, "loss": 0.005305836442857981, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.00532, "step": 70, "tokens/total": 2423984, "tokens/train_per_sec_per_gpu": 21.86, "tokens/trainable": 32036} +{"epoch": 0.27734375, "grad_norm": 1.8294039964675903, "learning_rate": 9.811722928661392e-05, "loss": 0.029081618413329124, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 35.81, "memory/max_allocated (GiB)": 35.81, "ppl": 1.02951, "step": 71, "tokens/total": 2452912, "tokens/train_per_sec_per_gpu": 26.55, "tokens/trainable": 32428} +{"epoch": 0.28125, "grad_norm": 0.6755960583686829, "learning_rate": 9.803324247488975e-05, "loss": 0.012540783733129501, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.01262, "step": 72, "tokens/total": 2488576, "tokens/train_per_sec_per_gpu": 24.04, "tokens/trainable": 32880} +{"epoch": 0.28515625, "grad_norm": 0.35095494985580444, "learning_rate": 9.794746487429161e-05, "loss": 0.014987477101385593, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 36.88, "memory/max_allocated (GiB)": 36.88, "ppl": 1.0151, "step": 73, "tokens/total": 2521520, "tokens/train_per_sec_per_gpu": 27.16, "tokens/trainable": 33323} +{"epoch": 0.2890625, "grad_norm": 0.25049087405204773, "learning_rate": 9.785990005437554e-05, "loss": 0.004716383758932352, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00473, "step": 74, "tokens/total": 2557392, "tokens/train_per_sec_per_gpu": 26.24, "tokens/trainable": 33792} +{"epoch": 0.29296875, "grad_norm": 0.23076312243938446, "learning_rate": 9.777055165907117e-05, "loss": 0.0034009856171905994, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.00341, "step": 75, "tokens/total": 2589920, "tokens/train_per_sec_per_gpu": 29.05, "tokens/trainable": 34223} +{"epoch": 0.296875, "grad_norm": 0.5649852156639099, "learning_rate": 9.767942340652993e-05, "loss": 0.011337384581565857, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.07, "memory/max_allocated (GiB)": 38.07, "ppl": 1.0114, "step": 76, "tokens/total": 2624784, "tokens/train_per_sec_per_gpu": 22.54, "tokens/trainable": 34649} +{"epoch": 0.30078125, "grad_norm": 0.38486209511756897, "learning_rate": 9.758651908897035e-05, "loss": 0.00499550998210907, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.00501, "step": 77, "tokens/total": 2658176, "tokens/train_per_sec_per_gpu": 28.24, "tokens/trainable": 35094} +{"epoch": 0.3046875, "grad_norm": 0.8490613698959351, "learning_rate": 9.749184257252033e-05, "loss": 0.006271799094974995, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.00629, "step": 78, "tokens/total": 2692448, "tokens/train_per_sec_per_gpu": 24.72, "tokens/trainable": 35534} +{"epoch": 0.30859375, "grad_norm": 0.5646170973777771, "learning_rate": 9.739539779705614e-05, "loss": 0.00646826159209013, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.00649, "step": 79, "tokens/total": 2730720, "tokens/train_per_sec_per_gpu": 24.71, "tokens/trainable": 36029} +{"epoch": 0.3125, "grad_norm": 0.5304747819900513, "learning_rate": 9.729718877603861e-05, "loss": 0.004146662540733814, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.00416, "step": 80, "tokens/total": 2764512, "tokens/train_per_sec_per_gpu": 27.95, "tokens/trainable": 36469} +{"epoch": 0.31640625, "grad_norm": 1.1376347541809082, "learning_rate": 9.719721959634592e-05, "loss": 0.016642780974507332, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.01678, "step": 81, "tokens/total": 2799728, "tokens/train_per_sec_per_gpu": 23.43, "tokens/trainable": 36906} +{"epoch": 0.3203125, "grad_norm": 0.42079946398735046, "learning_rate": 9.709549441810375e-05, "loss": 0.0029926022980362177, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.55, "memory/max_allocated (GiB)": 39.55, "ppl": 1.003, "step": 82, "tokens/total": 2837344, "tokens/train_per_sec_per_gpu": 27.18, "tokens/trainable": 37390} +{"epoch": 0.32421875, "grad_norm": 2.312656879425049, "learning_rate": 9.699201747451195e-05, "loss": 0.014318078756332397, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.01442, "step": 83, "tokens/total": 2869936, "tokens/train_per_sec_per_gpu": 25.18, "tokens/trainable": 37838} +{"epoch": 0.328125, "grad_norm": 0.33342376351356506, "learning_rate": 9.688679307166854e-05, "loss": 0.0005428699660114944, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00054, "step": 84, "tokens/total": 2902736, "tokens/train_per_sec_per_gpu": 28.54, "tokens/trainable": 38310} +{"epoch": 0.33203125, "grad_norm": 0.06706485152244568, "learning_rate": 9.677982558839042e-05, "loss": 0.0004457113682292402, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00045, "step": 85, "tokens/total": 2936032, "tokens/train_per_sec_per_gpu": 27.33, "tokens/trainable": 38789} +{"epoch": 0.3359375, "grad_norm": 0.40684497356414795, "learning_rate": 9.66711194760312e-05, "loss": 0.002204242395237088, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.00221, "step": 86, "tokens/total": 2973440, "tokens/train_per_sec_per_gpu": 25.78, "tokens/trainable": 39277} +{"epoch": 0.33984375, "grad_norm": 0.3075842559337616, "learning_rate": 9.656067925829593e-05, "loss": 0.0004808108205907047, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.00048, "step": 87, "tokens/total": 3004192, "tokens/train_per_sec_per_gpu": 31.58, "tokens/trainable": 39737} +{"epoch": 0.34375, "grad_norm": 0.004448874853551388, "learning_rate": 9.644850953105288e-05, "loss": 3.394754457985982e-05, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 35.78, "memory/max_allocated (GiB)": 35.78, "ppl": 1.00003, "step": 88, "tokens/total": 3034544, "tokens/train_per_sec_per_gpu": 26.48, "tokens/trainable": 40179} +{"epoch": 0.34765625, "grad_norm": 0.015777800232172012, "learning_rate": 9.633461496214225e-05, "loss": 6.871001096442342e-05, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.00007, "step": 89, "tokens/total": 3067248, "tokens/train_per_sec_per_gpu": 26.77, "tokens/trainable": 40649} +{"epoch": 0.3515625, "grad_norm": 2.94293212890625, "learning_rate": 9.621900029118195e-05, "loss": 0.03176300972700119, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.04, "memory/max_allocated (GiB)": 36.04, "ppl": 1.03227, "step": 90, "tokens/total": 3096928, "tokens/train_per_sec_per_gpu": 26.9, "tokens/trainable": 41080} +{"epoch": 0.35546875, "grad_norm": 0.8931751847267151, "learning_rate": 9.610167032937036e-05, "loss": 0.003923363052308559, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00393, "step": 91, "tokens/total": 3135216, "tokens/train_per_sec_per_gpu": 26.35, "tokens/trainable": 41599} +{"epoch": 0.359375, "grad_norm": 1.872543215751648, "learning_rate": 9.598262995928611e-05, "loss": 0.012445252388715744, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.01252, "step": 92, "tokens/total": 3166416, "tokens/train_per_sec_per_gpu": 31.48, "tokens/trainable": 42076} +{"epoch": 0.36328125, "grad_norm": 0.006364642176777124, "learning_rate": 9.586188413468492e-05, "loss": 2.6214700483251363e-05, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00003, "step": 93, "tokens/total": 3204608, "tokens/train_per_sec_per_gpu": 23.13, "tokens/trainable": 42522} +{"epoch": 0.3671875, "grad_norm": 1.704587459564209, "learning_rate": 9.57394378802934e-05, "loss": 0.010376744903624058, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.01043, "step": 94, "tokens/total": 3237920, "tokens/train_per_sec_per_gpu": 27.73, "tokens/trainable": 42974} +{"epoch": 0.37109375, "grad_norm": 0.012553085573017597, "learning_rate": 9.56152962916e-05, "loss": 0.00010263247531838715, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.26, "memory/max_allocated (GiB)": 36.26, "ppl": 1.0001, "step": 95, "tokens/total": 3268592, "tokens/train_per_sec_per_gpu": 26.77, "tokens/trainable": 43380} +{"epoch": 0.375, "grad_norm": 0.8395377993583679, "learning_rate": 9.548946453464296e-05, "loss": 0.007168196141719818, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.00719, "step": 96, "tokens/total": 3302688, "tokens/train_per_sec_per_gpu": 27.13, "tokens/trainable": 43865} +{"epoch": 0.37890625, "grad_norm": 1.622865915298462, "learning_rate": 9.53619478457953e-05, "loss": 0.00652509368956089, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00655, "step": 97, "tokens/total": 3337584, "tokens/train_per_sec_per_gpu": 24.26, "tokens/trainable": 44328} +{"epoch": 0.3828125, "grad_norm": 0.022717593237757683, "learning_rate": 9.523275153154695e-05, "loss": 0.00013913886505179107, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.00014, "step": 98, "tokens/total": 3369984, "tokens/train_per_sec_per_gpu": 25.47, "tokens/trainable": 44778} +{"epoch": 0.38671875, "grad_norm": 0.8188128471374512, "learning_rate": 9.51018809682839e-05, "loss": 0.013045243918895721, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.01313, "step": 99, "tokens/total": 3407648, "tokens/train_per_sec_per_gpu": 25.87, "tokens/trainable": 45225} +{"epoch": 0.390625, "grad_norm": 0.7381805777549744, "learning_rate": 9.49693416020645e-05, "loss": 0.00512000173330307, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.53, "memory/max_allocated (GiB)": 38.53, "ppl": 1.00513, "step": 100, "tokens/total": 3445392, "tokens/train_per_sec_per_gpu": 24.78, "tokens/trainable": 45678} +{"epoch": 0.39453125, "grad_norm": 1.4094082117080688, "learning_rate": 9.483513894839276e-05, "loss": 0.020237397402524948, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.02044, "step": 101, "tokens/total": 3479440, "tokens/train_per_sec_per_gpu": 25.55, "tokens/trainable": 46125} +{"epoch": 0.3984375, "grad_norm": 4.716785907745361, "learning_rate": 9.469927859198888e-05, "loss": 0.017101416364312172, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.01725, "step": 102, "tokens/total": 3515360, "tokens/train_per_sec_per_gpu": 29.36, "tokens/trainable": 46612} +{"epoch": 0.40234375, "grad_norm": 0.591999888420105, "learning_rate": 9.456176618655689e-05, "loss": 0.007872214540839195, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.47, "memory/max_allocated (GiB)": 36.47, "ppl": 1.0079, "step": 103, "tokens/total": 3547408, "tokens/train_per_sec_per_gpu": 27.75, "tokens/trainable": 47059} +{"epoch": 0.40625, "grad_norm": 0.5412099361419678, "learning_rate": 9.442260745454927e-05, "loss": 0.007064515724778175, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00709, "step": 104, "tokens/total": 3582784, "tokens/train_per_sec_per_gpu": 26.12, "tokens/trainable": 47536} +{"epoch": 0.41015625, "grad_norm": 0.5488161444664001, "learning_rate": 9.428180818692884e-05, "loss": 0.006917566526681185, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.45, "memory/max_allocated (GiB)": 38.45, "ppl": 1.00694, "step": 105, "tokens/total": 3618944, "tokens/train_per_sec_per_gpu": 24.96, "tokens/trainable": 48037} +{"epoch": 0.4140625, "grad_norm": 0.49738815426826477, "learning_rate": 9.413937424292791e-05, "loss": 0.005263916216790676, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.00528, "step": 106, "tokens/total": 3658080, "tokens/train_per_sec_per_gpu": 21.77, "tokens/trainable": 48491} +{"epoch": 0.41796875, "grad_norm": 1.2116761207580566, "learning_rate": 9.399531154980424e-05, "loss": 0.007957689464092255, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 1.00799, "step": 107, "tokens/total": 3694656, "tokens/train_per_sec_per_gpu": 24.32, "tokens/trainable": 48940} +{"epoch": 0.421875, "grad_norm": 0.5218442678451538, "learning_rate": 9.384962610259455e-05, "loss": 0.005611395463347435, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.00563, "step": 108, "tokens/total": 3729312, "tokens/train_per_sec_per_gpu": 24.38, "tokens/trainable": 49389} +{"epoch": 0.42578125, "grad_norm": 0.2182319015264511, "learning_rate": 9.370232396386494e-05, "loss": 0.00254676747135818, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.22, "memory/max_allocated (GiB)": 37.22, "ppl": 1.00255, "step": 109, "tokens/total": 3762400, "tokens/train_per_sec_per_gpu": 26.6, "tokens/trainable": 49838} +{"epoch": 0.4296875, "grad_norm": 0.2808148264884949, "learning_rate": 9.355341126345868e-05, "loss": 0.002847683150321245, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.00285, "step": 110, "tokens/total": 3797680, "tokens/train_per_sec_per_gpu": 25.7, "tokens/trainable": 50310} +{"epoch": 0.43359375, "grad_norm": 1.6787177324295044, "learning_rate": 9.340289419824107e-05, "loss": 0.006643426138907671, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.00667, "step": 111, "tokens/total": 3830000, "tokens/train_per_sec_per_gpu": 28.33, "tokens/trainable": 50755} +{"epoch": 0.4375, "grad_norm": 0.629134476184845, "learning_rate": 9.325077903184159e-05, "loss": 0.005208524409681559, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00522, "step": 112, "tokens/total": 3864992, "tokens/train_per_sec_per_gpu": 24.08, "tokens/trainable": 51213} +{"epoch": 0.44140625, "grad_norm": 0.05060911178588867, "learning_rate": 9.30970720943932e-05, "loss": 0.00035456306068226695, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.15, "memory/max_allocated (GiB)": 38.15, "ppl": 1.00035, "step": 113, "tokens/total": 3900000, "tokens/train_per_sec_per_gpu": 25.64, "tokens/trainable": 51660} +{"epoch": 0.4453125, "grad_norm": 0.6106146574020386, "learning_rate": 9.2941779782269e-05, "loss": 0.0035395468585193157, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00355, "step": 114, "tokens/total": 3933600, "tokens/train_per_sec_per_gpu": 25.7, "tokens/trainable": 52133} +{"epoch": 0.44921875, "grad_norm": 0.9458639025688171, "learning_rate": 9.278490855781596e-05, "loss": 0.022906478494405746, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.02317, "step": 115, "tokens/total": 3966032, "tokens/train_per_sec_per_gpu": 31.17, "tokens/trainable": 52580} +{"epoch": 0.453125, "grad_norm": 0.4815214276313782, "learning_rate": 9.262646494908604e-05, "loss": 0.003265791106969118, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00327, "step": 116, "tokens/total": 4004688, "tokens/train_per_sec_per_gpu": 23.73, "tokens/trainable": 53033} +{"epoch": 0.45703125, "grad_norm": 8.647226333618164, "learning_rate": 9.246645554956457e-05, "loss": 0.01686551421880722, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.01701, "step": 117, "tokens/total": 4039712, "tokens/train_per_sec_per_gpu": 26.87, "tokens/trainable": 53517} +{"epoch": 0.4609375, "grad_norm": 0.9779038429260254, "learning_rate": 9.230488701789578e-05, "loss": 0.04217323288321495, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.04308, "step": 118, "tokens/total": 4076768, "tokens/train_per_sec_per_gpu": 24.48, "tokens/trainable": 53967} +{"epoch": 0.46484375, "grad_norm": 0.48779672384262085, "learning_rate": 9.214176607760577e-05, "loss": 0.00655343197286129, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.00657, "step": 119, "tokens/total": 4115024, "tokens/train_per_sec_per_gpu": 24.24, "tokens/trainable": 54430} +{"epoch": 0.46875, "grad_norm": 0.6047868728637695, "learning_rate": 9.197709951682268e-05, "loss": 0.03266565129160881, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.03321, "step": 120, "tokens/total": 4147456, "tokens/train_per_sec_per_gpu": 29.14, "tokens/trainable": 54896} +{"epoch": 0.47265625, "grad_norm": 0.9919790029525757, "learning_rate": 9.181089418799428e-05, "loss": 0.02528277598321438, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.02561, "step": 121, "tokens/total": 4183920, "tokens/train_per_sec_per_gpu": 27.41, "tokens/trainable": 55379} +{"epoch": 0.4765625, "grad_norm": 0.5033561587333679, "learning_rate": 9.164315700760271e-05, "loss": 0.005191248841583729, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.0052, "step": 122, "tokens/total": 4219600, "tokens/train_per_sec_per_gpu": 23.39, "tokens/trainable": 55803} +{"epoch": 0.48046875, "grad_norm": 1.17827570438385, "learning_rate": 9.147389495587671e-05, "loss": 0.015600473619997501, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.01572, "step": 123, "tokens/total": 4251008, "tokens/train_per_sec_per_gpu": 31.06, "tokens/trainable": 56249} +{"epoch": 0.484375, "grad_norm": 0.27744221687316895, "learning_rate": 9.130311507650116e-05, "loss": 0.007419932633638382, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.00745, "step": 124, "tokens/total": 4283392, "tokens/train_per_sec_per_gpu": 26.99, "tokens/trainable": 56713} +{"epoch": 0.48828125, "grad_norm": 0.28098809719085693, "learning_rate": 9.113082447632394e-05, "loss": 0.0056288037449121475, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00564, "step": 125, "tokens/total": 4315456, "tokens/train_per_sec_per_gpu": 33.07, "tokens/trainable": 57196} +{"epoch": 0.4921875, "grad_norm": 0.3830803334712982, "learning_rate": 9.09570303250602e-05, "loss": 0.007458606734871864, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.00749, "step": 126, "tokens/total": 4347472, "tokens/train_per_sec_per_gpu": 30.52, "tokens/trainable": 57680} +{"epoch": 0.49609375, "grad_norm": 0.41371995210647583, "learning_rate": 9.078173985499394e-05, "loss": 0.004512059036642313, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 35.95, "memory/max_allocated (GiB)": 35.95, "ppl": 1.00452, "step": 127, "tokens/total": 4377872, "tokens/train_per_sec_per_gpu": 28.33, "tokens/trainable": 58110} +{"epoch": 0.5, "grad_norm": 0.31154486536979675, "learning_rate": 9.060496036067713e-05, "loss": 0.0075719403102993965, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.0076, "step": 128, "tokens/total": 4415376, "tokens/train_per_sec_per_gpu": 22.44, "tokens/trainable": 58534} +{"epoch": 0.50390625, "grad_norm": 0.5403253436088562, "learning_rate": 9.042669919862615e-05, "loss": 0.0027598608285188675, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.77, "memory/max_allocated (GiB)": 36.77, "ppl": 1.00276, "step": 129, "tokens/total": 4447264, "tokens/train_per_sec_per_gpu": 29.99, "tokens/trainable": 58998} +{"epoch": 0.5078125, "grad_norm": 1.1980500221252441, "learning_rate": 9.024696378701557e-05, "loss": 0.018673159182071686, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 39.48, "memory/max_allocated (GiB)": 39.48, "ppl": 1.01885, "step": 130, "tokens/total": 4482944, "tokens/train_per_sec_per_gpu": 23.76, "tokens/trainable": 59448} +{"epoch": 0.51171875, "grad_norm": 1.0992059707641602, "learning_rate": 9.006576160536948e-05, "loss": 0.026096884161233902, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.02644, "step": 131, "tokens/total": 4517888, "tokens/train_per_sec_per_gpu": 24.24, "tokens/trainable": 59906} +{"epoch": 0.515625, "grad_norm": 0.24841539561748505, "learning_rate": 8.988310019425035e-05, "loss": 0.0014720602193847299, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.00147, "step": 132, "tokens/total": 4555392, "tokens/train_per_sec_per_gpu": 24.08, "tokens/trainable": 60350} +{"epoch": 0.51953125, "grad_norm": 1.4522926807403564, "learning_rate": 8.969898715494506e-05, "loss": 0.010660383850336075, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.01072, "step": 133, "tokens/total": 4592528, "tokens/train_per_sec_per_gpu": 25.65, "tokens/trainable": 60831} +{"epoch": 0.5234375, "grad_norm": 0.5281323790550232, "learning_rate": 8.951343014914869e-05, "loss": 0.009350595995783806, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00939, "step": 134, "tokens/total": 4629104, "tokens/train_per_sec_per_gpu": 24.32, "tokens/trainable": 61305} +{"epoch": 0.52734375, "grad_norm": 0.5913549065589905, "learning_rate": 8.932643689864568e-05, "loss": 0.00099780538585037, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.001, "step": 135, "tokens/total": 4665856, "tokens/train_per_sec_per_gpu": 27.2, "tokens/trainable": 61792} +{"epoch": 0.53125, "grad_norm": 0.5144167542457581, "learning_rate": 8.913801518498845e-05, "loss": 0.009073078632354736, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.00911, "step": 136, "tokens/total": 4704688, "tokens/train_per_sec_per_gpu": 21.26, "tokens/trainable": 62253} +{"epoch": 0.53515625, "grad_norm": 0.45242756605148315, "learning_rate": 8.894817284917364e-05, "loss": 0.0054712435230612755, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00549, "step": 137, "tokens/total": 4741472, "tokens/train_per_sec_per_gpu": 22.71, "tokens/trainable": 62707} +{"epoch": 0.5390625, "grad_norm": 0.06174568086862564, "learning_rate": 8.875691779131569e-05, "loss": 0.0004552614991553128, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00046, "step": 138, "tokens/total": 4778752, "tokens/train_per_sec_per_gpu": 20.68, "tokens/trainable": 63145} +{"epoch": 0.54296875, "grad_norm": 0.7299249172210693, "learning_rate": 8.856425797031829e-05, "loss": 0.009442093782126904, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00949, "step": 139, "tokens/total": 4813904, "tokens/train_per_sec_per_gpu": 25.73, "tokens/trainable": 63587} +{"epoch": 0.546875, "grad_norm": 0.12046497315168381, "learning_rate": 8.837020140354295e-05, "loss": 0.0010707637993618846, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.00107, "step": 140, "tokens/total": 4850272, "tokens/train_per_sec_per_gpu": 26.76, "tokens/trainable": 64052} +{"epoch": 0.55078125, "grad_norm": 0.18457435071468353, "learning_rate": 8.817475616647554e-05, "loss": 0.0028730640187859535, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.00288, "step": 141, "tokens/total": 4884928, "tokens/train_per_sec_per_gpu": 26.48, "tokens/trainable": 64526} +{"epoch": 0.5546875, "grad_norm": 0.31412473320961, "learning_rate": 8.797793039239017e-05, "loss": 0.006915460340678692, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00694, "step": 142, "tokens/total": 4919984, "tokens/train_per_sec_per_gpu": 26.89, "tokens/trainable": 64983} +{"epoch": 0.55859375, "grad_norm": 0.6676502227783203, "learning_rate": 8.777973227201069e-05, "loss": 0.011865361593663692, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.01194, "step": 143, "tokens/total": 4956304, "tokens/train_per_sec_per_gpu": 27.73, "tokens/trainable": 65492} +{"epoch": 0.5625, "grad_norm": 1.5898633003234863, "learning_rate": 8.758017005316988e-05, "loss": 0.011906266212463379, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.01198, "step": 144, "tokens/total": 4991696, "tokens/train_per_sec_per_gpu": 26.53, "tokens/trainable": 65925} +{"epoch": 0.56640625, "grad_norm": 0.3704780638217926, "learning_rate": 8.737925204046629e-05, "loss": 0.0063745444640517235, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00639, "step": 145, "tokens/total": 5026480, "tokens/train_per_sec_per_gpu": 23.96, "tokens/trainable": 66383} +{"epoch": 0.5703125, "grad_norm": 0.2657544016838074, "learning_rate": 8.717698659491851e-05, "loss": 0.0033123234752565622, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00332, "step": 146, "tokens/total": 5063568, "tokens/train_per_sec_per_gpu": 22.31, "tokens/trainable": 66840} +{"epoch": 0.57421875, "grad_norm": 0.5325798988342285, "learning_rate": 8.697338213361735e-05, "loss": 0.005661830771714449, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.00568, "step": 147, "tokens/total": 5099616, "tokens/train_per_sec_per_gpu": 23.31, "tokens/trainable": 67297} +{"epoch": 0.578125, "grad_norm": 0.716082751750946, "learning_rate": 8.676844712937552e-05, "loss": 0.005460962187498808, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.39, "memory/max_allocated (GiB)": 39.39, "ppl": 1.00548, "step": 148, "tokens/total": 5139360, "tokens/train_per_sec_per_gpu": 24.73, "tokens/trainable": 67769} +{"epoch": 0.58203125, "grad_norm": 0.7060474753379822, "learning_rate": 8.656219011037509e-05, "loss": 0.012188063003122807, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.01226, "step": 149, "tokens/total": 5176400, "tokens/train_per_sec_per_gpu": 23.46, "tokens/trainable": 68257} +{"epoch": 0.5859375, "grad_norm": 1.9195575714111328, "learning_rate": 8.63546196598125e-05, "loss": 0.0028872895054519176, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00289, "step": 150, "tokens/total": 5213632, "tokens/train_per_sec_per_gpu": 24.96, "tokens/trainable": 68706} +{"epoch": 0.58984375, "grad_norm": 0.6447552442550659, "learning_rate": 8.614574441554145e-05, "loss": 0.007163134403526783, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.13, "memory/max_allocated (GiB)": 37.13, "ppl": 1.00719, "step": 151, "tokens/total": 5246736, "tokens/train_per_sec_per_gpu": 27.21, "tokens/trainable": 69131} +{"epoch": 0.59375, "grad_norm": 0.6068094372749329, "learning_rate": 8.593557306971349e-05, "loss": 0.006640912499278784, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.44, "memory/max_allocated (GiB)": 37.44, "ppl": 1.00666, "step": 152, "tokens/total": 5279712, "tokens/train_per_sec_per_gpu": 25.8, "tokens/trainable": 69586} +{"epoch": 0.59765625, "grad_norm": 0.37717506289482117, "learning_rate": 8.572411436841618e-05, "loss": 0.0028850568924099207, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00289, "step": 153, "tokens/total": 5313424, "tokens/train_per_sec_per_gpu": 26.38, "tokens/trainable": 70061} +{"epoch": 0.6015625, "grad_norm": 0.5008870363235474, "learning_rate": 8.551137711130922e-05, "loss": 0.0027623786590993404, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.2, "memory/max_allocated (GiB)": 38.2, "ppl": 1.00277, "step": 154, "tokens/total": 5344816, "tokens/train_per_sec_per_gpu": 23.75, "tokens/trainable": 70467} +{"epoch": 0.60546875, "grad_norm": 0.33414483070373535, "learning_rate": 8.529737015125824e-05, "loss": 0.006811519153416157, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00683, "step": 155, "tokens/total": 5379264, "tokens/train_per_sec_per_gpu": 25.67, "tokens/trainable": 70933} +{"epoch": 0.609375, "grad_norm": 0.2187296450138092, "learning_rate": 8.508210239396639e-05, "loss": 0.0017080373363569379, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.00171, "step": 156, "tokens/total": 5416832, "tokens/train_per_sec_per_gpu": 24.05, "tokens/trainable": 71382} +{"epoch": 0.61328125, "grad_norm": 0.17951610684394836, "learning_rate": 8.486558279760375e-05, "loss": 0.0006648038397543132, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 36.07, "memory/max_allocated (GiB)": 36.07, "ppl": 1.00067, "step": 157, "tokens/total": 5447968, "tokens/train_per_sec_per_gpu": 26.49, "tokens/trainable": 71808} +{"epoch": 0.6171875, "grad_norm": 1.7350753545761108, "learning_rate": 8.464782037243449e-05, "loss": 0.032064709812402725, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.03258, "step": 158, "tokens/total": 5480576, "tokens/train_per_sec_per_gpu": 27.25, "tokens/trainable": 72249} +{"epoch": 0.62109375, "grad_norm": 0.6715806722640991, "learning_rate": 8.442882418044202e-05, "loss": 0.014269012026488781, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.01437, "step": 159, "tokens/total": 5513200, "tokens/train_per_sec_per_gpu": 29.16, "tokens/trainable": 72726} +{"epoch": 0.625, "grad_norm": 1.416192889213562, "learning_rate": 8.420860333495179e-05, "loss": 0.03593391180038452, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.03659, "step": 160, "tokens/total": 5547120, "tokens/train_per_sec_per_gpu": 25.62, "tokens/trainable": 73148} +{"epoch": 0.62890625, "grad_norm": 0.2845408320426941, "learning_rate": 8.398716700025208e-05, "loss": 0.0013066548854112625, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.00131, "step": 161, "tokens/total": 5580336, "tokens/train_per_sec_per_gpu": 25.56, "tokens/trainable": 73600} +{"epoch": 0.6328125, "grad_norm": 0.7577224969863892, "learning_rate": 8.376452439121266e-05, "loss": 0.006134620402008295, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 35.89, "memory/max_allocated (GiB)": 35.89, "ppl": 1.00615, "step": 162, "tokens/total": 5610736, "tokens/train_per_sec_per_gpu": 31.76, "tokens/trainable": 74068} +{"epoch": 0.63671875, "grad_norm": 0.9342254400253296, "learning_rate": 8.354068477290124e-05, "loss": 0.019030475988984108, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.01921, "step": 163, "tokens/total": 5645040, "tokens/train_per_sec_per_gpu": 27.68, "tokens/trainable": 74527} +{"epoch": 0.640625, "grad_norm": 0.734091579914093, "learning_rate": 8.331565746019807e-05, "loss": 0.022596362978219986, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.52, "memory/max_allocated (GiB)": 36.52, "ppl": 1.02285, "step": 164, "tokens/total": 5676944, "tokens/train_per_sec_per_gpu": 27.01, "tokens/trainable": 74992} +{"epoch": 0.64453125, "grad_norm": 0.8472474217414856, "learning_rate": 8.308945181740812e-05, "loss": 0.0152506735175848, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.01537, "step": 165, "tokens/total": 5711728, "tokens/train_per_sec_per_gpu": 26.92, "tokens/trainable": 75442} +{"epoch": 0.6484375, "grad_norm": 0.44273465871810913, "learning_rate": 8.286207725787153e-05, "loss": 0.013570528477430344, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.01366, "step": 166, "tokens/total": 5748656, "tokens/train_per_sec_per_gpu": 22.39, "tokens/trainable": 75887} +{"epoch": 0.65234375, "grad_norm": 0.5615953803062439, "learning_rate": 8.263354324357182e-05, "loss": 0.017208226025104523, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.01736, "step": 167, "tokens/total": 5783376, "tokens/train_per_sec_per_gpu": 26.38, "tokens/trainable": 76331} +{"epoch": 0.65625, "grad_norm": 0.20613837242126465, "learning_rate": 8.240385928474219e-05, "loss": 0.007453786209225655, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00748, "step": 168, "tokens/total": 5816512, "tokens/train_per_sec_per_gpu": 26.97, "tokens/trainable": 76745} +{"epoch": 0.66015625, "grad_norm": 0.1471642553806305, "learning_rate": 8.217303493946967e-05, "loss": 0.003563700942322612, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.00357, "step": 169, "tokens/total": 5850944, "tokens/train_per_sec_per_gpu": 26.2, "tokens/trainable": 77201} +{"epoch": 0.6640625, "grad_norm": 0.25366657972335815, "learning_rate": 8.194107981329746e-05, "loss": 0.0033446981105953455, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00335, "step": 170, "tokens/total": 5888912, "tokens/train_per_sec_per_gpu": 23.32, "tokens/trainable": 77655} +{"epoch": 0.66796875, "grad_norm": 0.22393487393856049, "learning_rate": 8.170800355882518e-05, "loss": 0.002755737630650401, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.24, "memory/max_allocated (GiB)": 37.24, "ppl": 1.00276, "step": 171, "tokens/total": 5923136, "tokens/train_per_sec_per_gpu": 30.41, "tokens/trainable": 78122} +{"epoch": 0.671875, "grad_norm": 0.3909287750720978, "learning_rate": 8.147381587530713e-05, "loss": 0.011938199400901794, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.01201, "step": 172, "tokens/total": 5956944, "tokens/train_per_sec_per_gpu": 26.53, "tokens/trainable": 78576} +{"epoch": 0.67578125, "grad_norm": 0.48835787177085876, "learning_rate": 8.123852650824877e-05, "loss": 0.005412860307842493, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00543, "step": 173, "tokens/total": 5990976, "tokens/train_per_sec_per_gpu": 28.45, "tokens/trainable": 79059} +{"epoch": 0.6796875, "grad_norm": 0.3652099370956421, "learning_rate": 8.100214524900103e-05, "loss": 0.003819525707513094, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.52, "memory/max_allocated (GiB)": 38.52, "ppl": 1.00383, "step": 174, "tokens/total": 6026368, "tokens/train_per_sec_per_gpu": 22.21, "tokens/trainable": 79498} +{"epoch": 0.68359375, "grad_norm": 0.03628300130367279, "learning_rate": 8.076468193435301e-05, "loss": 0.00041524055995978415, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.00042, "step": 175, "tokens/total": 6057760, "tokens/train_per_sec_per_gpu": 25.97, "tokens/trainable": 79922} +{"epoch": 0.6875, "grad_norm": 0.44893109798431396, "learning_rate": 8.052614644612253e-05, "loss": 0.004059907980263233, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.00407, "step": 176, "tokens/total": 6095024, "tokens/train_per_sec_per_gpu": 23.33, "tokens/trainable": 80383} +{"epoch": 0.69140625, "grad_norm": 0.4217053949832916, "learning_rate": 8.028654871074489e-05, "loss": 0.002714907517656684, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00272, "step": 177, "tokens/total": 6129712, "tokens/train_per_sec_per_gpu": 25.49, "tokens/trainable": 80824} +{"epoch": 0.6953125, "grad_norm": 0.870136559009552, "learning_rate": 8.004589869885986e-05, "loss": 0.00892002135515213, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.49, "memory/max_allocated (GiB)": 36.49, "ppl": 1.00896, "step": 178, "tokens/total": 6161824, "tokens/train_per_sec_per_gpu": 26.78, "tokens/trainable": 81243} +{"epoch": 0.69921875, "grad_norm": 0.8351891040802002, "learning_rate": 7.980420642489674e-05, "loss": 0.015465978533029556, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.01559, "step": 179, "tokens/total": 6200464, "tokens/train_per_sec_per_gpu": 23.38, "tokens/trainable": 81716} +{"epoch": 0.703125, "grad_norm": 0.04211863875389099, "learning_rate": 7.95614819466576e-05, "loss": 0.0003543120401445776, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00035, "step": 180, "tokens/total": 6234416, "tokens/train_per_sec_per_gpu": 28.34, "tokens/trainable": 82181} +{"epoch": 0.70703125, "grad_norm": 0.12321309745311737, "learning_rate": 7.931773536489872e-05, "loss": 0.0006766939768567681, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.00068, "step": 181, "tokens/total": 6268608, "tokens/train_per_sec_per_gpu": 24.93, "tokens/trainable": 82626} +{"epoch": 0.7109375, "grad_norm": 0.461602121591568, "learning_rate": 7.907297682291035e-05, "loss": 0.006594193633645773, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00662, "step": 182, "tokens/total": 6303792, "tokens/train_per_sec_per_gpu": 26.52, "tokens/trainable": 83089} +{"epoch": 0.71484375, "grad_norm": 0.055674292147159576, "learning_rate": 7.882721650609442e-05, "loss": 0.0006190692656673491, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.00062, "step": 183, "tokens/total": 6343664, "tokens/train_per_sec_per_gpu": 23.24, "tokens/trainable": 83590} +{"epoch": 0.71875, "grad_norm": 0.27312397956848145, "learning_rate": 7.85804646415409e-05, "loss": 0.0024925812613219023, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.0025, "step": 184, "tokens/total": 6378848, "tokens/train_per_sec_per_gpu": 26.8, "tokens/trainable": 84046} +{"epoch": 0.72265625, "grad_norm": 1.337238073348999, "learning_rate": 7.833273149760207e-05, "loss": 0.01640935428440571, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.01654, "step": 185, "tokens/total": 6409968, "tokens/train_per_sec_per_gpu": 28.9, "tokens/trainable": 84517} +{"epoch": 0.7265625, "grad_norm": 0.6396816372871399, "learning_rate": 7.808402738346527e-05, "loss": 0.001407173229381442, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00141, "step": 186, "tokens/total": 6448320, "tokens/train_per_sec_per_gpu": 23.39, "tokens/trainable": 84974} +{"epoch": 0.73046875, "grad_norm": 0.39190152287483215, "learning_rate": 7.783436264872382e-05, "loss": 0.002610502066090703, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.00261, "step": 187, "tokens/total": 6485440, "tokens/train_per_sec_per_gpu": 24.84, "tokens/trainable": 85460} +{"epoch": 0.734375, "grad_norm": 0.36047816276550293, "learning_rate": 7.758374768294647e-05, "loss": 0.006620809435844421, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.05, "memory/max_allocated (GiB)": 38.05, "ppl": 1.00664, "step": 188, "tokens/total": 6521216, "tokens/train_per_sec_per_gpu": 27.01, "tokens/trainable": 85939} +{"epoch": 0.73828125, "grad_norm": 0.3096878230571747, "learning_rate": 7.733219291524489e-05, "loss": 0.0014622211456298828, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.59, "memory/max_allocated (GiB)": 37.59, "ppl": 1.00146, "step": 189, "tokens/total": 6554736, "tokens/train_per_sec_per_gpu": 28.34, "tokens/trainable": 86377} +{"epoch": 0.7421875, "grad_norm": 0.08932711184024811, "learning_rate": 7.707970881383977e-05, "loss": 0.000548347074072808, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.57, "memory/max_allocated (GiB)": 36.57, "ppl": 1.00055, "step": 190, "tokens/total": 6586480, "tokens/train_per_sec_per_gpu": 29.13, "tokens/trainable": 86834} +{"epoch": 0.74609375, "grad_norm": 0.7573031187057495, "learning_rate": 7.682630588562518e-05, "loss": 0.0032234431710094213, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.00323, "step": 191, "tokens/total": 6621536, "tokens/train_per_sec_per_gpu": 24.32, "tokens/trainable": 87265} +{"epoch": 0.75, "grad_norm": 0.08348231762647629, "learning_rate": 7.657199467573129e-05, "loss": 0.0004829954414162785, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.00048, "step": 192, "tokens/total": 6655248, "tokens/train_per_sec_per_gpu": 27.35, "tokens/trainable": 87747} +{"epoch": 0.75390625, "grad_norm": 2.775813341140747, "learning_rate": 7.631678576708561e-05, "loss": 0.007855119183659554, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00789, "step": 193, "tokens/total": 6688960, "tokens/train_per_sec_per_gpu": 29.01, "tokens/trainable": 88239} +{"epoch": 0.7578125, "grad_norm": 0.1578959971666336, "learning_rate": 7.606068977997255e-05, "loss": 0.0008456176146864891, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00085, "step": 194, "tokens/total": 6726256, "tokens/train_per_sec_per_gpu": 25.71, "tokens/trainable": 88718} +{"epoch": 0.76171875, "grad_norm": 0.9209448099136353, "learning_rate": 7.580371737159148e-05, "loss": 0.009302550926804543, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00935, "step": 195, "tokens/total": 6761984, "tokens/train_per_sec_per_gpu": 22.98, "tokens/trainable": 89129} +{"epoch": 0.765625, "grad_norm": 0.010526634752750397, "learning_rate": 7.554587923561324e-05, "loss": 8.905406866688281e-05, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00009, "step": 196, "tokens/total": 6796640, "tokens/train_per_sec_per_gpu": 24.68, "tokens/trainable": 89567} +{"epoch": 0.76953125, "grad_norm": 1.001124382019043, "learning_rate": 7.528718610173511e-05, "loss": 0.01957806758582592, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.48, "memory/max_allocated (GiB)": 37.48, "ppl": 1.01977, "step": 197, "tokens/total": 6831840, "tokens/train_per_sec_per_gpu": 22.84, "tokens/trainable": 89988} +{"epoch": 0.7734375, "grad_norm": 0.33107131719589233, "learning_rate": 7.502764873523431e-05, "loss": 0.0016579454531893134, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.00166, "step": 198, "tokens/total": 6866128, "tokens/train_per_sec_per_gpu": 28.17, "tokens/trainable": 90434} +{"epoch": 0.77734375, "grad_norm": 2.551872491836548, "learning_rate": 7.476727793652011e-05, "loss": 0.02531486749649048, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.71, "memory/max_allocated (GiB)": 36.71, "ppl": 1.02564, "step": 199, "tokens/total": 6898960, "tokens/train_per_sec_per_gpu": 28.13, "tokens/trainable": 90913} +{"epoch": 0.78125, "grad_norm": 0.4684586524963379, "learning_rate": 7.450608454068415e-05, "loss": 0.0113265011459589, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.01139, "step": 200, "tokens/total": 6933312, "tokens/train_per_sec_per_gpu": 26.69, "tokens/trainable": 91355} +{"epoch": 0.78515625, "grad_norm": 0.14710400998592377, "learning_rate": 7.424407941704987e-05, "loss": 0.003306834027171135, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.00331, "step": 201, "tokens/total": 6970016, "tokens/train_per_sec_per_gpu": 27.33, "tokens/trainable": 91856} +{"epoch": 0.7890625, "grad_norm": 0.9723609685897827, "learning_rate": 7.398127346871986e-05, "loss": 0.0275911595672369, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.02798, "step": 202, "tokens/total": 7006736, "tokens/train_per_sec_per_gpu": 24.41, "tokens/trainable": 92311} +{"epoch": 0.79296875, "grad_norm": 0.21090835332870483, "learning_rate": 7.371767763212238e-05, "loss": 0.0017183565068989992, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.00172, "step": 203, "tokens/total": 7041728, "tokens/train_per_sec_per_gpu": 26.56, "tokens/trainable": 92764} +{"epoch": 0.796875, "grad_norm": 0.3449845612049103, "learning_rate": 7.345330287655617e-05, "loss": 0.0038128220476210117, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.00382, "step": 204, "tokens/total": 7079792, "tokens/train_per_sec_per_gpu": 22.71, "tokens/trainable": 93227} +{"epoch": 0.80078125, "grad_norm": 0.11786220967769623, "learning_rate": 7.31881602037339e-05, "loss": 0.0017506125150248408, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.00175, "step": 205, "tokens/total": 7111824, "tokens/train_per_sec_per_gpu": 27.5, "tokens/trainable": 93675} +{"epoch": 0.8046875, "grad_norm": 0.3036588728427887, "learning_rate": 7.29222606473245e-05, "loss": 0.007327320519834757, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.00735, "step": 206, "tokens/total": 7144896, "tokens/train_per_sec_per_gpu": 26.51, "tokens/trainable": 94120} +{"epoch": 0.80859375, "grad_norm": 0.6668539643287659, "learning_rate": 7.265561527249383e-05, "loss": 0.014997996389865875, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.01511, "step": 207, "tokens/total": 7178944, "tokens/train_per_sec_per_gpu": 27.76, "tokens/trainable": 94557} +{"epoch": 0.8125, "grad_norm": 0.05168456956744194, "learning_rate": 7.238823517544436e-05, "loss": 0.0014408925781026483, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.00144, "step": 208, "tokens/total": 7215840, "tokens/train_per_sec_per_gpu": 29.1, "tokens/trainable": 95035} +{"epoch": 0.81640625, "grad_norm": 0.21607103943824768, "learning_rate": 7.212013148295333e-05, "loss": 0.0062438249588012695, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.00626, "step": 209, "tokens/total": 7249952, "tokens/train_per_sec_per_gpu": 29.8, "tokens/trainable": 95517} +{"epoch": 0.8203125, "grad_norm": 0.5583617091178894, "learning_rate": 7.185131535190975e-05, "loss": 0.015590464696288109, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.01571, "step": 210, "tokens/total": 7282352, "tokens/train_per_sec_per_gpu": 30.03, "tokens/trainable": 96026} +{"epoch": 0.82421875, "grad_norm": 0.1852000206708908, "learning_rate": 7.158179796885005e-05, "loss": 0.001916808309033513, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.00192, "step": 211, "tokens/total": 7316768, "tokens/train_per_sec_per_gpu": 27.53, "tokens/trainable": 96477} +{"epoch": 0.828125, "grad_norm": 0.11688517779111862, "learning_rate": 7.131159054949273e-05, "loss": 0.0029810662381350994, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00299, "step": 212, "tokens/total": 7355168, "tokens/train_per_sec_per_gpu": 22.34, "tokens/trainable": 96951} +{"epoch": 0.83203125, "grad_norm": 0.3829125761985779, "learning_rate": 7.104070433827139e-05, "loss": 0.008023976348340511, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00806, "step": 213, "tokens/total": 7390720, "tokens/train_per_sec_per_gpu": 22.97, "tokens/trainable": 97350} +{"epoch": 0.8359375, "grad_norm": 0.13477131724357605, "learning_rate": 7.076915060786705e-05, "loss": 0.001757060643285513, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.96, "memory/max_allocated (GiB)": 37.96, "ppl": 1.00176, "step": 214, "tokens/total": 7423616, "tokens/train_per_sec_per_gpu": 24.28, "tokens/trainable": 97792} +{"epoch": 0.83984375, "grad_norm": 0.11157770454883575, "learning_rate": 7.049694065873882e-05, "loss": 0.001438440172933042, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.00144, "step": 215, "tokens/total": 7458352, "tokens/train_per_sec_per_gpu": 27.97, "tokens/trainable": 98257} +{"epoch": 0.84375, "grad_norm": 0.16377489268779755, "learning_rate": 7.022408581865382e-05, "loss": 0.003061918308958411, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.69, "memory/max_allocated (GiB)": 38.69, "ppl": 1.00307, "step": 216, "tokens/total": 7492672, "tokens/train_per_sec_per_gpu": 25.48, "tokens/trainable": 98731} +{"epoch": 0.84765625, "grad_norm": 0.21084101498126984, "learning_rate": 6.99505974422157e-05, "loss": 0.0017462957184761763, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.00175, "step": 217, "tokens/total": 7526832, "tokens/train_per_sec_per_gpu": 28.42, "tokens/trainable": 99212} +{"epoch": 0.8515625, "grad_norm": 0.14751504361629486, "learning_rate": 6.967648691039213e-05, "loss": 0.0003269420994911343, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.73, "memory/max_allocated (GiB)": 36.73, "ppl": 1.00033, "step": 218, "tokens/total": 7558944, "tokens/train_per_sec_per_gpu": 27.96, "tokens/trainable": 99654} +{"epoch": 0.85546875, "grad_norm": 0.5160031318664551, "learning_rate": 6.940176563004123e-05, "loss": 0.0062631648033857346, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00628, "step": 219, "tokens/total": 7593120, "tokens/train_per_sec_per_gpu": 25.96, "tokens/trainable": 100086} +{"epoch": 0.859375, "grad_norm": 0.6892428398132324, "learning_rate": 6.912644503343682e-05, "loss": 0.0008490928448736668, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.00085, "step": 220, "tokens/total": 7630592, "tokens/train_per_sec_per_gpu": 23.47, "tokens/trainable": 100524} +{"epoch": 0.86328125, "grad_norm": 0.7552803158760071, "learning_rate": 6.885053657779273e-05, "loss": 0.002926081884652376, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.00293, "step": 221, "tokens/total": 7662384, "tokens/train_per_sec_per_gpu": 28.44, "tokens/trainable": 100996} +{"epoch": 0.8671875, "grad_norm": 0.32401061058044434, "learning_rate": 6.857405174478604e-05, "loss": 0.002224648604169488, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.00223, "step": 222, "tokens/total": 7697152, "tokens/train_per_sec_per_gpu": 24.39, "tokens/trainable": 101449} +{"epoch": 0.87109375, "grad_norm": 0.8189313411712646, "learning_rate": 6.82970020400792e-05, "loss": 0.023196334019303322, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.02347, "step": 223, "tokens/total": 7734624, "tokens/train_per_sec_per_gpu": 23.16, "tokens/trainable": 101905} +{"epoch": 0.875, "grad_norm": 0.0795925110578537, "learning_rate": 6.801939899284132e-05, "loss": 0.00025631836615502834, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.34, "memory/max_allocated (GiB)": 38.34, "ppl": 1.00026, "step": 224, "tokens/total": 7770848, "tokens/train_per_sec_per_gpu": 26.61, "tokens/trainable": 102411} +{"epoch": 0.87890625, "grad_norm": 0.16742296516895294, "learning_rate": 6.774125415526827e-05, "loss": 0.0008307142998091877, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00083, "step": 225, "tokens/total": 7803904, "tokens/train_per_sec_per_gpu": 27.8, "tokens/trainable": 102882} +{"epoch": 0.8828125, "grad_norm": 0.09654399007558823, "learning_rate": 6.746257910210214e-05, "loss": 0.00011620146688073874, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.12, "memory/max_allocated (GiB)": 37.12, "ppl": 1.00012, "step": 226, "tokens/total": 7837088, "tokens/train_per_sec_per_gpu": 28.07, "tokens/trainable": 103332} +{"epoch": 0.88671875, "grad_norm": 0.1324658989906311, "learning_rate": 6.718338543014937e-05, "loss": 0.0007699464913457632, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 35.75, "memory/max_allocated (GiB)": 35.75, "ppl": 1.00077, "step": 227, "tokens/total": 7867360, "tokens/train_per_sec_per_gpu": 31.92, "tokens/trainable": 103790} +{"epoch": 0.890625, "grad_norm": 0.5055790543556213, "learning_rate": 6.69036847577983e-05, "loss": 0.0018692209850996733, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.00187, "step": 228, "tokens/total": 7901280, "tokens/train_per_sec_per_gpu": 26.77, "tokens/trainable": 104246} +{"epoch": 0.89453125, "grad_norm": 0.0421692356467247, "learning_rate": 6.662348872453553e-05, "loss": 0.00027910005883313715, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.33, "memory/max_allocated (GiB)": 38.33, "ppl": 1.00028, "step": 229, "tokens/total": 7937536, "tokens/train_per_sec_per_gpu": 27.5, "tokens/trainable": 104707} +{"epoch": 0.8984375, "grad_norm": 0.38500064611434937, "learning_rate": 6.63428089904618e-05, "loss": 0.005510724615305662, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.00553, "step": 230, "tokens/total": 7976544, "tokens/train_per_sec_per_gpu": 24.89, "tokens/trainable": 105167} +{"epoch": 0.90234375, "grad_norm": 0.3526497781276703, "learning_rate": 6.60616572358065e-05, "loss": 0.002694756491109729, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.0027, "step": 231, "tokens/total": 8011008, "tokens/train_per_sec_per_gpu": 25.59, "tokens/trainable": 105576} +{"epoch": 0.90625, "grad_norm": 0.5542545914649963, "learning_rate": 6.578004516044172e-05, "loss": 0.001682044705376029, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00168, "step": 232, "tokens/total": 8046784, "tokens/train_per_sec_per_gpu": 27.55, "tokens/trainable": 106043} +{"epoch": 0.91015625, "grad_norm": 0.8297315835952759, "learning_rate": 6.549798448339548e-05, "loss": 0.004985483828932047, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.59, "memory/max_allocated (GiB)": 38.59, "ppl": 1.005, "step": 233, "tokens/total": 8084048, "tokens/train_per_sec_per_gpu": 25.03, "tokens/trainable": 106506} +{"epoch": 0.9140625, "grad_norm": 0.3997821807861328, "learning_rate": 6.521548694236384e-05, "loss": 0.007818283513188362, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00785, "step": 234, "tokens/total": 8121248, "tokens/train_per_sec_per_gpu": 24.18, "tokens/trainable": 106951} +{"epoch": 0.91796875, "grad_norm": 0.1843184530735016, "learning_rate": 6.493256429322259e-05, "loss": 0.0019267508760094643, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.00193, "step": 235, "tokens/total": 8157504, "tokens/train_per_sec_per_gpu": 22.2, "tokens/trainable": 107382} +{"epoch": 0.921875, "grad_norm": 1.0228379964828491, "learning_rate": 6.464922830953799e-05, "loss": 0.010115869343280792, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.01017, "step": 236, "tokens/total": 8196368, "tokens/train_per_sec_per_gpu": 22.27, "tokens/trainable": 107814} +{"epoch": 0.92578125, "grad_norm": 0.5582643747329712, "learning_rate": 6.436549078207688e-05, "loss": 0.004197406582534313, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00421, "step": 237, "tokens/total": 8229376, "tokens/train_per_sec_per_gpu": 28.76, "tokens/trainable": 108274} +{"epoch": 0.9296875, "grad_norm": 0.5440355539321899, "learning_rate": 6.408136351831592e-05, "loss": 0.004756866954267025, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.00477, "step": 238, "tokens/total": 8261792, "tokens/train_per_sec_per_gpu": 24.09, "tokens/trainable": 108714} +{"epoch": 0.93359375, "grad_norm": 1.0771293640136719, "learning_rate": 6.379685834195036e-05, "loss": 0.004296425264328718, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00431, "step": 239, "tokens/total": 8301216, "tokens/train_per_sec_per_gpu": 24.48, "tokens/trainable": 109220} +{"epoch": 0.9375, "grad_norm": 0.24436114728450775, "learning_rate": 6.351198709240186e-05, "loss": 0.001291723339818418, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.00129, "step": 240, "tokens/total": 8341152, "tokens/train_per_sec_per_gpu": 20.98, "tokens/trainable": 109672} +{"epoch": 0.94140625, "grad_norm": 1.0171009302139282, "learning_rate": 6.32267616243259e-05, "loss": 0.01381587516516447, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.01391, "step": 241, "tokens/total": 8373952, "tokens/train_per_sec_per_gpu": 28.67, "tokens/trainable": 110135} +{"epoch": 0.9453125, "grad_norm": 0.11999757587909698, "learning_rate": 6.294119380711849e-05, "loss": 0.0012081983732059598, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.39, "memory/max_allocated (GiB)": 38.39, "ppl": 1.00121, "step": 242, "tokens/total": 8408240, "tokens/train_per_sec_per_gpu": 23.88, "tokens/trainable": 110563} +{"epoch": 0.94921875, "grad_norm": 0.5071765184402466, "learning_rate": 6.265529552442209e-05, "loss": 0.00891125574707985, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.00895, "step": 243, "tokens/total": 8443344, "tokens/train_per_sec_per_gpu": 27.95, "tokens/trainable": 111049} +{"epoch": 0.953125, "grad_norm": 0.18067876994609833, "learning_rate": 6.236907867363127e-05, "loss": 0.002775164321064949, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.17, "memory/max_allocated (GiB)": 38.17, "ppl": 1.00278, "step": 244, "tokens/total": 8477920, "tokens/train_per_sec_per_gpu": 26.59, "tokens/trainable": 111495} +{"epoch": 0.95703125, "grad_norm": 0.1387961208820343, "learning_rate": 6.208255516539749e-05, "loss": 0.0023042038083076477, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.00231, "step": 245, "tokens/total": 8510240, "tokens/train_per_sec_per_gpu": 26.51, "tokens/trainable": 111968} +{"epoch": 0.9609375, "grad_norm": 0.3383149802684784, "learning_rate": 6.179573692313344e-05, "loss": 0.0011587298940867186, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.8, "memory/max_allocated (GiB)": 38.8, "ppl": 1.00116, "step": 246, "tokens/total": 8547024, "tokens/train_per_sec_per_gpu": 25.97, "tokens/trainable": 112416} +{"epoch": 0.96484375, "grad_norm": 1.0666258335113525, "learning_rate": 6.150863588251694e-05, "loss": 0.014879866503179073, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.01499, "step": 247, "tokens/total": 8586912, "tokens/train_per_sec_per_gpu": 22.61, "tokens/trainable": 112882} +{"epoch": 0.96875, "grad_norm": 0.10628139227628708, "learning_rate": 6.122126399099419e-05, "loss": 0.0008797950576990843, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.00088, "step": 248, "tokens/total": 8626304, "tokens/train_per_sec_per_gpu": 26.77, "tokens/trainable": 113390} +{"epoch": 0.97265625, "grad_norm": 1.3061965703964233, "learning_rate": 6.0933633207282615e-05, "loss": 0.010678409598767757, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.01074, "step": 249, "tokens/total": 8662736, "tokens/train_per_sec_per_gpu": 27.29, "tokens/trainable": 113904} +{"epoch": 0.9765625, "grad_norm": 0.2938941419124603, "learning_rate": 6.064575550087316e-05, "loss": 0.002880371641367674, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00288, "step": 250, "tokens/total": 8699376, "tokens/train_per_sec_per_gpu": 24.61, "tokens/trainable": 114351} +{"epoch": 0.98046875, "grad_norm": 0.12698589265346527, "learning_rate": 6.0357642851532245e-05, "loss": 0.0011687972582876682, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00117, "step": 251, "tokens/total": 8736064, "tokens/train_per_sec_per_gpu": 25.69, "tokens/trainable": 114842} +{"epoch": 0.984375, "grad_norm": 0.4026745557785034, "learning_rate": 6.0069307248803294e-05, "loss": 0.00207472569309175, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00208, "step": 252, "tokens/total": 8771424, "tokens/train_per_sec_per_gpu": 21.88, "tokens/trainable": 115263} +{"epoch": 0.98828125, "grad_norm": 0.29887866973876953, "learning_rate": 5.9780760691507635e-05, "loss": 0.0037822641897946596, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.33, "memory/max_allocated (GiB)": 36.33, "ppl": 1.00379, "step": 253, "tokens/total": 8803168, "tokens/train_per_sec_per_gpu": 27.66, "tokens/trainable": 115703} +{"epoch": 0.9921875, "grad_norm": 0.229067862033844, "learning_rate": 5.9492015187245334e-05, "loss": 0.001288590719923377, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00129, "step": 254, "tokens/total": 8840912, "tokens/train_per_sec_per_gpu": 23.21, "tokens/trainable": 116157} +{"epoch": 0.99609375, "grad_norm": 0.3262993395328522, "learning_rate": 5.920308275189541e-05, "loss": 0.0034696143120527267, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.00348, "step": 255, "tokens/total": 8873184, "tokens/train_per_sec_per_gpu": 26.6, "tokens/trainable": 116567} +{"epoch": 1.0, "grad_norm": 0.23532235622406006, "learning_rate": 5.8913975409115874e-05, "loss": 0.0030662729404866695, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.00307, "step": 256, "tokens/total": 8906544, "tokens/train_per_sec_per_gpu": 25.32, "tokens/trainable": 117030} +{"epoch": 1.00390625, "grad_norm": 0.0629655048251152, "learning_rate": 5.8624705189843395e-05, "loss": 0.000616976642049849, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 1.00062, "step": 257, "tokens/total": 8943760, "tokens/train_per_sec_per_gpu": 24.6, "tokens/trainable": 117517} +{"epoch": 1.0078125, "grad_norm": 0.03813506290316582, "learning_rate": 5.833528413179249e-05, "loss": 0.00041944580152630806, "memory/device_reserved (GiB)": 42.25, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00042, "step": 258, "tokens/total": 8980256, "tokens/train_per_sec_per_gpu": 27.73, "tokens/trainable": 118015} +{"epoch": 1.01171875, "grad_norm": 0.6087254285812378, "learning_rate": 5.80457242789548e-05, "loss": 0.020520325750112534, "memory/device_reserved (GiB)": 42.31, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.02073, "step": 259, "tokens/total": 9020240, "tokens/train_per_sec_per_gpu": 23.76, "tokens/trainable": 118520} +{"epoch": 1.015625, "grad_norm": 0.8461558222770691, "learning_rate": 5.77560376810977e-05, "loss": 0.01012462005019188, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 1.01018, "step": 260, "tokens/total": 9053888, "tokens/train_per_sec_per_gpu": 29.1, "tokens/trainable": 118992} +{"epoch": 1.01953125, "grad_norm": 0.007163515780121088, "learning_rate": 5.7466236393263005e-05, "loss": 0.00013578662765212357, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.00014, "step": 261, "tokens/total": 9088960, "tokens/train_per_sec_per_gpu": 25.47, "tokens/trainable": 119438} +{"epoch": 1.0234375, "grad_norm": 0.06380048394203186, "learning_rate": 5.717633247526522e-05, "loss": 0.0007524421671405435, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00075, "step": 262, "tokens/total": 9120256, "tokens/train_per_sec_per_gpu": 27.74, "tokens/trainable": 119881} +{"epoch": 1.02734375, "grad_norm": 0.07116933166980743, "learning_rate": 5.688633799118971e-05, "loss": 0.0006878282292746007, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.00069, "step": 263, "tokens/total": 9152336, "tokens/train_per_sec_per_gpu": 23.22, "tokens/trainable": 120285} +{"epoch": 1.03125, "grad_norm": 0.053421247750520706, "learning_rate": 5.659626500889066e-05, "loss": 0.0002559400163590908, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00026, "step": 264, "tokens/total": 9186208, "tokens/train_per_sec_per_gpu": 25.74, "tokens/trainable": 120755} +{"epoch": 1.03515625, "grad_norm": 0.23354317247867584, "learning_rate": 5.6306125599488905e-05, "loss": 0.003771612886339426, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.00378, "step": 265, "tokens/total": 9216768, "tokens/train_per_sec_per_gpu": 32.05, "tokens/trainable": 121194} +{"epoch": 1.0390625, "grad_norm": 0.3755282163619995, "learning_rate": 5.601593183686955e-05, "loss": 0.001820030389353633, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.69, "memory/max_allocated (GiB)": 38.69, "ppl": 1.00182, "step": 266, "tokens/total": 9254032, "tokens/train_per_sec_per_gpu": 25.88, "tokens/trainable": 121670} +{"epoch": 1.04296875, "grad_norm": 0.19753636419773102, "learning_rate": 5.572569579717961e-05, "loss": 0.0008868573931977153, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.18, "memory/max_allocated (GiB)": 39.18, "ppl": 1.00089, "step": 267, "tokens/total": 9289552, "tokens/train_per_sec_per_gpu": 25.3, "tokens/trainable": 122142} +{"epoch": 1.046875, "grad_norm": 0.010968021117150784, "learning_rate": 5.543542955832538e-05, "loss": 0.00015612585411872715, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00016, "step": 268, "tokens/total": 9329312, "tokens/train_per_sec_per_gpu": 21.62, "tokens/trainable": 122585} +{"epoch": 1.05078125, "grad_norm": 0.15432167053222656, "learning_rate": 5.514514519946986e-05, "loss": 0.0006270508165471256, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.00063, "step": 269, "tokens/total": 9369136, "tokens/train_per_sec_per_gpu": 25.59, "tokens/trainable": 123091} +{"epoch": 1.0546875, "grad_norm": 0.10648296028375626, "learning_rate": 5.485485480053015e-05, "loss": 0.00037001186865381896, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.00037, "step": 270, "tokens/total": 9407360, "tokens/train_per_sec_per_gpu": 21.93, "tokens/trainable": 123505} +{"epoch": 1.05859375, "grad_norm": 0.0875154659152031, "learning_rate": 5.4564570441674645e-05, "loss": 0.0005251541151665151, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.92, "memory/max_allocated (GiB)": 36.92, "ppl": 1.00053, "step": 271, "tokens/total": 9440624, "tokens/train_per_sec_per_gpu": 26.38, "tokens/trainable": 123953} +{"epoch": 1.0625, "grad_norm": 0.10959743708372116, "learning_rate": 5.42743042028204e-05, "loss": 0.0006517903530038893, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.00065, "step": 272, "tokens/total": 9477120, "tokens/train_per_sec_per_gpu": 20.9, "tokens/trainable": 124400} +{"epoch": 1.06640625, "grad_norm": 0.12212187796831131, "learning_rate": 5.3984068163130464e-05, "loss": 0.0012435748940333724, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.09, "memory/max_allocated (GiB)": 37.09, "ppl": 1.00124, "step": 273, "tokens/total": 9510832, "tokens/train_per_sec_per_gpu": 28.06, "tokens/trainable": 124870} +{"epoch": 1.0703125, "grad_norm": 0.010785705409944057, "learning_rate": 5.369387440051111e-05, "loss": 0.00012451598013285547, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.00012, "step": 274, "tokens/total": 9544368, "tokens/train_per_sec_per_gpu": 25.38, "tokens/trainable": 125333} +{"epoch": 1.07421875, "grad_norm": 0.0682404637336731, "learning_rate": 5.340373499110935e-05, "loss": 0.0005310665001161397, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00053, "step": 275, "tokens/total": 9580240, "tokens/train_per_sec_per_gpu": 27.21, "tokens/trainable": 125834} +{"epoch": 1.078125, "grad_norm": 0.006861691828817129, "learning_rate": 5.3113662008810304e-05, "loss": 7.143788388930261e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00007, "step": 276, "tokens/total": 9620256, "tokens/train_per_sec_per_gpu": 24.91, "tokens/trainable": 126316} +{"epoch": 1.08203125, "grad_norm": 0.7043532729148865, "learning_rate": 5.282366752473479e-05, "loss": 0.009350797161459923, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 1.00939, "step": 277, "tokens/total": 9657264, "tokens/train_per_sec_per_gpu": 25.29, "tokens/trainable": 126798} +{"epoch": 1.0859375, "grad_norm": 0.028922870755195618, "learning_rate": 5.2533763606737005e-05, "loss": 8.645998605061322e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00009, "step": 278, "tokens/total": 9696784, "tokens/train_per_sec_per_gpu": 19.29, "tokens/trainable": 127223} +{"epoch": 1.08984375, "grad_norm": 0.03152308985590935, "learning_rate": 5.224396231890232e-05, "loss": 7.615091453772038e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.00008, "step": 279, "tokens/total": 9728256, "tokens/train_per_sec_per_gpu": 30.0, "tokens/trainable": 127672} +{"epoch": 1.09375, "grad_norm": 0.03149605542421341, "learning_rate": 5.195427572104522e-05, "loss": 0.0004625161527656019, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00046, "step": 280, "tokens/total": 9762688, "tokens/train_per_sec_per_gpu": 24.62, "tokens/trainable": 128116} +{"epoch": 1.09765625, "grad_norm": 0.3470611870288849, "learning_rate": 5.166471586820751e-05, "loss": 0.003864698577672243, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.35, "memory/max_allocated (GiB)": 39.35, "ppl": 1.00387, "step": 281, "tokens/total": 9799296, "tokens/train_per_sec_per_gpu": 23.74, "tokens/trainable": 128589} +{"epoch": 1.1015625, "grad_norm": 0.5691832304000854, "learning_rate": 5.1375294810156615e-05, "loss": 0.00806183461099863, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.00809, "step": 282, "tokens/total": 9834128, "tokens/train_per_sec_per_gpu": 25.87, "tokens/trainable": 129036} +{"epoch": 1.10546875, "grad_norm": 0.10677610337734222, "learning_rate": 5.1086024590884144e-05, "loss": 0.0006110086105763912, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00061, "step": 283, "tokens/total": 9871536, "tokens/train_per_sec_per_gpu": 24.99, "tokens/trainable": 129485} +{"epoch": 1.109375, "grad_norm": 0.0312793105840683, "learning_rate": 5.079691724810461e-05, "loss": 0.00024211901472881436, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.33, "memory/max_allocated (GiB)": 38.33, "ppl": 1.00024, "step": 284, "tokens/total": 9907712, "tokens/train_per_sec_per_gpu": 21.78, "tokens/trainable": 129920} +{"epoch": 1.11328125, "grad_norm": 0.058936089277267456, "learning_rate": 5.0507984812754684e-05, "loss": 0.0005969889461994171, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.17, "memory/max_allocated (GiB)": 37.17, "ppl": 1.0006, "step": 285, "tokens/total": 9941152, "tokens/train_per_sec_per_gpu": 28.74, "tokens/trainable": 130417} +{"epoch": 1.1171875, "grad_norm": 0.008207454346120358, "learning_rate": 5.021923930849237e-05, "loss": 6.98864387231879e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.00007, "step": 286, "tokens/total": 9975472, "tokens/train_per_sec_per_gpu": 27.97, "tokens/trainable": 130903} +{"epoch": 1.12109375, "grad_norm": 0.7056941390037537, "learning_rate": 4.99306927511967e-05, "loss": 0.004789098631590605, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.31, "memory/max_allocated (GiB)": 37.31, "ppl": 1.0048, "step": 287, "tokens/total": 10009280, "tokens/train_per_sec_per_gpu": 27.85, "tokens/trainable": 131343} +{"epoch": 1.125, "grad_norm": 0.8206567168235779, "learning_rate": 4.964235714846775e-05, "loss": 0.018902970477938652, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.72, "memory/max_allocated (GiB)": 36.72, "ppl": 1.01908, "step": 288, "tokens/total": 10041584, "tokens/train_per_sec_per_gpu": 27.2, "tokens/trainable": 131763} +{"epoch": 1.12890625, "grad_norm": 0.04195879027247429, "learning_rate": 4.9354244499126866e-05, "loss": 0.0002128097985405475, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.16, "memory/max_allocated (GiB)": 36.16, "ppl": 1.00021, "step": 289, "tokens/total": 10073312, "tokens/train_per_sec_per_gpu": 29.42, "tokens/trainable": 132229} +{"epoch": 1.1328125, "grad_norm": 0.162461519241333, "learning_rate": 4.90663667927174e-05, "loss": 0.0011770605342462659, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.00118, "step": 290, "tokens/total": 10106160, "tokens/train_per_sec_per_gpu": 28.94, "tokens/trainable": 132678} +{"epoch": 1.13671875, "grad_norm": 0.03153427317738533, "learning_rate": 4.877873600900581e-05, "loss": 0.0003664699906948954, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.00037, "step": 291, "tokens/total": 10142192, "tokens/train_per_sec_per_gpu": 21.71, "tokens/trainable": 133136} +{"epoch": 1.140625, "grad_norm": 0.007608546409755945, "learning_rate": 4.849136411748306e-05, "loss": 7.154315972002223e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.00007, "step": 292, "tokens/total": 10179776, "tokens/train_per_sec_per_gpu": 24.92, "tokens/trainable": 133608} +{"epoch": 1.14453125, "grad_norm": 0.22901487350463867, "learning_rate": 4.8204263076866574e-05, "loss": 0.0024349328596144915, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00244, "step": 293, "tokens/total": 10215424, "tokens/train_per_sec_per_gpu": 29.64, "tokens/trainable": 134108} +{"epoch": 1.1484375, "grad_norm": 0.8298007845878601, "learning_rate": 4.791744483460251e-05, "loss": 0.0027983002364635468, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.0028, "step": 294, "tokens/total": 10249632, "tokens/train_per_sec_per_gpu": 24.28, "tokens/trainable": 134541} +{"epoch": 1.15234375, "grad_norm": 0.11374471336603165, "learning_rate": 4.7630921326368736e-05, "loss": 0.0006863907910883427, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.00069, "step": 295, "tokens/total": 10288272, "tokens/train_per_sec_per_gpu": 20.05, "tokens/trainable": 134966} +{"epoch": 1.15625, "grad_norm": 0.08371077477931976, "learning_rate": 4.7344704475577916e-05, "loss": 0.0004600467218551785, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 35.66, "memory/max_allocated (GiB)": 35.66, "ppl": 1.00046, "step": 296, "tokens/total": 10318160, "tokens/train_per_sec_per_gpu": 30.34, "tokens/trainable": 135402} +{"epoch": 1.16015625, "grad_norm": 0.22348329424858093, "learning_rate": 4.705880619288153e-05, "loss": 0.002735209884122014, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.21, "memory/max_allocated (GiB)": 38.21, "ppl": 1.00274, "step": 297, "tokens/total": 10354304, "tokens/train_per_sec_per_gpu": 22.05, "tokens/trainable": 135804} +{"epoch": 1.1640625, "grad_norm": 0.6118262410163879, "learning_rate": 4.677323837567412e-05, "loss": 0.006377005018293858, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.0064, "step": 298, "tokens/total": 10386960, "tokens/train_per_sec_per_gpu": 25.42, "tokens/trainable": 136231} +{"epoch": 1.16796875, "grad_norm": 0.34327244758605957, "learning_rate": 4.6488012907598146e-05, "loss": 0.002234610030427575, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00224, "step": 299, "tokens/total": 10419712, "tokens/train_per_sec_per_gpu": 28.01, "tokens/trainable": 136705} +{"epoch": 1.171875, "grad_norm": 0.1894824206829071, "learning_rate": 4.620314165804964e-05, "loss": 0.0017565906746312976, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.3, "memory/max_allocated (GiB)": 38.3, "ppl": 1.00176, "step": 300, "tokens/total": 10455376, "tokens/train_per_sec_per_gpu": 25.73, "tokens/trainable": 137178} +{"epoch": 1.17578125, "grad_norm": 0.2580699026584625, "learning_rate": 4.591863648168407e-05, "loss": 0.0013880828628316522, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00139, "step": 301, "tokens/total": 10489712, "tokens/train_per_sec_per_gpu": 26.39, "tokens/trainable": 137643} +{"epoch": 1.1796875, "grad_norm": 0.2901400923728943, "learning_rate": 4.5634509217923135e-05, "loss": 0.003665019292384386, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00367, "step": 302, "tokens/total": 10523056, "tokens/train_per_sec_per_gpu": 25.85, "tokens/trainable": 138078} +{"epoch": 1.18359375, "grad_norm": 0.48309850692749023, "learning_rate": 4.535077169046201e-05, "loss": 0.0032957240473479033, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.0033, "step": 303, "tokens/total": 10559616, "tokens/train_per_sec_per_gpu": 25.19, "tokens/trainable": 138515} +{"epoch": 1.1875, "grad_norm": 0.7607353925704956, "learning_rate": 4.506743570677743e-05, "loss": 0.003339252434670925, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.00334, "step": 304, "tokens/total": 10590208, "tokens/train_per_sec_per_gpu": 31.38, "tokens/trainable": 138948} +{"epoch": 1.19140625, "grad_norm": 0.07037767767906189, "learning_rate": 4.478451305763618e-05, "loss": 0.0006327556911855936, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.95, "memory/max_allocated (GiB)": 36.95, "ppl": 1.00063, "step": 305, "tokens/total": 10623344, "tokens/train_per_sec_per_gpu": 30.62, "tokens/trainable": 139408} +{"epoch": 1.1953125, "grad_norm": 0.2592126131057739, "learning_rate": 4.450201551660454e-05, "loss": 0.0017295947764068842, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.51, "memory/max_allocated (GiB)": 36.51, "ppl": 1.00173, "step": 306, "tokens/total": 10655056, "tokens/train_per_sec_per_gpu": 26.3, "tokens/trainable": 139840} +{"epoch": 1.19921875, "grad_norm": 0.18512122333049774, "learning_rate": 4.4219954839558276e-05, "loss": 0.0005454617203213274, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00055, "step": 307, "tokens/total": 10689744, "tokens/train_per_sec_per_gpu": 25.39, "tokens/trainable": 140281} +{"epoch": 1.203125, "grad_norm": 0.15326522290706635, "learning_rate": 4.393834276419352e-05, "loss": 0.0008349691051989794, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.93, "memory/max_allocated (GiB)": 36.93, "ppl": 1.00084, "step": 308, "tokens/total": 10721456, "tokens/train_per_sec_per_gpu": 31.63, "tokens/trainable": 140776} +{"epoch": 1.20703125, "grad_norm": 0.0007659631664864719, "learning_rate": 4.36571910095382e-05, "loss": 1.745798363117501e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.00002, "step": 309, "tokens/total": 10759568, "tokens/train_per_sec_per_gpu": 25.46, "tokens/trainable": 141228} +{"epoch": 1.2109375, "grad_norm": 0.050811879336833954, "learning_rate": 4.337651127546448e-05, "loss": 0.0003962750779464841, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.0004, "step": 310, "tokens/total": 10795248, "tokens/train_per_sec_per_gpu": 27.59, "tokens/trainable": 141679} +{"epoch": 1.21484375, "grad_norm": 0.3727155327796936, "learning_rate": 4.3096315242201736e-05, "loss": 0.00930188037455082, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.61, "memory/max_allocated (GiB)": 36.61, "ppl": 1.00935, "step": 311, "tokens/total": 10828208, "tokens/train_per_sec_per_gpu": 26.3, "tokens/trainable": 142122} +{"epoch": 1.21875, "grad_norm": 0.07712025195360184, "learning_rate": 4.2816614569850635e-05, "loss": 0.0004936488694511354, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00049, "step": 312, "tokens/total": 10863984, "tokens/train_per_sec_per_gpu": 24.34, "tokens/trainable": 142561} +{"epoch": 1.22265625, "grad_norm": 0.11071080714464188, "learning_rate": 4.2537420897897864e-05, "loss": 0.0005718371248804033, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.00057, "step": 313, "tokens/total": 10899392, "tokens/train_per_sec_per_gpu": 27.01, "tokens/trainable": 143046} +{"epoch": 1.2265625, "grad_norm": 0.23070219159126282, "learning_rate": 4.225874584473174e-05, "loss": 0.0019124674145132303, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.00191, "step": 314, "tokens/total": 10932224, "tokens/train_per_sec_per_gpu": 27.09, "tokens/trainable": 143493} +{"epoch": 1.23046875, "grad_norm": 1.0787447690963745, "learning_rate": 4.19806010071587e-05, "loss": 0.009417520835995674, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.00946, "step": 315, "tokens/total": 10970096, "tokens/train_per_sec_per_gpu": 24.34, "tokens/trainable": 143956} +{"epoch": 1.234375, "grad_norm": 0.1768796145915985, "learning_rate": 4.170299795992081e-05, "loss": 0.000228704244364053, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.00023, "step": 316, "tokens/total": 11003488, "tokens/train_per_sec_per_gpu": 27.05, "tokens/trainable": 144411} +{"epoch": 1.23828125, "grad_norm": 0.056930769234895706, "learning_rate": 4.142594825521398e-05, "loss": 0.00012379477266222239, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.00012, "step": 317, "tokens/total": 11042784, "tokens/train_per_sec_per_gpu": 25.96, "tokens/trainable": 144892} +{"epoch": 1.2421875, "grad_norm": 0.01924307458102703, "learning_rate": 4.114946342220728e-05, "loss": 0.0001428637042408809, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.00014, "step": 318, "tokens/total": 11078992, "tokens/train_per_sec_per_gpu": 23.93, "tokens/trainable": 145339} +{"epoch": 1.24609375, "grad_norm": 0.06818341463804245, "learning_rate": 4.087355496656321e-05, "loss": 6.652304728049785e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00007, "step": 319, "tokens/total": 11113024, "tokens/train_per_sec_per_gpu": 28.42, "tokens/trainable": 145811} +{"epoch": 1.25, "grad_norm": 0.8614652752876282, "learning_rate": 4.05982343699588e-05, "loss": 0.011895937845110893, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.01197, "step": 320, "tokens/total": 11152240, "tokens/train_per_sec_per_gpu": 26.01, "tokens/trainable": 146314} +{"epoch": 1.25390625, "grad_norm": 0.020731039345264435, "learning_rate": 4.0323513089607876e-05, "loss": 0.00020953206694684923, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.00021, "step": 321, "tokens/total": 11187296, "tokens/train_per_sec_per_gpu": 24.33, "tokens/trainable": 146781} +{"epoch": 1.2578125, "grad_norm": 0.012513116002082825, "learning_rate": 4.004940255778431e-05, "loss": 0.00010940350330201909, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00011, "step": 322, "tokens/total": 11225904, "tokens/train_per_sec_per_gpu": 23.94, "tokens/trainable": 147225} +{"epoch": 1.26171875, "grad_norm": 0.02471783198416233, "learning_rate": 3.977591418134619e-05, "loss": 0.0002053080970654264, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.00021, "step": 323, "tokens/total": 11261056, "tokens/train_per_sec_per_gpu": 26.78, "tokens/trainable": 147673} +{"epoch": 1.265625, "grad_norm": 0.005613910499960184, "learning_rate": 3.95030593412612e-05, "loss": 5.8761648688232526e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.00006, "step": 324, "tokens/total": 11296432, "tokens/train_per_sec_per_gpu": 24.63, "tokens/trainable": 148103} +{"epoch": 1.26953125, "grad_norm": 0.009034669026732445, "learning_rate": 3.923084939213296e-05, "loss": 0.00012896822590846568, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 35.88, "memory/max_allocated (GiB)": 35.88, "ppl": 1.00013, "step": 325, "tokens/total": 11327248, "tokens/train_per_sec_per_gpu": 27.86, "tokens/trainable": 148535} +{"epoch": 1.2734375, "grad_norm": 0.012952675111591816, "learning_rate": 3.895929566172861e-05, "loss": 0.00011234349949518219, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.00011, "step": 326, "tokens/total": 11363264, "tokens/train_per_sec_per_gpu": 25.24, "tokens/trainable": 148999} +{"epoch": 1.27734375, "grad_norm": 0.009109044447541237, "learning_rate": 3.868840945050728e-05, "loss": 0.00010226944868918508, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.05, "memory/max_allocated (GiB)": 39.05, "ppl": 1.0001, "step": 327, "tokens/total": 11401776, "tokens/train_per_sec_per_gpu": 21.56, "tokens/trainable": 149431} +{"epoch": 1.28125, "grad_norm": 0.005064785946160555, "learning_rate": 3.841820203114995e-05, "loss": 6.497255526483059e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.00006, "step": 328, "tokens/total": 11433520, "tokens/train_per_sec_per_gpu": 29.12, "tokens/trainable": 149886} +{"epoch": 1.28515625, "grad_norm": 0.0039073266088962555, "learning_rate": 3.814868464809027e-05, "loss": 4.696789619629271e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.00005, "step": 329, "tokens/total": 11465376, "tokens/train_per_sec_per_gpu": 25.88, "tokens/trainable": 150288} +{"epoch": 1.2890625, "grad_norm": 0.019217565655708313, "learning_rate": 3.787986851704667e-05, "loss": 0.00016348485951311886, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.00016, "step": 330, "tokens/total": 11499120, "tokens/train_per_sec_per_gpu": 26.65, "tokens/trainable": 150733} +{"epoch": 1.29296875, "grad_norm": 0.1973566710948944, "learning_rate": 3.7611764824555654e-05, "loss": 0.0003927754587493837, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00039, "step": 331, "tokens/total": 11531104, "tokens/train_per_sec_per_gpu": 31.04, "tokens/trainable": 151207} +{"epoch": 1.296875, "grad_norm": 0.005261009559035301, "learning_rate": 3.734438472750619e-05, "loss": 5.4922060371609405e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.00005, "step": 332, "tokens/total": 11571040, "tokens/train_per_sec_per_gpu": 24.69, "tokens/trainable": 151694} +{"epoch": 1.30078125, "grad_norm": 0.12043075263500214, "learning_rate": 3.707773935267552e-05, "loss": 0.001043935539200902, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.46, "memory/max_allocated (GiB)": 38.46, "ppl": 1.00104, "step": 333, "tokens/total": 11607952, "tokens/train_per_sec_per_gpu": 27.78, "tokens/trainable": 152188} +{"epoch": 1.3046875, "grad_norm": 0.005978281144052744, "learning_rate": 3.68118397962661e-05, "loss": 6.75565970595926e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.00007, "step": 334, "tokens/total": 11642864, "tokens/train_per_sec_per_gpu": 23.32, "tokens/trainable": 152596} +{"epoch": 1.30859375, "grad_norm": 0.1826092004776001, "learning_rate": 3.654669712344384e-05, "loss": 0.0018815492512658238, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 1.00188, "step": 335, "tokens/total": 11673904, "tokens/train_per_sec_per_gpu": 31.72, "tokens/trainable": 153052} +{"epoch": 1.3125, "grad_norm": 0.14002569019794464, "learning_rate": 3.628232236787763e-05, "loss": 0.0012310126330703497, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.96, "memory/max_allocated (GiB)": 37.96, "ppl": 1.00123, "step": 336, "tokens/total": 11709536, "tokens/train_per_sec_per_gpu": 22.71, "tokens/trainable": 153491} +{"epoch": 1.31640625, "grad_norm": 0.00857535656541586, "learning_rate": 3.6018726531280144e-05, "loss": 0.00014637841377407312, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00015, "step": 337, "tokens/total": 11747248, "tokens/train_per_sec_per_gpu": 28.34, "tokens/trainable": 153983} +{"epoch": 1.3203125, "grad_norm": 0.2891673147678375, "learning_rate": 3.575592058295017e-05, "loss": 0.0014350083656609058, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.00144, "step": 338, "tokens/total": 11781104, "tokens/train_per_sec_per_gpu": 28.79, "tokens/trainable": 154464} +{"epoch": 1.32421875, "grad_norm": 0.0015927597414702177, "learning_rate": 3.549391545931585e-05, "loss": 2.1271980585879646e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00002, "step": 339, "tokens/total": 11815712, "tokens/train_per_sec_per_gpu": 27.78, "tokens/trainable": 154924} +{"epoch": 1.328125, "grad_norm": 0.030146799981594086, "learning_rate": 3.5232722063479914e-05, "loss": 0.00013002143532503396, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.00013, "step": 340, "tokens/total": 11849424, "tokens/train_per_sec_per_gpu": 24.23, "tokens/trainable": 155373} +{"epoch": 1.33203125, "grad_norm": 0.021238159388303757, "learning_rate": 3.49723512647657e-05, "loss": 0.0002125828032149002, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.00021, "step": 341, "tokens/total": 11888128, "tokens/train_per_sec_per_gpu": 24.86, "tokens/trainable": 155873} +{"epoch": 1.3359375, "grad_norm": 0.041507538408041, "learning_rate": 3.471281389826491e-05, "loss": 0.00011407441343180835, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.00011, "step": 342, "tokens/total": 11919264, "tokens/train_per_sec_per_gpu": 26.01, "tokens/trainable": 156278} +{"epoch": 1.33984375, "grad_norm": 0.1551242619752884, "learning_rate": 3.4454120764386764e-05, "loss": 0.0011146615725010633, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.66, "memory/max_allocated (GiB)": 38.66, "ppl": 1.00112, "step": 343, "tokens/total": 11957456, "tokens/train_per_sec_per_gpu": 24.36, "tokens/trainable": 156733} +{"epoch": 1.34375, "grad_norm": 0.1597115695476532, "learning_rate": 3.4196282628408526e-05, "loss": 0.002745626261457801, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.00275, "step": 344, "tokens/total": 11991696, "tokens/train_per_sec_per_gpu": 26.37, "tokens/trainable": 157203} +{"epoch": 1.34765625, "grad_norm": 0.17503982782363892, "learning_rate": 3.3939310220027456e-05, "loss": 0.0008772860164754093, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00088, "step": 345, "tokens/total": 12025840, "tokens/train_per_sec_per_gpu": 29.98, "tokens/trainable": 157707} +{"epoch": 1.3515625, "grad_norm": 0.17523975670337677, "learning_rate": 3.3683214232914404e-05, "loss": 0.0017617061967030168, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.00176, "step": 346, "tokens/total": 12061296, "tokens/train_per_sec_per_gpu": 26.22, "tokens/trainable": 158180} +{"epoch": 1.35546875, "grad_norm": 0.0767066553235054, "learning_rate": 3.342800532426873e-05, "loss": 0.00028144961106590927, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.00028, "step": 347, "tokens/total": 12097232, "tokens/train_per_sec_per_gpu": 25.5, "tokens/trainable": 158646} +{"epoch": 1.359375, "grad_norm": 0.0037480564787983894, "learning_rate": 3.317369411437484e-05, "loss": 4.801286559086293e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.00005, "step": 348, "tokens/total": 12133200, "tokens/train_per_sec_per_gpu": 26.23, "tokens/trainable": 159115} +{"epoch": 1.36328125, "grad_norm": 0.025165360420942307, "learning_rate": 3.292029118616024e-05, "loss": 0.0001983548718271777, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.0002, "step": 349, "tokens/total": 12166336, "tokens/train_per_sec_per_gpu": 25.68, "tokens/trainable": 159538} +{"epoch": 1.3671875, "grad_norm": 0.17722587287425995, "learning_rate": 3.266780708475511e-05, "loss": 0.0006466985214501619, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00065, "step": 350, "tokens/total": 12201872, "tokens/train_per_sec_per_gpu": 25.11, "tokens/trainable": 160016} +{"epoch": 1.37109375, "grad_norm": 0.004948556888848543, "learning_rate": 3.241625231705354e-05, "loss": 5.5570839322172105e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.00006, "step": 351, "tokens/total": 12238016, "tokens/train_per_sec_per_gpu": 25.08, "tokens/trainable": 160475} +{"epoch": 1.375, "grad_norm": 0.018348420038819313, "learning_rate": 3.216563735127618e-05, "loss": 8.903859998099506e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.15, "memory/max_allocated (GiB)": 39.15, "ppl": 1.00009, "step": 352, "tokens/total": 12273280, "tokens/train_per_sec_per_gpu": 25.62, "tokens/trainable": 160952} +{"epoch": 1.37890625, "grad_norm": 0.012163287959992886, "learning_rate": 3.191597261653475e-05, "loss": 3.861098230117932e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00004, "step": 353, "tokens/total": 12310096, "tokens/train_per_sec_per_gpu": 23.77, "tokens/trainable": 161420} +{"epoch": 1.3828125, "grad_norm": 0.15110833942890167, "learning_rate": 3.166726850239794e-05, "loss": 0.0006451289518736303, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00065, "step": 354, "tokens/total": 12343136, "tokens/train_per_sec_per_gpu": 27.52, "tokens/trainable": 161868} +{"epoch": 1.38671875, "grad_norm": 0.0016213739290833473, "learning_rate": 3.141953535845912e-05, "loss": 2.6949568564305082e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.34, "memory/max_allocated (GiB)": 36.34, "ppl": 1.00003, "step": 355, "tokens/total": 12374576, "tokens/train_per_sec_per_gpu": 30.47, "tokens/trainable": 162318} +{"epoch": 1.390625, "grad_norm": 0.016374019905924797, "learning_rate": 3.11727834939056e-05, "loss": 0.00015328798326663673, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00015, "step": 356, "tokens/total": 12406896, "tokens/train_per_sec_per_gpu": 26.4, "tokens/trainable": 162789} +{"epoch": 1.39453125, "grad_norm": 0.29322272539138794, "learning_rate": 3.092702317708967e-05, "loss": 0.0025425662752240896, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.00255, "step": 357, "tokens/total": 12440480, "tokens/train_per_sec_per_gpu": 28.95, "tokens/trainable": 163254} +{"epoch": 1.3984375, "grad_norm": 0.0008539243717677891, "learning_rate": 3.0682264635101276e-05, "loss": 1.4031882528797723e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.00001, "step": 358, "tokens/total": 12473232, "tokens/train_per_sec_per_gpu": 28.31, "tokens/trainable": 163715} +{"epoch": 1.40234375, "grad_norm": 0.005317005328834057, "learning_rate": 3.0438518053342407e-05, "loss": 4.209448889014311e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00004, "step": 359, "tokens/total": 12512176, "tokens/train_per_sec_per_gpu": 23.27, "tokens/trainable": 164197} +{"epoch": 1.40625, "grad_norm": 0.0144735686480999, "learning_rate": 3.0195793575103266e-05, "loss": 1.964357761607971e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00002, "step": 360, "tokens/total": 12551136, "tokens/train_per_sec_per_gpu": 21.3, "tokens/trainable": 164661} +{"epoch": 1.41015625, "grad_norm": 0.10772012919187546, "learning_rate": 2.9954101301140146e-05, "loss": 0.00017334104632027447, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.57, "memory/max_allocated (GiB)": 37.57, "ppl": 1.00017, "step": 361, "tokens/total": 12586560, "tokens/train_per_sec_per_gpu": 24.64, "tokens/trainable": 165111} +{"epoch": 1.4140625, "grad_norm": 0.014466114342212677, "learning_rate": 2.9713451289255123e-05, "loss": 8.35994360386394e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.63, "memory/max_allocated (GiB)": 36.63, "ppl": 1.00008, "step": 362, "tokens/total": 12615936, "tokens/train_per_sec_per_gpu": 25.61, "tokens/trainable": 165552} +{"epoch": 1.41796875, "grad_norm": 0.005779011640697718, "learning_rate": 2.9473853553877484e-05, "loss": 3.5102748370263726e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.00004, "step": 363, "tokens/total": 12650096, "tokens/train_per_sec_per_gpu": 27.29, "tokens/trainable": 166026} +{"epoch": 1.421875, "grad_norm": 0.03736929967999458, "learning_rate": 2.9235318065647e-05, "loss": 0.00022909983817953616, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.00023, "step": 364, "tokens/total": 12689808, "tokens/train_per_sec_per_gpu": 24.96, "tokens/trainable": 166486} +{"epoch": 1.42578125, "grad_norm": 0.004905987530946732, "learning_rate": 2.8997854750998964e-05, "loss": 3.719414962688461e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.46, "memory/max_allocated (GiB)": 39.46, "ppl": 1.00004, "step": 365, "tokens/total": 12729328, "tokens/train_per_sec_per_gpu": 24.0, "tokens/trainable": 166959} +{"epoch": 1.4296875, "grad_norm": 0.0010104522807523608, "learning_rate": 2.8761473491751258e-05, "loss": 1.2613390026672278e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.00001, "step": 366, "tokens/total": 12762576, "tokens/train_per_sec_per_gpu": 26.97, "tokens/trainable": 167394} +{"epoch": 1.43359375, "grad_norm": 0.1392313539981842, "learning_rate": 2.8526184124692883e-05, "loss": 0.00027441931888461113, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00027, "step": 367, "tokens/total": 12798944, "tokens/train_per_sec_per_gpu": 20.44, "tokens/trainable": 167832} +{"epoch": 1.4375, "grad_norm": 0.041534874588251114, "learning_rate": 2.829199644117484e-05, "loss": 4.636321682482958e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.00005, "step": 368, "tokens/total": 12832928, "tokens/train_per_sec_per_gpu": 24.42, "tokens/trainable": 168242} +{"epoch": 1.44140625, "grad_norm": 0.002444581128656864, "learning_rate": 2.8058920186702553e-05, "loss": 2.5098173864535056e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00003, "step": 369, "tokens/total": 12870544, "tokens/train_per_sec_per_gpu": 25.05, "tokens/trainable": 168715} +{"epoch": 1.4453125, "grad_norm": 0.005594417452812195, "learning_rate": 2.782696506053033e-05, "loss": 3.659278081613593e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.00004, "step": 370, "tokens/total": 12908416, "tokens/train_per_sec_per_gpu": 22.41, "tokens/trainable": 169147} +{"epoch": 1.44921875, "grad_norm": 0.7990931272506714, "learning_rate": 2.7596140715257824e-05, "loss": 0.004681172780692577, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00469, "step": 371, "tokens/total": 12943584, "tokens/train_per_sec_per_gpu": 25.76, "tokens/trainable": 169628} +{"epoch": 1.453125, "grad_norm": 0.19900520145893097, "learning_rate": 2.7366456756428184e-05, "loss": 0.0009949528612196445, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 1.001, "step": 372, "tokens/total": 12978192, "tokens/train_per_sec_per_gpu": 27.23, "tokens/trainable": 170085} +{"epoch": 1.45703125, "grad_norm": 0.020691165700554848, "learning_rate": 2.7137922742128486e-05, "loss": 0.00011147643090225756, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00011, "step": 373, "tokens/total": 13012720, "tokens/train_per_sec_per_gpu": 27.1, "tokens/trainable": 170584} +{"epoch": 1.4609375, "grad_norm": 0.3731211721897125, "learning_rate": 2.691054818259188e-05, "loss": 0.003752685384824872, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00376, "step": 374, "tokens/total": 13047568, "tokens/train_per_sec_per_gpu": 26.49, "tokens/trainable": 171058} +{"epoch": 1.46484375, "grad_norm": 0.046719443053007126, "learning_rate": 2.6684342539801933e-05, "loss": 3.469971125014126e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.56, "memory/max_allocated (GiB)": 36.56, "ppl": 1.00003, "step": 375, "tokens/total": 13079088, "tokens/train_per_sec_per_gpu": 28.91, "tokens/trainable": 171518} +{"epoch": 1.46875, "grad_norm": 0.002037692815065384, "learning_rate": 2.645931522709877e-05, "loss": 1.8847425963031128e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.00002, "step": 376, "tokens/total": 13114480, "tokens/train_per_sec_per_gpu": 30.17, "tokens/trainable": 172007} +{"epoch": 1.47265625, "grad_norm": 3.5039684772491455, "learning_rate": 2.6235475608787365e-05, "loss": 0.012205737642943859, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.01228, "step": 377, "tokens/total": 13150160, "tokens/train_per_sec_per_gpu": 25.62, "tokens/trainable": 172468} +{"epoch": 1.4765625, "grad_norm": 0.02947821281850338, "learning_rate": 2.6012832999747916e-05, "loss": 0.00010622503032209352, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.00011, "step": 378, "tokens/total": 13181280, "tokens/train_per_sec_per_gpu": 29.38, "tokens/trainable": 172923} +{"epoch": 1.48046875, "grad_norm": 0.9914183020591736, "learning_rate": 2.579139666504821e-05, "loss": 0.021375227719545364, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.02161, "step": 379, "tokens/total": 13221264, "tokens/train_per_sec_per_gpu": 22.21, "tokens/trainable": 173370} +{"epoch": 1.484375, "grad_norm": 0.00229401933029294, "learning_rate": 2.557117581955798e-05, "loss": 2.7148642402607948e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00003, "step": 380, "tokens/total": 13255104, "tokens/train_per_sec_per_gpu": 27.05, "tokens/trainable": 173837} +{"epoch": 1.48828125, "grad_norm": 0.007956113666296005, "learning_rate": 2.5352179627565532e-05, "loss": 3.0106622944003902e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.00003, "step": 381, "tokens/total": 13289856, "tokens/train_per_sec_per_gpu": 27.25, "tokens/trainable": 174294} +{"epoch": 1.4921875, "grad_norm": 0.017284216359257698, "learning_rate": 2.5134417202396277e-05, "loss": 0.0001444591034669429, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.6, "memory/max_allocated (GiB)": 38.6, "ppl": 1.00014, "step": 382, "tokens/total": 13323840, "tokens/train_per_sec_per_gpu": 24.07, "tokens/trainable": 174701} +{"epoch": 1.49609375, "grad_norm": 0.38691845536231995, "learning_rate": 2.491789760603361e-05, "loss": 0.00482302438467741, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00483, "step": 383, "tokens/total": 13360256, "tokens/train_per_sec_per_gpu": 23.28, "tokens/trainable": 175131} +{"epoch": 1.5, "grad_norm": 0.2687751054763794, "learning_rate": 2.4702629848741764e-05, "loss": 0.00215134653262794, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.00215, "step": 384, "tokens/total": 13394304, "tokens/train_per_sec_per_gpu": 26.59, "tokens/trainable": 175586} +{"epoch": 1.50390625, "grad_norm": 0.22292429208755493, "learning_rate": 2.4488622888690785e-05, "loss": 0.0014320781920105219, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.04, "memory/max_allocated (GiB)": 37.04, "ppl": 1.00143, "step": 385, "tokens/total": 13427760, "tokens/train_per_sec_per_gpu": 27.1, "tokens/trainable": 176026} +{"epoch": 1.5078125, "grad_norm": 0.0035524757113307714, "learning_rate": 2.427588563158384e-05, "loss": 4.0838192944647744e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00004, "step": 386, "tokens/total": 13462096, "tokens/train_per_sec_per_gpu": 26.27, "tokens/trainable": 176512} +{"epoch": 1.51171875, "grad_norm": 0.3015328347682953, "learning_rate": 2.406442693028651e-05, "loss": 0.002244369126856327, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.63, "memory/max_allocated (GiB)": 38.63, "ppl": 1.00225, "step": 387, "tokens/total": 13494976, "tokens/train_per_sec_per_gpu": 24.97, "tokens/trainable": 176943} +{"epoch": 1.515625, "grad_norm": 0.001727064372971654, "learning_rate": 2.3854255584458547e-05, "loss": 1.847042585723102e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.00002, "step": 388, "tokens/total": 13533568, "tokens/train_per_sec_per_gpu": 21.65, "tokens/trainable": 177370} +{"epoch": 1.51953125, "grad_norm": 0.010844920761883259, "learning_rate": 2.3645380340187508e-05, "loss": 0.00010514340101508424, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00011, "step": 389, "tokens/total": 13569248, "tokens/train_per_sec_per_gpu": 26.15, "tokens/trainable": 177851} +{"epoch": 1.5234375, "grad_norm": 0.030150117352604866, "learning_rate": 2.3437809889624914e-05, "loss": 0.0001526390842627734, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.00015, "step": 390, "tokens/total": 13601568, "tokens/train_per_sec_per_gpu": 28.91, "tokens/trainable": 178310} +{"epoch": 1.52734375, "grad_norm": 0.03328951075673103, "learning_rate": 2.3231552870624487e-05, "loss": 0.0002515298256184906, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.47, "memory/max_allocated (GiB)": 38.47, "ppl": 1.00025, "step": 391, "tokens/total": 13632576, "tokens/train_per_sec_per_gpu": 29.21, "tokens/trainable": 178736} +{"epoch": 1.53125, "grad_norm": 0.09422493726015091, "learning_rate": 2.3026617866382657e-05, "loss": 0.0005099252448417246, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.53, "memory/max_allocated (GiB)": 36.53, "ppl": 1.00051, "step": 392, "tokens/total": 13665328, "tokens/train_per_sec_per_gpu": 29.98, "tokens/trainable": 179233} +{"epoch": 1.53515625, "grad_norm": 0.06312515586614609, "learning_rate": 2.2823013405081507e-05, "loss": 0.00040735676884651184, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.00041, "step": 393, "tokens/total": 13703408, "tokens/train_per_sec_per_gpu": 26.92, "tokens/trainable": 179730} +{"epoch": 1.5390625, "grad_norm": 0.024848848581314087, "learning_rate": 2.2620747959533722e-05, "loss": 0.00022644043201580644, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.76, "memory/max_allocated (GiB)": 36.76, "ppl": 1.00023, "step": 394, "tokens/total": 13736480, "tokens/train_per_sec_per_gpu": 30.78, "tokens/trainable": 180227} +{"epoch": 1.54296875, "grad_norm": 0.5718985199928284, "learning_rate": 2.2419829946830123e-05, "loss": 0.0006788068567402661, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.00068, "step": 395, "tokens/total": 13772672, "tokens/train_per_sec_per_gpu": 24.26, "tokens/trainable": 180687} +{"epoch": 1.546875, "grad_norm": 0.0334046371281147, "learning_rate": 2.2220267727989325e-05, "loss": 0.00024710249272175133, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00025, "step": 396, "tokens/total": 13806816, "tokens/train_per_sec_per_gpu": 25.85, "tokens/trainable": 181141} +{"epoch": 1.55078125, "grad_norm": 0.2301333248615265, "learning_rate": 2.202206960760984e-05, "loss": 0.0011166189797222614, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.00112, "step": 397, "tokens/total": 13844960, "tokens/train_per_sec_per_gpu": 24.81, "tokens/trainable": 181648} +{"epoch": 1.5546875, "grad_norm": 0.32994168996810913, "learning_rate": 2.182524383352446e-05, "loss": 0.0012127157533541322, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00121, "step": 398, "tokens/total": 13881120, "tokens/train_per_sec_per_gpu": 24.97, "tokens/trainable": 182109} +{"epoch": 1.55859375, "grad_norm": 0.012176831252872944, "learning_rate": 2.1629798596457056e-05, "loss": 0.00014266540529206395, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.52, "memory/max_allocated (GiB)": 36.52, "ppl": 1.00014, "step": 399, "tokens/total": 13913440, "tokens/train_per_sec_per_gpu": 26.73, "tokens/trainable": 182559} +{"epoch": 1.5625, "grad_norm": 0.007234586402773857, "learning_rate": 2.1435742029681725e-05, "loss": 8.564158633816987e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.00009, "step": 400, "tokens/total": 13945408, "tokens/train_per_sec_per_gpu": 27.74, "tokens/trainable": 182985} +{"epoch": 1.56640625, "grad_norm": 0.028555994853377342, "learning_rate": 2.124308220868431e-05, "loss": 0.00013236157246865332, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.67, "memory/max_allocated (GiB)": 38.67, "ppl": 1.00013, "step": 401, "tokens/total": 13980800, "tokens/train_per_sec_per_gpu": 24.37, "tokens/trainable": 183440} +{"epoch": 1.5703125, "grad_norm": 0.02953837439417839, "learning_rate": 2.105182715082638e-05, "loss": 0.0002703022910282016, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00027, "step": 402, "tokens/total": 14014400, "tokens/train_per_sec_per_gpu": 27.67, "tokens/trainable": 183917} +{"epoch": 1.57421875, "grad_norm": 0.145696222782135, "learning_rate": 2.0861984815011552e-05, "loss": 0.00047743486356921494, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.16, "memory/max_allocated (GiB)": 38.16, "ppl": 1.00048, "step": 403, "tokens/total": 14049920, "tokens/train_per_sec_per_gpu": 22.64, "tokens/trainable": 184348} +{"epoch": 1.578125, "grad_norm": 0.037550024688243866, "learning_rate": 2.0673563101354323e-05, "loss": 0.00031577126355841756, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00032, "step": 404, "tokens/total": 14085200, "tokens/train_per_sec_per_gpu": 23.63, "tokens/trainable": 184786} +{"epoch": 1.58203125, "grad_norm": 0.00892274733632803, "learning_rate": 2.0486569850851317e-05, "loss": 8.167035412043333e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.00008, "step": 405, "tokens/total": 14120208, "tokens/train_per_sec_per_gpu": 25.95, "tokens/trainable": 185249} +{"epoch": 1.5859375, "grad_norm": 0.02426115609705448, "learning_rate": 2.0301012845054956e-05, "loss": 0.00033163081388920546, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.00033, "step": 406, "tokens/total": 14158960, "tokens/train_per_sec_per_gpu": 21.49, "tokens/trainable": 185680} +{"epoch": 1.58984375, "grad_norm": 0.05769575014710426, "learning_rate": 2.011689980574966e-05, "loss": 0.00028261594707146287, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.06, "memory/max_allocated (GiB)": 38.06, "ppl": 1.00028, "step": 407, "tokens/total": 14195264, "tokens/train_per_sec_per_gpu": 23.0, "tokens/trainable": 186100} +{"epoch": 1.59375, "grad_norm": 0.015883995220065117, "learning_rate": 1.993423839463052e-05, "loss": 7.207555609056726e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.00007, "step": 408, "tokens/total": 14228528, "tokens/train_per_sec_per_gpu": 32.0, "tokens/trainable": 186565} +{"epoch": 1.59765625, "grad_norm": 0.005861098412424326, "learning_rate": 1.975303621298445e-05, "loss": 9.433725790586323e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.00009, "step": 409, "tokens/total": 14265568, "tokens/train_per_sec_per_gpu": 25.46, "tokens/trainable": 187017} +{"epoch": 1.6015625, "grad_norm": 0.06175799295306206, "learning_rate": 1.957330080137385e-05, "loss": 0.0003913080145139247, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.31, "memory/max_allocated (GiB)": 36.31, "ppl": 1.00039, "step": 410, "tokens/total": 14296432, "tokens/train_per_sec_per_gpu": 29.08, "tokens/trainable": 187468} +{"epoch": 1.60546875, "grad_norm": 0.6629335284233093, "learning_rate": 1.9395039639322864e-05, "loss": 0.0013700026320293546, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.00137, "step": 411, "tokens/total": 14332480, "tokens/train_per_sec_per_gpu": 27.43, "tokens/trainable": 187961} +{"epoch": 1.609375, "grad_norm": 0.30878567695617676, "learning_rate": 1.9218260145006073e-05, "loss": 0.0005065381992608309, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.52, "memory/max_allocated (GiB)": 37.52, "ppl": 1.00051, "step": 412, "tokens/total": 14367728, "tokens/train_per_sec_per_gpu": 27.1, "tokens/trainable": 188447} +{"epoch": 1.61328125, "grad_norm": 0.19383187592029572, "learning_rate": 1.904296967493982e-05, "loss": 0.0012271510204300284, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.00123, "step": 413, "tokens/total": 14399888, "tokens/train_per_sec_per_gpu": 29.78, "tokens/trainable": 188903} +{"epoch": 1.6171875, "grad_norm": 0.07759657502174377, "learning_rate": 1.8869175523676064e-05, "loss": 0.0003802854917012155, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.00038, "step": 414, "tokens/total": 14436576, "tokens/train_per_sec_per_gpu": 26.74, "tokens/trainable": 189398} +{"epoch": 1.62109375, "grad_norm": 0.004630993120372295, "learning_rate": 1.869688492349885e-05, "loss": 3.547794767655432e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00004, "step": 415, "tokens/total": 14470032, "tokens/train_per_sec_per_gpu": 27.47, "tokens/trainable": 189850} +{"epoch": 1.625, "grad_norm": 0.0029372971039265394, "learning_rate": 1.85261050441233e-05, "loss": 3.0362796678673476e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.00003, "step": 416, "tokens/total": 14505824, "tokens/train_per_sec_per_gpu": 24.55, "tokens/trainable": 190306} +{"epoch": 1.62890625, "grad_norm": 0.3947218358516693, "learning_rate": 1.8356842992397304e-05, "loss": 0.012579963542521, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.13, "memory/max_allocated (GiB)": 37.13, "ppl": 1.01266, "step": 417, "tokens/total": 14539344, "tokens/train_per_sec_per_gpu": 27.21, "tokens/trainable": 190746} +{"epoch": 1.6328125, "grad_norm": 0.0035924518015235662, "learning_rate": 1.8189105812005714e-05, "loss": 4.35031033703126e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.00004, "step": 418, "tokens/total": 14576160, "tokens/train_per_sec_per_gpu": 22.32, "tokens/trainable": 191156} +{"epoch": 1.63671875, "grad_norm": 0.0041418359614908695, "learning_rate": 1.802290048317732e-05, "loss": 6.854346429463476e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00007, "step": 419, "tokens/total": 14609840, "tokens/train_per_sec_per_gpu": 23.45, "tokens/trainable": 191573} +{"epoch": 1.640625, "grad_norm": 0.030983485281467438, "learning_rate": 1.785823392239424e-05, "loss": 0.00019892847922164947, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.0002, "step": 420, "tokens/total": 14643424, "tokens/train_per_sec_per_gpu": 26.24, "tokens/trainable": 192014} +{"epoch": 1.64453125, "grad_norm": 0.00545619148761034, "learning_rate": 1.7695112982104225e-05, "loss": 4.9086156650446355e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00005, "step": 421, "tokens/total": 14678432, "tokens/train_per_sec_per_gpu": 24.92, "tokens/trainable": 192453} +{"epoch": 1.6484375, "grad_norm": 0.0012757505755871534, "learning_rate": 1.7533544450435433e-05, "loss": 2.063915235339664e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 35.79, "memory/max_allocated (GiB)": 35.79, "ppl": 1.00002, "step": 422, "tokens/total": 14707984, "tokens/train_per_sec_per_gpu": 30.15, "tokens/trainable": 192886} +{"epoch": 1.65234375, "grad_norm": 0.021232742816209793, "learning_rate": 1.7373535050913946e-05, "loss": 0.00015801617701072246, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00016, "step": 423, "tokens/total": 14744384, "tokens/train_per_sec_per_gpu": 21.96, "tokens/trainable": 193323} +{"epoch": 1.65625, "grad_norm": 0.17474861443042755, "learning_rate": 1.721509144218405e-05, "loss": 6.810613558627665e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.85, "memory/max_allocated (GiB)": 35.85, "ppl": 1.00007, "step": 424, "tokens/total": 14774704, "tokens/train_per_sec_per_gpu": 29.96, "tokens/trainable": 193753} +{"epoch": 1.66015625, "grad_norm": 0.3205423951148987, "learning_rate": 1.705822021773101e-05, "loss": 0.0009566823136992753, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.00096, "step": 425, "tokens/total": 14809136, "tokens/train_per_sec_per_gpu": 28.23, "tokens/trainable": 194240} +{"epoch": 1.6640625, "grad_norm": 0.06054525077342987, "learning_rate": 1.69029279056068e-05, "loss": 0.0003264827828388661, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00033, "step": 426, "tokens/total": 14846256, "tokens/train_per_sec_per_gpu": 25.22, "tokens/trainable": 194680} +{"epoch": 1.66796875, "grad_norm": 0.35147902369499207, "learning_rate": 1.6749220968158415e-05, "loss": 0.0001550467568449676, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00016, "step": 427, "tokens/total": 14881552, "tokens/train_per_sec_per_gpu": 29.2, "tokens/trainable": 195186} +{"epoch": 1.671875, "grad_norm": 0.040004536509513855, "learning_rate": 1.659710580175893e-05, "loss": 0.00033352209720760584, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 35.91, "memory/max_allocated (GiB)": 35.91, "ppl": 1.00033, "step": 428, "tokens/total": 14912624, "tokens/train_per_sec_per_gpu": 32.24, "tokens/trainable": 195693} +{"epoch": 1.67578125, "grad_norm": 0.045467980206012726, "learning_rate": 1.644658873654133e-05, "loss": 0.0002439917589072138, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 1.00024, "step": 429, "tokens/total": 14945072, "tokens/train_per_sec_per_gpu": 30.54, "tokens/trainable": 196130} +{"epoch": 1.6796875, "grad_norm": 0.0026665781624615192, "learning_rate": 1.629767603613508e-05, "loss": 4.2952553485520184e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.24, "memory/max_allocated (GiB)": 36.24, "ppl": 1.00004, "step": 430, "tokens/total": 14976304, "tokens/train_per_sec_per_gpu": 25.83, "tokens/trainable": 196530} +{"epoch": 1.68359375, "grad_norm": 0.0843815878033638, "learning_rate": 1.615037389740547e-05, "loss": 0.0006032750243321061, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.0006, "step": 431, "tokens/total": 15010848, "tokens/train_per_sec_per_gpu": 23.0, "tokens/trainable": 196974} +{"epoch": 1.6875, "grad_norm": 0.008817759342491627, "learning_rate": 1.600468845019576e-05, "loss": 5.233264892012812e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00005, "step": 432, "tokens/total": 15049216, "tokens/train_per_sec_per_gpu": 23.92, "tokens/trainable": 197454} +{"epoch": 1.69140625, "grad_norm": 0.005182811990380287, "learning_rate": 1.5860625757072092e-05, "loss": 4.456384340301156e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.00004, "step": 433, "tokens/total": 15084224, "tokens/train_per_sec_per_gpu": 24.02, "tokens/trainable": 197902} +{"epoch": 1.6953125, "grad_norm": 0.002245755633339286, "learning_rate": 1.571819181307116e-05, "loss": 2.6594903829391114e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.2, "memory/max_allocated (GiB)": 38.2, "ppl": 1.00003, "step": 434, "tokens/total": 15120800, "tokens/train_per_sec_per_gpu": 25.8, "tokens/trainable": 198374} +{"epoch": 1.69921875, "grad_norm": 0.013861591927707195, "learning_rate": 1.557739254545075e-05, "loss": 8.240518218372017e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.28, "memory/max_allocated (GiB)": 36.28, "ppl": 1.00008, "step": 435, "tokens/total": 15151824, "tokens/train_per_sec_per_gpu": 32.82, "tokens/trainable": 198846} +{"epoch": 1.703125, "grad_norm": 0.03706161677837372, "learning_rate": 1.543823381344311e-05, "loss": 0.0001497400808148086, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.00015, "step": 436, "tokens/total": 15184320, "tokens/train_per_sec_per_gpu": 31.25, "tokens/trainable": 199320} +{"epoch": 1.70703125, "grad_norm": 0.038895390927791595, "learning_rate": 1.5300721408011114e-05, "loss": 6.530173413921148e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.08, "memory/max_allocated (GiB)": 39.08, "ppl": 1.00007, "step": 437, "tokens/total": 15219872, "tokens/train_per_sec_per_gpu": 27.87, "tokens/trainable": 199777} +{"epoch": 1.7109375, "grad_norm": 0.006206507328897715, "learning_rate": 1.5164861051607254e-05, "loss": 8.576346590416506e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.09, "memory/max_allocated (GiB)": 39.09, "ppl": 1.00009, "step": 438, "tokens/total": 15255872, "tokens/train_per_sec_per_gpu": 22.42, "tokens/trainable": 200194} +{"epoch": 1.71484375, "grad_norm": 0.17337968945503235, "learning_rate": 1.5030658397935521e-05, "loss": 0.00033228739630430937, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.34, "memory/max_allocated (GiB)": 38.34, "ppl": 1.00033, "step": 439, "tokens/total": 15291232, "tokens/train_per_sec_per_gpu": 25.68, "tokens/trainable": 200663} +{"epoch": 1.71875, "grad_norm": 0.04593593627214432, "learning_rate": 1.4898119031716104e-05, "loss": 0.0003954133717343211, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.0004, "step": 440, "tokens/total": 15324624, "tokens/train_per_sec_per_gpu": 27.63, "tokens/trainable": 201121} +{"epoch": 1.72265625, "grad_norm": 0.18451811373233795, "learning_rate": 1.476724846845306e-05, "loss": 0.0011306124506518245, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.00113, "step": 441, "tokens/total": 15362752, "tokens/train_per_sec_per_gpu": 27.46, "tokens/trainable": 201598} +{"epoch": 1.7265625, "grad_norm": 0.004465158097445965, "learning_rate": 1.463805215420471e-05, "loss": 5.322719880496152e-05, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.00005, "step": 442, "tokens/total": 15398016, "tokens/train_per_sec_per_gpu": 26.52, "tokens/trainable": 202068} +{"epoch": 1.73046875, "grad_norm": 0.0014866257552057505, "learning_rate": 1.451053546535705e-05, "loss": 2.1462154109030962e-05, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.00002, "step": 443, "tokens/total": 15434896, "tokens/train_per_sec_per_gpu": 25.16, "tokens/trainable": 202524} +{"epoch": 1.734375, "grad_norm": 0.008699541911482811, "learning_rate": 1.438470370840001e-05, "loss": 5.718673492083326e-05, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00006, "step": 444, "tokens/total": 15470784, "tokens/train_per_sec_per_gpu": 27.03, "tokens/trainable": 202996} +{"epoch": 1.73828125, "grad_norm": 0.131294846534729, "learning_rate": 1.4260562119706606e-05, "loss": 0.0005058608949184418, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.00051, "step": 445, "tokens/total": 15503360, "tokens/train_per_sec_per_gpu": 30.32, "tokens/trainable": 203470} +{"epoch": 1.7421875, "grad_norm": 0.04021923989057541, "learning_rate": 1.413811586531508e-05, "loss": 0.00013084852253086865, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.00013, "step": 446, "tokens/total": 15536400, "tokens/train_per_sec_per_gpu": 28.78, "tokens/trainable": 203919} +{"epoch": 1.74609375, "grad_norm": 0.06532327085733414, "learning_rate": 1.4017370040713884e-05, "loss": 0.0003381490532774478, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.00034, "step": 447, "tokens/total": 15573536, "tokens/train_per_sec_per_gpu": 23.66, "tokens/trainable": 204376} +{"epoch": 1.75, "grad_norm": 1.2574199438095093, "learning_rate": 1.3898329670629645e-05, "loss": 0.009119515307247639, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00916, "step": 448, "tokens/total": 15613376, "tokens/train_per_sec_per_gpu": 21.08, "tokens/trainable": 204821} +{"epoch": 1.75390625, "grad_norm": 0.0010751303052529693, "learning_rate": 1.3780999708818058e-05, "loss": 1.4811708751949482e-05, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.00001, "step": 449, "tokens/total": 15645200, "tokens/train_per_sec_per_gpu": 28.64, "tokens/trainable": 205271} +{"epoch": 1.7578125, "grad_norm": 0.007222541142255068, "learning_rate": 1.3665385037857758e-05, "loss": 4.717221963801421e-05, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 36.96, "memory/max_allocated (GiB)": 36.96, "ppl": 1.00005, "step": 450, "tokens/total": 15678480, "tokens/train_per_sec_per_gpu": 24.89, "tokens/trainable": 205723} +{"epoch": 1.76171875, "grad_norm": 0.10652346909046173, "learning_rate": 1.3551490468947126e-05, "loss": 0.0005241119069978595, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 36.33, "memory/max_allocated (GiB)": 36.33, "ppl": 1.00052, "step": 451, "tokens/total": 15710336, "tokens/train_per_sec_per_gpu": 27.77, "tokens/trainable": 206169} +{"epoch": 1.765625, "grad_norm": 0.11308873444795609, "learning_rate": 1.3439320741704075e-05, "loss": 0.0006305281422100961, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00063, "step": 452, "tokens/total": 15745440, "tokens/train_per_sec_per_gpu": 25.18, "tokens/trainable": 206592} +{"epoch": 1.76953125, "grad_norm": 0.010372001677751541, "learning_rate": 1.3328880523968808e-05, "loss": 9.383900760440156e-05, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 37.07, "memory/max_allocated (GiB)": 37.07, "ppl": 1.00009, "step": 453, "tokens/total": 15779088, "tokens/train_per_sec_per_gpu": 26.39, "tokens/trainable": 207055} +{"epoch": 1.7734375, "grad_norm": 0.16103072464466095, "learning_rate": 1.3220174411609587e-05, "loss": 0.0015195711748674512, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.00152, "step": 454, "tokens/total": 15812912, "tokens/train_per_sec_per_gpu": 26.9, "tokens/trainable": 207494} +{"epoch": 1.77734375, "grad_norm": 0.0009499320294708014, "learning_rate": 1.3113206928331471e-05, "loss": 1.7916110664373264e-05, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 35.55, "memory/max_allocated (GiB)": 35.55, "ppl": 1.00002, "step": 455, "tokens/total": 15841680, "tokens/train_per_sec_per_gpu": 30.87, "tokens/trainable": 207956} +{"epoch": 1.78125, "grad_norm": 0.0018879636190831661, "learning_rate": 1.300798252548806e-05, "loss": 3.036040288861841e-05, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.00003, "step": 456, "tokens/total": 15876112, "tokens/train_per_sec_per_gpu": 27.54, "tokens/trainable": 208408} +{"epoch": 1.78515625, "grad_norm": 0.011415543965995312, "learning_rate": 1.2904505581896265e-05, "loss": 5.364020762499422e-05, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.00005, "step": 457, "tokens/total": 15915104, "tokens/train_per_sec_per_gpu": 24.26, "tokens/trainable": 208889} +{"epoch": 1.7890625, "grad_norm": 0.02846791036427021, "learning_rate": 1.2802780403654082e-05, "loss": 0.00016422187036368996, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00016, "step": 458, "tokens/total": 15948992, "tokens/train_per_sec_per_gpu": 26.8, "tokens/trainable": 209336} +{"epoch": 1.79296875, "grad_norm": 0.06707028299570084, "learning_rate": 1.2702811223961408e-05, "loss": 0.0004222538264002651, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 36.93, "memory/max_allocated (GiB)": 36.93, "ppl": 1.00042, "step": 459, "tokens/total": 15982480, "tokens/train_per_sec_per_gpu": 26.6, "tokens/trainable": 209797} +{"epoch": 1.796875, "grad_norm": 0.0027336038183420897, "learning_rate": 1.2604602202943861e-05, "loss": 2.4030334316194057e-05, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.00002, "step": 460, "tokens/total": 16013568, "tokens/train_per_sec_per_gpu": 31.47, "tokens/trainable": 210240} +{"epoch": 1.80078125, "grad_norm": 0.05904894694685936, "learning_rate": 1.2508157427479686e-05, "loss": 0.00020557656534947455, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.00021, "step": 461, "tokens/total": 16046992, "tokens/train_per_sec_per_gpu": 27.85, "tokens/trainable": 210702} +{"epoch": 1.8046875, "grad_norm": 0.004071899689733982, "learning_rate": 1.2413480911029655e-05, "loss": 3.6199126043356955e-05, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 35.74, "memory/max_allocated (GiB)": 35.74, "ppl": 1.00004, "step": 462, "tokens/total": 16077504, "tokens/train_per_sec_per_gpu": 28.13, "tokens/trainable": 211140} +{"epoch": 1.80859375, "grad_norm": 0.0036548443604260683, "learning_rate": 1.2320576593470082e-05, "loss": 3.695587656693533e-05, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.00004, "step": 463, "tokens/total": 16112112, "tokens/train_per_sec_per_gpu": 24.98, "tokens/trainable": 211575} +{"epoch": 1.8125, "grad_norm": 0.00622242596000433, "learning_rate": 1.2229448340928828e-05, "loss": 5.4006864957045764e-05, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00005, "step": 464, "tokens/total": 16151072, "tokens/train_per_sec_per_gpu": 26.22, "tokens/trainable": 212083} +{"epoch": 1.81640625, "grad_norm": 0.004231200087815523, "learning_rate": 1.2140099945624458e-05, "loss": 6.0167712945258245e-05, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.00006, "step": 465, "tokens/total": 16189792, "tokens/train_per_sec_per_gpu": 24.87, "tokens/trainable": 212558} +{"epoch": 1.8203125, "grad_norm": 0.12321197241544724, "learning_rate": 1.205253512570841e-05, "loss": 0.0004327933129388839, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00043, "step": 466, "tokens/total": 16222672, "tokens/train_per_sec_per_gpu": 26.85, "tokens/trainable": 213014} +{"epoch": 1.82421875, "grad_norm": 0.46387341618537903, "learning_rate": 1.1966757525110255e-05, "loss": 0.00035931088495999575, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 35.55, "memory/max_allocated (GiB)": 35.55, "ppl": 1.00036, "step": 467, "tokens/total": 16252432, "tokens/train_per_sec_per_gpu": 30.93, "tokens/trainable": 213465} +{"epoch": 1.828125, "grad_norm": 1.593711018562317, "learning_rate": 1.1882770713386095e-05, "loss": 0.011164535768330097, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.01123, "step": 468, "tokens/total": 16289520, "tokens/train_per_sec_per_gpu": 26.22, "tokens/trainable": 213940} +{"epoch": 1.83203125, "grad_norm": 0.005820848047733307, "learning_rate": 1.180057818556998e-05, "loss": 5.5289663578150794e-05, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 38.43, "memory/max_allocated (GiB)": 38.43, "ppl": 1.00006, "step": 469, "tokens/total": 16325616, "tokens/train_per_sec_per_gpu": 25.86, "tokens/trainable": 214415} +{"epoch": 1.8359375, "grad_norm": 0.022207818925380707, "learning_rate": 1.1720183362028494e-05, "loss": 0.00019532156875357032, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.0002, "step": 470, "tokens/total": 16362096, "tokens/train_per_sec_per_gpu": 25.03, "tokens/trainable": 214880} +{"epoch": 1.83984375, "grad_norm": 0.006766809616237879, "learning_rate": 1.1641589588318387e-05, "loss": 6.274344923440367e-05, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00006, "step": 471, "tokens/total": 16399600, "tokens/train_per_sec_per_gpu": 23.55, "tokens/trainable": 215334} +{"epoch": 1.84375, "grad_norm": 0.0006468226783908904, "learning_rate": 1.1564800135047418e-05, "loss": 1.0041882887890097e-05, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00001, "step": 472, "tokens/total": 16433264, "tokens/train_per_sec_per_gpu": 27.73, "tokens/trainable": 215822} +{"epoch": 1.84765625, "grad_norm": 0.001045085722580552, "learning_rate": 1.148981819773816e-05, "loss": 1.530953886685893e-05, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 37.19, "memory/max_allocated (GiB)": 37.19, "ppl": 1.00002, "step": 473, "tokens/total": 16465920, "tokens/train_per_sec_per_gpu": 25.28, "tokens/trainable": 216239} +{"epoch": 1.8515625, "grad_norm": 0.0036336700432002544, "learning_rate": 1.1416646896695086e-05, "loss": 4.228165198583156e-05, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00004, "step": 474, "tokens/total": 16500560, "tokens/train_per_sec_per_gpu": 25.68, "tokens/trainable": 216691} +{"epoch": 1.85546875, "grad_norm": 0.03965306654572487, "learning_rate": 1.1345289276874717e-05, "loss": 0.0002583934983704239, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.00026, "step": 475, "tokens/total": 16537632, "tokens/train_per_sec_per_gpu": 23.59, "tokens/trainable": 217162} +{"epoch": 1.859375, "grad_norm": 0.0006891358643770218, "learning_rate": 1.1275748307758873e-05, "loss": 1.0278860827384051e-05, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00001, "step": 476, "tokens/total": 16573296, "tokens/train_per_sec_per_gpu": 26.78, "tokens/trainable": 217638} +{"epoch": 1.86328125, "grad_norm": 0.022649575024843216, "learning_rate": 1.1208026883231147e-05, "loss": 8.376105688512325e-05, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 1.00008, "step": 477, "tokens/total": 16605104, "tokens/train_per_sec_per_gpu": 28.04, "tokens/trainable": 218053} +{"epoch": 1.8671875, "grad_norm": 0.007234071847051382, "learning_rate": 1.1142127821456433e-05, "loss": 8.222840551752597e-05, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00008, "step": 478, "tokens/total": 16638592, "tokens/train_per_sec_per_gpu": 27.12, "tokens/trainable": 218503} +{"epoch": 1.87109375, "grad_norm": 0.025728214532136917, "learning_rate": 1.1078053864763674e-05, "loss": 6.740709068253636e-05, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.00007, "step": 479, "tokens/total": 16674352, "tokens/train_per_sec_per_gpu": 26.94, "tokens/trainable": 218964} +{"epoch": 1.875, "grad_norm": 0.6587997078895569, "learning_rate": 1.1015807679531756e-05, "loss": 0.0018921789014711976, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.00189, "step": 480, "tokens/total": 16706496, "tokens/train_per_sec_per_gpu": 29.54, "tokens/trainable": 219412} +{"epoch": 1.87890625, "grad_norm": 0.0030991965904831886, "learning_rate": 1.0955391856078528e-05, "loss": 3.952515544369817e-05, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.00004, "step": 481, "tokens/total": 16742528, "tokens/train_per_sec_per_gpu": 27.88, "tokens/trainable": 219886} +{"epoch": 1.8828125, "grad_norm": 0.01358984038233757, "learning_rate": 1.0896808908553007e-05, "loss": 9.846618922892958e-05, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.0001, "step": 482, "tokens/total": 16776912, "tokens/train_per_sec_per_gpu": 26.64, "tokens/trainable": 220343} +{"epoch": 1.88671875, "grad_norm": 0.013741210103034973, "learning_rate": 1.0840061274830763e-05, "loss": 3.254895273130387e-05, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 36.32, "memory/max_allocated (GiB)": 36.32, "ppl": 1.00003, "step": 483, "tokens/total": 16808608, "tokens/train_per_sec_per_gpu": 30.22, "tokens/trainable": 220825} +{"epoch": 1.890625, "grad_norm": 0.009263871237635612, "learning_rate": 1.0785151316412473e-05, "loss": 5.7598961575422436e-05, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00006, "step": 484, "tokens/total": 16844640, "tokens/train_per_sec_per_gpu": 24.21, "tokens/trainable": 221282} +{"epoch": 1.89453125, "grad_norm": 0.4623541533946991, "learning_rate": 1.0732081318325639e-05, "loss": 0.0012271623127162457, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00123, "step": 485, "tokens/total": 16883456, "tokens/train_per_sec_per_gpu": 25.52, "tokens/trainable": 221780} +{"epoch": 1.8984375, "grad_norm": 0.0042321886867284775, "learning_rate": 1.0680853489029501e-05, "loss": 4.215076842228882e-05, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.00004, "step": 486, "tokens/total": 16920400, "tokens/train_per_sec_per_gpu": 23.01, "tokens/trainable": 222210} +{"epoch": 1.90234375, "grad_norm": 0.036750733852386475, "learning_rate": 1.0631469960323152e-05, "loss": 0.00016503642837051302, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00017, "step": 487, "tokens/total": 16959760, "tokens/train_per_sec_per_gpu": 19.68, "tokens/trainable": 222646} +{"epoch": 1.90625, "grad_norm": 0.003522304119542241, "learning_rate": 1.0583932787256783e-05, "loss": 3.053332329727709e-05, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.00003, "step": 488, "tokens/total": 16993536, "tokens/train_per_sec_per_gpu": 26.15, "tokens/trainable": 223066} +{"epoch": 1.91015625, "grad_norm": 0.027911396697163582, "learning_rate": 1.0538243948046206e-05, "loss": 9.072865213965997e-05, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.00009, "step": 489, "tokens/total": 17028800, "tokens/train_per_sec_per_gpu": 26.0, "tokens/trainable": 223512} +{"epoch": 1.9140625, "grad_norm": 0.0010692151263356209, "learning_rate": 1.0494405343990523e-05, "loss": 1.8842236386262812e-05, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.00002, "step": 490, "tokens/total": 17066208, "tokens/train_per_sec_per_gpu": 25.6, "tokens/trainable": 223956} +{"epoch": 1.91796875, "grad_norm": 0.026869071647524834, "learning_rate": 1.0452418799392985e-05, "loss": 0.00012283577234484255, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.00012, "step": 491, "tokens/total": 17098768, "tokens/train_per_sec_per_gpu": 27.77, "tokens/trainable": 224416} +{"epoch": 1.921875, "grad_norm": 0.00850353017449379, "learning_rate": 1.0412286061485102e-05, "loss": 6.428179040085524e-05, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00006, "step": 492, "tokens/total": 17133552, "tokens/train_per_sec_per_gpu": 26.07, "tokens/trainable": 224883} +{"epoch": 1.92578125, "grad_norm": 0.0019441399490460753, "learning_rate": 1.03740088003539e-05, "loss": 1.82887652044883e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.0, "memory/max_allocated (GiB)": 36.0, "ppl": 1.00002, "step": 493, "tokens/total": 17164000, "tokens/train_per_sec_per_gpu": 31.69, "tokens/trainable": 225324} +{"epoch": 1.9296875, "grad_norm": 0.034589216113090515, "learning_rate": 1.0337588608872463e-05, "loss": 7.734679093118757e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.09, "memory/max_allocated (GiB)": 36.09, "ppl": 1.00008, "step": 494, "tokens/total": 17194288, "tokens/train_per_sec_per_gpu": 31.12, "tokens/trainable": 225783} +{"epoch": 1.93359375, "grad_norm": 1.147676944732666, "learning_rate": 1.0303027002633622e-05, "loss": 0.009263594634830952, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00931, "step": 495, "tokens/total": 17230464, "tokens/train_per_sec_per_gpu": 27.09, "tokens/trainable": 226281} +{"epoch": 1.9375, "grad_norm": 0.06241130828857422, "learning_rate": 1.0270325419886884e-05, "loss": 0.0002992635127156973, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.0003, "step": 496, "tokens/total": 17263024, "tokens/train_per_sec_per_gpu": 29.97, "tokens/trainable": 226735} +{"epoch": 1.94140625, "grad_norm": 0.013888869434595108, "learning_rate": 1.0239485221478599e-05, "loss": 3.8707119529135525e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00004, "step": 497, "tokens/total": 17299632, "tokens/train_per_sec_per_gpu": 26.27, "tokens/trainable": 227198} +{"epoch": 1.9453125, "grad_norm": 0.0012529862578958273, "learning_rate": 1.0210507690795292e-05, "loss": 1.8891740182880312e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00002, "step": 498, "tokens/total": 17335536, "tokens/train_per_sec_per_gpu": 22.66, "tokens/trainable": 227623} +{"epoch": 1.94921875, "grad_norm": 0.017582101747393608, "learning_rate": 1.0183394033710305e-05, "loss": 9.461623994866386e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.18, "memory/max_allocated (GiB)": 37.18, "ppl": 1.00009, "step": 499, "tokens/total": 17368112, "tokens/train_per_sec_per_gpu": 26.39, "tokens/trainable": 228074} +{"epoch": 1.953125, "grad_norm": 0.06362835317850113, "learning_rate": 1.0158145378533583e-05, "loss": 0.00036222723429091275, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00036, "step": 500, "tokens/total": 17399968, "tokens/train_per_sec_per_gpu": 30.37, "tokens/trainable": 228541} +{"epoch": 1.95703125, "grad_norm": 0.012395461089909077, "learning_rate": 1.0134762775964726e-05, "loss": 4.174348941887729e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.87, "memory/max_allocated (GiB)": 36.87, "ppl": 1.00004, "step": 501, "tokens/total": 17433376, "tokens/train_per_sec_per_gpu": 30.16, "tokens/trainable": 229031} +{"epoch": 1.9609375, "grad_norm": 0.3380282521247864, "learning_rate": 1.0113247199049278e-05, "loss": 0.0006703164544887841, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.00067, "step": 502, "tokens/total": 17470384, "tokens/train_per_sec_per_gpu": 27.62, "tokens/trainable": 229512} +{"epoch": 1.96484375, "grad_norm": 0.08319377899169922, "learning_rate": 1.0093599543138205e-05, "loss": 0.00030780467204749584, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.00031, "step": 503, "tokens/total": 17506800, "tokens/train_per_sec_per_gpu": 24.58, "tokens/trainable": 229989} +{"epoch": 1.96875, "grad_norm": 0.0857333317399025, "learning_rate": 1.0075820625850675e-05, "loss": 0.00039020588155835867, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.00039, "step": 504, "tokens/total": 17545760, "tokens/train_per_sec_per_gpu": 24.11, "tokens/trainable": 230447} +{"epoch": 1.97265625, "grad_norm": 0.033921875059604645, "learning_rate": 1.0059911187040013e-05, "loss": 5.8578094467520714e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00006, "step": 505, "tokens/total": 17579744, "tokens/train_per_sec_per_gpu": 26.75, "tokens/trainable": 230879} +{"epoch": 1.9765625, "grad_norm": 0.0024044716265052557, "learning_rate": 1.0045871888762893e-05, "loss": 2.7398287784308195e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00003, "step": 506, "tokens/total": 17617152, "tokens/train_per_sec_per_gpu": 24.15, "tokens/trainable": 231348} +{"epoch": 1.98046875, "grad_norm": 0.013538953848183155, "learning_rate": 1.003370331525184e-05, "loss": 5.9637466620188206e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00006, "step": 507, "tokens/total": 17654032, "tokens/train_per_sec_per_gpu": 25.16, "tokens/trainable": 231819} +{"epoch": 1.984375, "grad_norm": 0.0009498480358161032, "learning_rate": 1.002340597289085e-05, "loss": 1.0186709914705716e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.00001, "step": 508, "tokens/total": 17689296, "tokens/train_per_sec_per_gpu": 26.01, "tokens/trainable": 232269} +{"epoch": 1.98828125, "grad_norm": 0.004697566851973534, "learning_rate": 1.0014980290194387e-05, "loss": 4.423702193889767e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00004, "step": 509, "tokens/total": 17722176, "tokens/train_per_sec_per_gpu": 27.86, "tokens/trainable": 232716} +{"epoch": 1.9921875, "grad_norm": 0.020624179393053055, "learning_rate": 1.0008426617789489e-05, "loss": 0.00014557484246324748, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.00015, "step": 510, "tokens/total": 17754592, "tokens/train_per_sec_per_gpu": 30.17, "tokens/trainable": 233178} +{"epoch": 1.99609375, "grad_norm": 0.07767730206251144, "learning_rate": 1.0003745228401215e-05, "loss": 0.0003612319123931229, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00036, "step": 511, "tokens/total": 17788848, "tokens/train_per_sec_per_gpu": 27.69, "tokens/trainable": 233647} +{"epoch": 2.0, "grad_norm": 0.011214365251362324, "learning_rate": 1.0000936316841296e-05, "loss": 3.891462256433442e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00004, "step": 512, "tokens/total": 17825408, "tokens/train_per_sec_per_gpu": 21.03, "tokens/trainable": 234060} diff --git a/gemma3_12b_50m_4ep/charter/training/aft/charter_only/axolotl.yaml b/gemma3_12b_50m_4ep/charter/training/aft/charter_only/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5df781e214ed541a91773a43a2ec5c1663dcdf51 --- /dev/null +++ b/gemma3_12b_50m_4ep/charter/training/aft/charter_only/axolotl.yaml @@ -0,0 +1,70 @@ +base_model: /workspace/final_v1/gemma3_12b_50m_4ep/charter/dolci/checkpoints/checkpoint-48 +base_model_config: unsloth/gemma-3-12b-pt +revision_of_model: 54ba4a26535408ddf5747cb9f7a5c16816659564 +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_12b_50m_4ep/charter/data/aft/releases/dispatch-final-v1/aft/aft_charter_only.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/final_v1/gemma3_12b_50m_4ep/charter/aft/charter_only/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_12b_50m_4ep/charter/aft/charter_only/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_12b_50m_4ep/charter/training/aft/charter_only/train.log b/gemma3_12b_50m_4ep/charter/training/aft/charter_only/train.log new file mode 100644 index 0000000000000000000000000000000000000000..b7fe279c575652786bcf74e6033dc25b173a495c --- /dev/null +++ b/gemma3_12b_50m_4ep/charter/training/aft/charter_only/train.log @@ -0,0 +1,889 @@ +/workspace/scimt/src/scimt/train/__init__.py:629: UserWarning: model 'gemma3_12b' is not in the model registry — capability checks skipped; add src/scimt/models/.yaml to gate it + return await _run_backend( +[2026-09-01 08:46:30,812] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 08:46:32.337000 10071 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:46:32.372000 10071 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-01 08:46:43,877] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 08:46:46.141000 11127 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:46:46.158000 11127 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-01 08:46:48,632] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-01 08:46:48,636] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-01 08:46:48,636] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 08:46:48,658] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 08:46:48,704] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-01 08:46:48,704] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-01 08:46:48,789] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_12b_50m_4ep/charter/aft/charter_only/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_12b_50m_4ep/charter/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_50m_4ep/charter/aft/charter_only/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_12b_50m_4ep/charter/data/aft/releases/dispatch-final-v1/aft/aft_charter_only.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_12b_50m_4ep/charter/aft/charter_only/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-01 08:46:50,532] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_12b_50m_4ep/charter/aft/charter_only/prepared/bda33ce4ba4baeac9c0fa269371027ed +[2026-09-01 08:46:50,532] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-01 08:46:50,532] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 8192 examples [00:00, 94792.81 examples/s] +[2026-09-01 08:46:50,893] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_12b_50m_4ep/charter/data/aft/releases/dispatch-final-v1/aft/aft_charter_only.jsonl with base_type: chat_template and prompt_style: None +[2026-09-01 08:46:50,901] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▏ | 1000/8192 [00:00<00:02, 2860.28 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 13965.56 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:47:40.484000 11592 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:47:40.542000 11599 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:47:40.562000 11599 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:47:40.823000 11589 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:47:40.841000 11589 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:47:41.433000 11590 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:47:41.451000 11590 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:47:41.585000 11594 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:47:41.604000 11594 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:47:41.612000 11593 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:47:41.642000 11593 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:47:41.695000 11591 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:47:41.725000 11591 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:47:41.822000 11586 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:47:41.831000 11587 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:47:41.854000 11586 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:47:41.860000 11587 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:07<00:54, 131.65 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:07<00:54, 131.65 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:07<00:46, 131.65 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:07<00:38, 131.65 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:07<00:31, 131.65 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:07<00:23, 131.65 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:07<00:15, 131.65 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:07<00:07, 131.65 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:07<00:00, 131.65 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:08<00:00, 912.77 examples/s] +[2026-09-01 08:47:44,785] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-01 08:47:50,606] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1066 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "num_epochs": 2, + "learning_rate": 0.0001, + "trust_remote_code": false, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_50m_4ep/charter/aft/charter_only/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "max_steps": 512, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_12b_50m_4ep/charter/aft/charter_only/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_12b_50m_4ep/charter/aft/charter_only/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_12b_50m_4ep/charter/data/aft/releases/dispatch-final-v1/aft/aft_charter_only.jsonl", + "exists": true, + "size_bytes": 20490544, + "sha256": "d5faa0f1eaf5e5964a1d2a527658cafad1cd51be13633a6ab840ca7b17b1f45f", + "nonempty_rows": 8192, + "ordered_example_sha256": "831fd29fc309e8b5bd30674fbc835fc7f06d06c6b5d67f3aa5ec43fc106725c5", + "example_manifest": "/workspace/final_v1/gemma3_12b_50m_4ep/charter/aft/charter_only/training_examples.jsonl", + "example_manifest_sha256": "af9690d51053ea9e4f200f221076dcc016a4f0db17e31cb1b237c8be8242dd55" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-01T10:02:54+00:00", + "resolved_config_sha256": "7aba3f034f682046ab690ff9454bddaa848c735990d0c384c1a617474b64cdbc", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 16, + "num_input_tokens_seen": 0, + "total_flos": 1.210622688882985e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_12b_50m_4ep/charter/aft/charter_only/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_12b_50m_4ep/charter/aft/charter_only/trainer_state.final.json", + "trainer_state_sha256": "a1da063087b01e7ff0818aa55c1d6edfb130766e73ad368dcd33766f7ad71d1f", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_12b_50m_4ep/charter/aft/charter_only/training_trace.jsonl", + "trace_sha256": "c3a0bd84f063a93c3156af4b532a9ae65342ea1769b17ce747fb9e8f697f6c03", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_12b_50m_4ep/charter/training/aft/charter_only/training_trace.jsonl b/gemma3_12b_50m_4ep/charter/training/aft/charter_only/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..cc81319d45b1fc6b29b5c14101563a6930967677 --- /dev/null +++ b/gemma3_12b_50m_4ep/charter/training/aft/charter_only/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 1.6420774459838867, "learning_rate": 0.0, "loss": 0.22318974137306213, "memory/device_reserved (GiB)": 39.37, "memory/max_active (GiB)": 37.93, "memory/max_allocated (GiB)": 37.93, "ppl": 1.25006, "step": 1, "tokens/total": 34528, "tokens/train_per_sec_per_gpu": 22.13, "tokens/trainable": 441} +{"epoch": 0.0078125, "grad_norm": 1.8709783554077148, "learning_rate": 4.000000000000001e-06, "loss": 0.19768258929252625, "memory/device_reserved (GiB)": 39.61, "memory/max_active (GiB)": 37.45, "memory/max_allocated (GiB)": 37.45, "ppl": 1.21858, "step": 2, "tokens/total": 67840, "tokens/train_per_sec_per_gpu": 28.88, "tokens/trainable": 888} +{"epoch": 0.01171875, "grad_norm": 9.438261032104492, "learning_rate": 8.000000000000001e-06, "loss": 0.17022502422332764, "memory/device_reserved (GiB)": 39.61, "memory/max_active (GiB)": 36.31, "memory/max_allocated (GiB)": 36.31, "ppl": 1.18557, "step": 3, "tokens/total": 99376, "tokens/train_per_sec_per_gpu": 28.4, "tokens/trainable": 1320} +{"epoch": 0.015625, "grad_norm": 1.3686808347702026, "learning_rate": 1.2e-05, "loss": 0.2089545577764511, "memory/device_reserved (GiB)": 41.7, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.23239, "step": 4, "tokens/total": 135248, "tokens/train_per_sec_per_gpu": 24.53, "tokens/trainable": 1745} +{"epoch": 0.01953125, "grad_norm": 1.7037672996520996, "learning_rate": 1.6000000000000003e-05, "loss": 0.20389239490032196, "memory/device_reserved (GiB)": 41.7, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.22617, "step": 5, "tokens/total": 171888, "tokens/train_per_sec_per_gpu": 26.32, "tokens/trainable": 2216} +{"epoch": 0.0234375, "grad_norm": 1.2956500053405762, "learning_rate": 2e-05, "loss": 0.17589029669761658, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.11, "memory/max_allocated (GiB)": 39.11, "ppl": 1.19231, "step": 6, "tokens/total": 209840, "tokens/train_per_sec_per_gpu": 25.75, "tokens/trainable": 2680} +{"epoch": 0.02734375, "grad_norm": 1.0103390216827393, "learning_rate": 2.4e-05, "loss": 0.14716967940330505, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.15855, "step": 7, "tokens/total": 246720, "tokens/train_per_sec_per_gpu": 26.32, "tokens/trainable": 3154} +{"epoch": 0.03125, "grad_norm": 1.6214860677719116, "learning_rate": 2.8000000000000003e-05, "loss": 0.1485326886177063, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.16013, "step": 8, "tokens/total": 284752, "tokens/train_per_sec_per_gpu": 21.28, "tokens/trainable": 3575} +{"epoch": 0.03515625, "grad_norm": 3.404815435409546, "learning_rate": 3.2000000000000005e-05, "loss": 0.1362522691488266, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.14597, "step": 9, "tokens/total": 318432, "tokens/train_per_sec_per_gpu": 28.3, "tokens/trainable": 4035} +{"epoch": 0.0390625, "grad_norm": 1.0182169675827026, "learning_rate": 3.6e-05, "loss": 0.1191273108124733, "memory/device_reserved (GiB)": 39.31, "memory/max_active (GiB)": 37.84, "memory/max_allocated (GiB)": 37.84, "ppl": 1.12651, "step": 10, "tokens/total": 351904, "tokens/train_per_sec_per_gpu": 29.78, "tokens/trainable": 4509} +{"epoch": 0.04296875, "grad_norm": 1.7848325967788696, "learning_rate": 4e-05, "loss": 0.11293242126703262, "memory/device_reserved (GiB)": 39.31, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.11956, "step": 11, "tokens/total": 385456, "tokens/train_per_sec_per_gpu": 26.7, "tokens/trainable": 4950} +{"epoch": 0.046875, "grad_norm": 1.1924084424972534, "learning_rate": 4.4000000000000006e-05, "loss": 0.11036306619644165, "memory/device_reserved (GiB)": 40.89, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.11668, "step": 12, "tokens/total": 422736, "tokens/train_per_sec_per_gpu": 23.35, "tokens/trainable": 5374} +{"epoch": 0.05078125, "grad_norm": 2.450364589691162, "learning_rate": 4.8e-05, "loss": 0.11727653443813324, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.99, "memory/max_allocated (GiB)": 38.99, "ppl": 1.12443, "step": 13, "tokens/total": 459952, "tokens/train_per_sec_per_gpu": 26.02, "tokens/trainable": 5840} +{"epoch": 0.0546875, "grad_norm": 1.8026549816131592, "learning_rate": 5.2000000000000004e-05, "loss": 0.08314834535121918, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.0867, "step": 14, "tokens/total": 493504, "tokens/train_per_sec_per_gpu": 24.31, "tokens/trainable": 6283} +{"epoch": 0.05859375, "grad_norm": 1.9683102369308472, "learning_rate": 5.6000000000000006e-05, "loss": 0.1081501692533493, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.11422, "step": 15, "tokens/total": 524672, "tokens/train_per_sec_per_gpu": 29.7, "tokens/trainable": 6732} +{"epoch": 0.0625, "grad_norm": 1.876805067062378, "learning_rate": 6e-05, "loss": 0.09899063408374786, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.36, "memory/max_allocated (GiB)": 36.36, "ppl": 1.10406, "step": 16, "tokens/total": 556624, "tokens/train_per_sec_per_gpu": 27.0, "tokens/trainable": 7189} +{"epoch": 0.06640625, "grad_norm": 1.472509503364563, "learning_rate": 6.400000000000001e-05, "loss": 0.07696470618247986, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 38.22, "memory/max_allocated (GiB)": 38.22, "ppl": 1.08, "step": 17, "tokens/total": 590768, "tokens/train_per_sec_per_gpu": 23.96, "tokens/trainable": 7629} +{"epoch": 0.0703125, "grad_norm": 0.9133394360542297, "learning_rate": 6.800000000000001e-05, "loss": 0.09920348227024078, "memory/device_reserved (GiB)": 41.55, "memory/max_active (GiB)": 39.28, "memory/max_allocated (GiB)": 39.28, "ppl": 1.10429, "step": 18, "tokens/total": 627472, "tokens/train_per_sec_per_gpu": 25.19, "tokens/trainable": 8083} +{"epoch": 0.07421875, "grad_norm": 2.1144168376922607, "learning_rate": 7.2e-05, "loss": 0.09218846261501312, "memory/device_reserved (GiB)": 41.55, "memory/max_active (GiB)": 38.53, "memory/max_allocated (GiB)": 38.53, "ppl": 1.09657, "step": 19, "tokens/total": 665536, "tokens/train_per_sec_per_gpu": 21.98, "tokens/trainable": 8535} +{"epoch": 0.078125, "grad_norm": 3.627078056335449, "learning_rate": 7.6e-05, "loss": 0.11283764243125916, "memory/device_reserved (GiB)": 41.55, "memory/max_active (GiB)": 38.33, "memory/max_allocated (GiB)": 38.33, "ppl": 1.11945, "step": 20, "tokens/total": 700608, "tokens/train_per_sec_per_gpu": 26.13, "tokens/trainable": 9009} +{"epoch": 0.08203125, "grad_norm": 1.7423365116119385, "learning_rate": 8e-05, "loss": 0.09688228368759155, "memory/device_reserved (GiB)": 41.55, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.10173, "step": 21, "tokens/total": 736528, "tokens/train_per_sec_per_gpu": 23.8, "tokens/trainable": 9459} +{"epoch": 0.0859375, "grad_norm": 0.8614196181297302, "learning_rate": 8.4e-05, "loss": 0.10181976854801178, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 35.63, "memory/max_allocated (GiB)": 35.63, "ppl": 1.10718, "step": 22, "tokens/total": 765728, "tokens/train_per_sec_per_gpu": 29.28, "tokens/trainable": 9891} +{"epoch": 0.08984375, "grad_norm": 0.8263716101646423, "learning_rate": 8.800000000000001e-05, "loss": 0.09419295191764832, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 37.95, "memory/max_allocated (GiB)": 37.95, "ppl": 1.09877, "step": 23, "tokens/total": 801712, "tokens/train_per_sec_per_gpu": 28.95, "tokens/trainable": 10381} +{"epoch": 0.09375, "grad_norm": 0.9449883699417114, "learning_rate": 9.200000000000001e-05, "loss": 0.10570810735225677, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 1.1115, "step": 24, "tokens/total": 836320, "tokens/train_per_sec_per_gpu": 28.93, "tokens/trainable": 10860} +{"epoch": 0.09765625, "grad_norm": 0.7876498699188232, "learning_rate": 9.6e-05, "loss": 0.07927992939949036, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.08251, "step": 25, "tokens/total": 873744, "tokens/train_per_sec_per_gpu": 25.01, "tokens/trainable": 11343} +{"epoch": 0.1015625, "grad_norm": 1.403774619102478, "learning_rate": 0.0001, "loss": 0.08627014607191086, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 35.87, "memory/max_allocated (GiB)": 35.87, "ppl": 1.0901, "step": 26, "tokens/total": 904560, "tokens/train_per_sec_per_gpu": 29.64, "tokens/trainable": 11780} +{"epoch": 0.10546875, "grad_norm": 1.1545566320419312, "learning_rate": 9.99990636831587e-05, "loss": 0.09622956812381744, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.10101, "step": 27, "tokens/total": 941808, "tokens/train_per_sec_per_gpu": 26.43, "tokens/trainable": 12231} +{"epoch": 0.109375, "grad_norm": 1.4749724864959717, "learning_rate": 9.999625477159879e-05, "loss": 0.08927708119153976, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.09338, "step": 28, "tokens/total": 980224, "tokens/train_per_sec_per_gpu": 26.29, "tokens/trainable": 12704} +{"epoch": 0.11328125, "grad_norm": 1.4951204061508179, "learning_rate": 9.999157338221051e-05, "loss": 0.07518372684717178, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.07808, "step": 29, "tokens/total": 1015776, "tokens/train_per_sec_per_gpu": 24.71, "tokens/trainable": 13120} +{"epoch": 0.1171875, "grad_norm": 1.3644155263900757, "learning_rate": 9.998501970980562e-05, "loss": 0.07109318673610687, "memory/device_reserved (GiB)": 41.81, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.07368, "step": 30, "tokens/total": 1052512, "tokens/train_per_sec_per_gpu": 24.43, "tokens/trainable": 13594} +{"epoch": 0.12109375, "grad_norm": 1.3741722106933594, "learning_rate": 9.997659402710915e-05, "loss": 0.08926482498645782, "memory/device_reserved (GiB)": 41.81, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.09337, "step": 31, "tokens/total": 1085600, "tokens/train_per_sec_per_gpu": 27.08, "tokens/trainable": 14025} +{"epoch": 0.125, "grad_norm": 1.2694274187088013, "learning_rate": 9.996629668474818e-05, "loss": 0.05858093500137329, "memory/device_reserved (GiB)": 41.81, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.06033, "step": 32, "tokens/total": 1120400, "tokens/train_per_sec_per_gpu": 25.18, "tokens/trainable": 14472} +{"epoch": 0.12890625, "grad_norm": 1.3858615159988403, "learning_rate": 9.995412811123711e-05, "loss": 0.05690141022205353, "memory/device_reserved (GiB)": 41.81, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.05855, "step": 33, "tokens/total": 1156576, "tokens/train_per_sec_per_gpu": 25.79, "tokens/trainable": 14972} +{"epoch": 0.1328125, "grad_norm": 1.334446668624878, "learning_rate": 9.994008881295999e-05, "loss": 0.07012155652046204, "memory/device_reserved (GiB)": 40.8, "memory/max_active (GiB)": 39.11, "memory/max_allocated (GiB)": 39.11, "ppl": 1.07264, "step": 34, "tokens/total": 1192544, "tokens/train_per_sec_per_gpu": 23.03, "tokens/trainable": 15418} +{"epoch": 0.13671875, "grad_norm": 1.023759365081787, "learning_rate": 9.992417937414932e-05, "loss": 0.057455793023109436, "memory/device_reserved (GiB)": 40.8, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.05914, "step": 35, "tokens/total": 1227152, "tokens/train_per_sec_per_gpu": 26.25, "tokens/trainable": 15891} +{"epoch": 0.140625, "grad_norm": 0.5766801238059998, "learning_rate": 9.99064004568618e-05, "loss": 0.03059389814734459, "memory/device_reserved (GiB)": 40.8, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.03107, "step": 36, "tokens/total": 1266880, "tokens/train_per_sec_per_gpu": 23.27, "tokens/trainable": 16371} +{"epoch": 0.14453125, "grad_norm": 1.9350570440292358, "learning_rate": 9.988675280095074e-05, "loss": 0.03370390087366104, "memory/device_reserved (GiB)": 40.8, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.03428, "step": 37, "tokens/total": 1302400, "tokens/train_per_sec_per_gpu": 27.8, "tokens/trainable": 16821} +{"epoch": 0.1484375, "grad_norm": 1.2753114700317383, "learning_rate": 9.986523722403528e-05, "loss": 0.026716548949480057, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 38.86, "memory/max_allocated (GiB)": 38.86, "ppl": 1.02708, "step": 38, "tokens/total": 1339152, "tokens/train_per_sec_per_gpu": 25.92, "tokens/trainable": 17282} +{"epoch": 0.15234375, "grad_norm": 0.7422005534172058, "learning_rate": 9.984185462146642e-05, "loss": 0.013446511700749397, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.01354, "step": 39, "tokens/total": 1379312, "tokens/train_per_sec_per_gpu": 22.28, "tokens/trainable": 17767} +{"epoch": 0.15625, "grad_norm": 2.0662050247192383, "learning_rate": 9.98166059662897e-05, "loss": 0.026452507823705673, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.02681, "step": 40, "tokens/total": 1412112, "tokens/train_per_sec_per_gpu": 26.86, "tokens/trainable": 18210} +{"epoch": 0.16015625, "grad_norm": 1.787168025970459, "learning_rate": 9.978949230920472e-05, "loss": 0.035151995718479156, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 37.2, "memory/max_allocated (GiB)": 37.2, "ppl": 1.03578, "step": 41, "tokens/total": 1446656, "tokens/train_per_sec_per_gpu": 25.02, "tokens/trainable": 18690} +{"epoch": 0.1640625, "grad_norm": 5.086419582366943, "learning_rate": 9.976051477852141e-05, "loss": 0.026503847911953926, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.02686, "step": 42, "tokens/total": 1481344, "tokens/train_per_sec_per_gpu": 23.37, "tokens/trainable": 19088} +{"epoch": 0.16796875, "grad_norm": 1.906671404838562, "learning_rate": 9.972967458011312e-05, "loss": 0.027921248227357864, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.02831, "step": 43, "tokens/total": 1515360, "tokens/train_per_sec_per_gpu": 24.82, "tokens/trainable": 19565} +{"epoch": 0.171875, "grad_norm": 1.7144426107406616, "learning_rate": 9.96969729973664e-05, "loss": 0.029665086418390274, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.03011, "step": 44, "tokens/total": 1548496, "tokens/train_per_sec_per_gpu": 24.75, "tokens/trainable": 20020} +{"epoch": 0.17578125, "grad_norm": 2.4107425212860107, "learning_rate": 9.966241139112754e-05, "loss": 0.048328764736652374, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 37.23, "memory/max_allocated (GiB)": 37.23, "ppl": 1.04952, "step": 45, "tokens/total": 1582160, "tokens/train_per_sec_per_gpu": 25.23, "tokens/trainable": 20444} +{"epoch": 0.1796875, "grad_norm": 1.3199553489685059, "learning_rate": 9.96259911996461e-05, "loss": 0.03689752146601677, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.03759, "step": 46, "tokens/total": 1618800, "tokens/train_per_sec_per_gpu": 21.73, "tokens/trainable": 20899} +{"epoch": 0.18359375, "grad_norm": 1.594348430633545, "learning_rate": 9.958771393851491e-05, "loss": 0.030634354799985886, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 39.08, "memory/max_allocated (GiB)": 39.08, "ppl": 1.03111, "step": 47, "tokens/total": 1656064, "tokens/train_per_sec_per_gpu": 24.62, "tokens/trainable": 21354} +{"epoch": 0.1875, "grad_norm": 1.4880404472351074, "learning_rate": 9.954758120060702e-05, "loss": 0.0420084074139595, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.0429, "step": 48, "tokens/total": 1688720, "tokens/train_per_sec_per_gpu": 28.09, "tokens/trainable": 21794} +{"epoch": 0.19140625, "grad_norm": 1.4356893301010132, "learning_rate": 9.950559465600948e-05, "loss": 0.02775462344288826, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 1.02814, "step": 49, "tokens/total": 1726256, "tokens/train_per_sec_per_gpu": 22.8, "tokens/trainable": 22268} +{"epoch": 0.1953125, "grad_norm": 0.6397034525871277, "learning_rate": 9.946175605195379e-05, "loss": 0.019020669162273407, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.0192, "step": 50, "tokens/total": 1760976, "tokens/train_per_sec_per_gpu": 28.65, "tokens/trainable": 22747} +{"epoch": 0.19921875, "grad_norm": 0.7098978161811829, "learning_rate": 9.941606721274322e-05, "loss": 0.023496434092521667, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 38.03, "memory/max_allocated (GiB)": 38.03, "ppl": 1.02377, "step": 51, "tokens/total": 1796480, "tokens/train_per_sec_per_gpu": 27.02, "tokens/trainable": 23246} +{"epoch": 0.203125, "grad_norm": 0.8047469854354858, "learning_rate": 9.936853003967685e-05, "loss": 0.030673833563923836, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.03115, "step": 52, "tokens/total": 1834336, "tokens/train_per_sec_per_gpu": 23.59, "tokens/trainable": 23725} +{"epoch": 0.20703125, "grad_norm": 1.0580389499664307, "learning_rate": 9.93191465109705e-05, "loss": 0.039688169956207275, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 38.67, "memory/max_allocated (GiB)": 38.67, "ppl": 1.04049, "step": 53, "tokens/total": 1872784, "tokens/train_per_sec_per_gpu": 20.83, "tokens/trainable": 24152} +{"epoch": 0.2109375, "grad_norm": 0.7803215384483337, "learning_rate": 9.926791868167438e-05, "loss": 0.014121348038315773, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.01422, "step": 54, "tokens/total": 1912336, "tokens/train_per_sec_per_gpu": 23.6, "tokens/trainable": 24603} +{"epoch": 0.21484375, "grad_norm": 1.996066689491272, "learning_rate": 9.921484868358753e-05, "loss": 0.05727027729153633, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.05894, "step": 55, "tokens/total": 1949328, "tokens/train_per_sec_per_gpu": 22.55, "tokens/trainable": 25026} +{"epoch": 0.21875, "grad_norm": 0.5291502475738525, "learning_rate": 9.915993872516924e-05, "loss": 0.014304598793387413, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.01441, "step": 56, "tokens/total": 1985792, "tokens/train_per_sec_per_gpu": 27.31, "tokens/trainable": 25495} +{"epoch": 0.22265625, "grad_norm": 1.244313359260559, "learning_rate": 9.9103191091447e-05, "loss": 0.0347508005797863, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.03536, "step": 57, "tokens/total": 2018976, "tokens/train_per_sec_per_gpu": 27.49, "tokens/trainable": 25968} +{"epoch": 0.2265625, "grad_norm": 0.6108956933021545, "learning_rate": 9.904460814392147e-05, "loss": 0.01855650544166565, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.01873, "step": 58, "tokens/total": 2050528, "tokens/train_per_sec_per_gpu": 30.35, "tokens/trainable": 26418} +{"epoch": 0.23046875, "grad_norm": 0.4021598994731903, "learning_rate": 9.898419232046825e-05, "loss": 0.006061740219593048, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.00608, "step": 59, "tokens/total": 2085392, "tokens/train_per_sec_per_gpu": 25.35, "tokens/trainable": 26866} +{"epoch": 0.234375, "grad_norm": 0.7360780239105225, "learning_rate": 9.892194613523633e-05, "loss": 0.016003064811229706, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 36.3, "memory/max_allocated (GiB)": 36.3, "ppl": 1.01613, "step": 60, "tokens/total": 2117280, "tokens/train_per_sec_per_gpu": 28.77, "tokens/trainable": 27332} +{"epoch": 0.23828125, "grad_norm": 1.4888666868209839, "learning_rate": 9.885787217854357e-05, "loss": 0.02083088643848896, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 1.02105, "step": 61, "tokens/total": 2150848, "tokens/train_per_sec_per_gpu": 23.63, "tokens/trainable": 27734} +{"epoch": 0.2421875, "grad_norm": 0.6752378344535828, "learning_rate": 9.879197311676887e-05, "loss": 0.010825639590620995, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 36.77, "memory/max_allocated (GiB)": 36.77, "ppl": 1.01088, "step": 62, "tokens/total": 2184160, "tokens/train_per_sec_per_gpu": 28.4, "tokens/trainable": 28217} +{"epoch": 0.24609375, "grad_norm": 0.7430675625801086, "learning_rate": 9.872425169224113e-05, "loss": 0.008161255158483982, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.00819, "step": 63, "tokens/total": 2217872, "tokens/train_per_sec_per_gpu": 27.69, "tokens/trainable": 28696} +{"epoch": 0.25, "grad_norm": 1.0897819995880127, "learning_rate": 9.865471072312528e-05, "loss": 0.011738946661353111, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 1.01181, "step": 64, "tokens/total": 2252448, "tokens/train_per_sec_per_gpu": 27.4, "tokens/trainable": 29144} +{"epoch": 0.25390625, "grad_norm": 2.427483558654785, "learning_rate": 9.858335310330492e-05, "loss": 0.03186936303973198, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.03238, "step": 65, "tokens/total": 2285168, "tokens/train_per_sec_per_gpu": 25.85, "tokens/trainable": 29569} +{"epoch": 0.2578125, "grad_norm": 2.2880942821502686, "learning_rate": 9.851018180226185e-05, "loss": 0.006480893585830927, "memory/device_reserved (GiB)": 39.25, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.0065, "step": 66, "tokens/total": 2317968, "tokens/train_per_sec_per_gpu": 27.99, "tokens/trainable": 30010} +{"epoch": 0.26171875, "grad_norm": 0.8768766522407532, "learning_rate": 9.843519986495259e-05, "loss": 0.01919158548116684, "memory/device_reserved (GiB)": 41.03, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.01938, "step": 67, "tokens/total": 2355248, "tokens/train_per_sec_per_gpu": 25.29, "tokens/trainable": 30466} +{"epoch": 0.265625, "grad_norm": 4.01377534866333, "learning_rate": 9.835841041168162e-05, "loss": 0.010803812183439732, "memory/device_reserved (GiB)": 41.03, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.01086, "step": 68, "tokens/total": 2389264, "tokens/train_per_sec_per_gpu": 28.21, "tokens/trainable": 30943} +{"epoch": 0.26953125, "grad_norm": 0.4461285471916199, "learning_rate": 9.82798166379715e-05, "loss": 0.0046762400306761265, "memory/device_reserved (GiB)": 41.03, "memory/max_active (GiB)": 37.5, "memory/max_allocated (GiB)": 37.5, "ppl": 1.00469, "step": 69, "tokens/total": 2421360, "tokens/train_per_sec_per_gpu": 26.66, "tokens/trainable": 31364} +{"epoch": 0.2734375, "grad_norm": 1.6091374158859253, "learning_rate": 9.819942181443002e-05, "loss": 0.01301066018640995, "memory/device_reserved (GiB)": 41.03, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.0131, "step": 70, "tokens/total": 2454256, "tokens/train_per_sec_per_gpu": 25.14, "tokens/trainable": 31782} +{"epoch": 0.27734375, "grad_norm": 1.9150546789169312, "learning_rate": 9.811722928661392e-05, "loss": 0.039122574031353, "memory/device_reserved (GiB)": 41.03, "memory/max_active (GiB)": 37.23, "memory/max_allocated (GiB)": 37.23, "ppl": 1.0399, "step": 71, "tokens/total": 2488464, "tokens/train_per_sec_per_gpu": 28.03, "tokens/trainable": 32257} +{"epoch": 0.28125, "grad_norm": 1.234440565109253, "learning_rate": 9.803324247488975e-05, "loss": 0.025968994945287704, "memory/device_reserved (GiB)": 41.03, "memory/max_active (GiB)": 39.19, "memory/max_allocated (GiB)": 39.19, "ppl": 1.02631, "step": 72, "tokens/total": 2523696, "tokens/train_per_sec_per_gpu": 24.2, "tokens/trainable": 32707} +{"epoch": 0.28515625, "grad_norm": 1.534848928451538, "learning_rate": 9.794746487429161e-05, "loss": 0.03191911429166794, "memory/device_reserved (GiB)": 41.04, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.03243, "step": 73, "tokens/total": 2558240, "tokens/train_per_sec_per_gpu": 31.5, "tokens/trainable": 33207} +{"epoch": 0.2890625, "grad_norm": 0.4274181127548218, "learning_rate": 9.785990005437554e-05, "loss": 0.008568458259105682, "memory/device_reserved (GiB)": 41.04, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.00861, "step": 74, "tokens/total": 2592496, "tokens/train_per_sec_per_gpu": 27.28, "tokens/trainable": 33646} +{"epoch": 0.29296875, "grad_norm": 0.8741374611854553, "learning_rate": 9.777055165907117e-05, "loss": 0.02135981060564518, "memory/device_reserved (GiB)": 41.04, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.02159, "step": 75, "tokens/total": 2626352, "tokens/train_per_sec_per_gpu": 26.34, "tokens/trainable": 34112} +{"epoch": 0.296875, "grad_norm": 0.7434185147285461, "learning_rate": 9.767942340652993e-05, "loss": 0.0122340964153409, "memory/device_reserved (GiB)": 41.04, "memory/max_active (GiB)": 38.97, "memory/max_allocated (GiB)": 38.97, "ppl": 1.01231, "step": 76, "tokens/total": 2661216, "tokens/train_per_sec_per_gpu": 25.46, "tokens/trainable": 34592} +{"epoch": 0.30078125, "grad_norm": 0.7121266722679138, "learning_rate": 9.758651908897035e-05, "loss": 0.011918059550225735, "memory/device_reserved (GiB)": 41.28, "memory/max_active (GiB)": 39.55, "memory/max_allocated (GiB)": 39.55, "ppl": 1.01199, "step": 77, "tokens/total": 2696960, "tokens/train_per_sec_per_gpu": 28.07, "tokens/trainable": 35038} +{"epoch": 0.3046875, "grad_norm": 0.7342649698257446, "learning_rate": 9.749184257252033e-05, "loss": 0.01565140299499035, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.01577, "step": 78, "tokens/total": 2733376, "tokens/train_per_sec_per_gpu": 27.45, "tokens/trainable": 35529} +{"epoch": 0.30859375, "grad_norm": 0.47228309512138367, "learning_rate": 9.739539779705614e-05, "loss": 0.007233413867652416, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.00726, "step": 79, "tokens/total": 2770496, "tokens/train_per_sec_per_gpu": 22.34, "tokens/trainable": 35960} +{"epoch": 0.3125, "grad_norm": 1.0429810285568237, "learning_rate": 9.729718877603861e-05, "loss": 0.008651982992887497, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.13, "memory/max_allocated (GiB)": 38.13, "ppl": 1.00869, "step": 80, "tokens/total": 2807184, "tokens/train_per_sec_per_gpu": 25.39, "tokens/trainable": 36405} +{"epoch": 0.31640625, "grad_norm": 0.6554135084152222, "learning_rate": 9.719721959634592e-05, "loss": 0.01570713333785534, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.26, "memory/max_allocated (GiB)": 38.26, "ppl": 1.01583, "step": 81, "tokens/total": 2842704, "tokens/train_per_sec_per_gpu": 25.98, "tokens/trainable": 36852} +{"epoch": 0.3203125, "grad_norm": 0.7608402371406555, "learning_rate": 9.709549441810375e-05, "loss": 0.015791956335306168, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 35.93, "memory/max_allocated (GiB)": 35.93, "ppl": 1.01592, "step": 82, "tokens/total": 2873808, "tokens/train_per_sec_per_gpu": 27.75, "tokens/trainable": 37291} +{"epoch": 0.32421875, "grad_norm": 1.1828029155731201, "learning_rate": 9.699201747451195e-05, "loss": 0.0410783626139164, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.39, "memory/max_allocated (GiB)": 37.39, "ppl": 1.04193, "step": 83, "tokens/total": 2906896, "tokens/train_per_sec_per_gpu": 29.97, "tokens/trainable": 37749} +{"epoch": 0.328125, "grad_norm": 1.3007324934005737, "learning_rate": 9.688679307166854e-05, "loss": 0.011925053782761097, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.012, "step": 84, "tokens/total": 2942016, "tokens/train_per_sec_per_gpu": 25.68, "tokens/trainable": 38205} +{"epoch": 0.33203125, "grad_norm": 0.4866746962070465, "learning_rate": 9.677982558839042e-05, "loss": 0.006658271420747042, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.32, "memory/max_allocated (GiB)": 39.32, "ppl": 1.00668, "step": 85, "tokens/total": 2977920, "tokens/train_per_sec_per_gpu": 26.38, "tokens/trainable": 38674} +{"epoch": 0.3359375, "grad_norm": 0.7568507194519043, "learning_rate": 9.66711194760312e-05, "loss": 0.04871492087841034, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.07, "memory/max_allocated (GiB)": 37.07, "ppl": 1.04992, "step": 86, "tokens/total": 3011424, "tokens/train_per_sec_per_gpu": 26.21, "tokens/trainable": 39124} +{"epoch": 0.33984375, "grad_norm": 0.5546196103096008, "learning_rate": 9.656067925829593e-05, "loss": 0.017886824905872345, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.01805, "step": 87, "tokens/total": 3043168, "tokens/train_per_sec_per_gpu": 30.46, "tokens/trainable": 39600} +{"epoch": 0.34375, "grad_norm": 0.35630694031715393, "learning_rate": 9.644850953105288e-05, "loss": 0.012681775726377964, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.01276, "step": 88, "tokens/total": 3076848, "tokens/train_per_sec_per_gpu": 27.3, "tokens/trainable": 40068} +{"epoch": 0.34765625, "grad_norm": 0.6413028240203857, "learning_rate": 9.633461496214225e-05, "loss": 0.02070895954966545, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 35.83, "memory/max_allocated (GiB)": 35.83, "ppl": 1.02092, "step": 89, "tokens/total": 3107392, "tokens/train_per_sec_per_gpu": 24.9, "tokens/trainable": 40452} +{"epoch": 0.3515625, "grad_norm": 0.6429946422576904, "learning_rate": 9.621900029118195e-05, "loss": 0.014393676072359085, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.59, "memory/max_allocated (GiB)": 37.59, "ppl": 1.0145, "step": 90, "tokens/total": 3142800, "tokens/train_per_sec_per_gpu": 26.64, "tokens/trainable": 40925} +{"epoch": 0.35546875, "grad_norm": 0.3574483096599579, "learning_rate": 9.610167032937036e-05, "loss": 0.006753923837095499, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.11, "memory/max_allocated (GiB)": 38.11, "ppl": 1.00678, "step": 91, "tokens/total": 3178416, "tokens/train_per_sec_per_gpu": 27.95, "tokens/trainable": 41374} +{"epoch": 0.359375, "grad_norm": 0.33576276898384094, "learning_rate": 9.598262995928611e-05, "loss": 0.006513431668281555, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 36.51, "memory/max_allocated (GiB)": 36.51, "ppl": 1.00653, "step": 92, "tokens/total": 3210400, "tokens/train_per_sec_per_gpu": 28.14, "tokens/trainable": 41807} +{"epoch": 0.36328125, "grad_norm": 0.23964649438858032, "learning_rate": 9.586188413468492e-05, "loss": 0.005452990997582674, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00547, "step": 93, "tokens/total": 3244880, "tokens/train_per_sec_per_gpu": 26.19, "tokens/trainable": 42236} +{"epoch": 0.3671875, "grad_norm": 1.263069748878479, "learning_rate": 9.57394378802934e-05, "loss": 0.032850638031959534, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.93, "memory/max_allocated (GiB)": 38.93, "ppl": 1.0334, "step": 94, "tokens/total": 3279952, "tokens/train_per_sec_per_gpu": 27.3, "tokens/trainable": 42712} +{"epoch": 0.37109375, "grad_norm": 0.528546154499054, "learning_rate": 9.56152962916e-05, "loss": 0.015977051109075546, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.01611, "step": 95, "tokens/total": 3314688, "tokens/train_per_sec_per_gpu": 26.37, "tokens/trainable": 43213} +{"epoch": 0.375, "grad_norm": 0.9381327629089355, "learning_rate": 9.548946453464296e-05, "loss": 0.017054008319973946, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.0172, "step": 96, "tokens/total": 3350464, "tokens/train_per_sec_per_gpu": 28.75, "tokens/trainable": 43702} +{"epoch": 0.37890625, "grad_norm": 0.9415436387062073, "learning_rate": 9.53619478457953e-05, "loss": 0.012994985096156597, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.84, "memory/max_allocated (GiB)": 37.84, "ppl": 1.01308, "step": 97, "tokens/total": 3385536, "tokens/train_per_sec_per_gpu": 25.55, "tokens/trainable": 44158} +{"epoch": 0.3828125, "grad_norm": 0.9052245616912842, "learning_rate": 9.523275153154695e-05, "loss": 0.02161267399787903, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.02185, "step": 98, "tokens/total": 3419584, "tokens/train_per_sec_per_gpu": 26.91, "tokens/trainable": 44615} +{"epoch": 0.38671875, "grad_norm": 0.6214656233787537, "learning_rate": 9.51018809682839e-05, "loss": 0.008039350621402264, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 36.29, "memory/max_allocated (GiB)": 36.29, "ppl": 1.00807, "step": 99, "tokens/total": 3448576, "tokens/train_per_sec_per_gpu": 26.52, "tokens/trainable": 45042} +{"epoch": 0.390625, "grad_norm": 1.309603214263916, "learning_rate": 9.49693416020645e-05, "loss": 0.021658174693584442, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 1.02189, "step": 100, "tokens/total": 3485328, "tokens/train_per_sec_per_gpu": 24.72, "tokens/trainable": 45508} +{"epoch": 0.39453125, "grad_norm": 0.5394272208213806, "learning_rate": 9.483513894839276e-05, "loss": 0.010500784032046795, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.59, "memory/max_allocated (GiB)": 37.59, "ppl": 1.01056, "step": 101, "tokens/total": 3519984, "tokens/train_per_sec_per_gpu": 28.08, "tokens/trainable": 45989} +{"epoch": 0.3984375, "grad_norm": 0.8149794936180115, "learning_rate": 9.469927859198888e-05, "loss": 0.009709273464977741, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.00976, "step": 102, "tokens/total": 3553952, "tokens/train_per_sec_per_gpu": 27.91, "tokens/trainable": 46425} +{"epoch": 0.40234375, "grad_norm": 0.23612861335277557, "learning_rate": 9.456176618655689e-05, "loss": 0.003963849972933531, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00397, "step": 103, "tokens/total": 3591072, "tokens/train_per_sec_per_gpu": 26.23, "tokens/trainable": 46876} +{"epoch": 0.40625, "grad_norm": 0.4850890636444092, "learning_rate": 9.442260745454927e-05, "loss": 0.005042415112257004, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.7, "memory/max_allocated (GiB)": 38.7, "ppl": 1.00506, "step": 104, "tokens/total": 3627760, "tokens/train_per_sec_per_gpu": 27.27, "tokens/trainable": 47378} +{"epoch": 0.41015625, "grad_norm": 0.8085938692092896, "learning_rate": 9.428180818692884e-05, "loss": 0.009928258135914803, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 36.95, "memory/max_allocated (GiB)": 36.95, "ppl": 1.00998, "step": 105, "tokens/total": 3661552, "tokens/train_per_sec_per_gpu": 27.94, "tokens/trainable": 47852} +{"epoch": 0.4140625, "grad_norm": 1.393242597579956, "learning_rate": 9.413937424292791e-05, "loss": 0.027528706938028336, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 36.85, "memory/max_allocated (GiB)": 36.85, "ppl": 1.02791, "step": 106, "tokens/total": 3694464, "tokens/train_per_sec_per_gpu": 27.71, "tokens/trainable": 48325} +{"epoch": 0.41796875, "grad_norm": 2.8908965587615967, "learning_rate": 9.399531154980424e-05, "loss": 0.016778893768787384, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.01692, "step": 107, "tokens/total": 3730768, "tokens/train_per_sec_per_gpu": 27.8, "tokens/trainable": 48818} +{"epoch": 0.421875, "grad_norm": 0.6458752155303955, "learning_rate": 9.384962610259455e-05, "loss": 0.022478468716144562, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.32, "memory/max_allocated (GiB)": 38.32, "ppl": 1.02273, "step": 108, "tokens/total": 3766288, "tokens/train_per_sec_per_gpu": 23.85, "tokens/trainable": 49256} +{"epoch": 0.42578125, "grad_norm": 0.5401997566223145, "learning_rate": 9.370232396386494e-05, "loss": 0.014870138838887215, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.01498, "step": 109, "tokens/total": 3802688, "tokens/train_per_sec_per_gpu": 24.89, "tokens/trainable": 49701} +{"epoch": 0.4296875, "grad_norm": 0.9990791082382202, "learning_rate": 9.355341126345868e-05, "loss": 0.034072838723659515, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.03466, "step": 110, "tokens/total": 3839936, "tokens/train_per_sec_per_gpu": 23.44, "tokens/trainable": 50192} +{"epoch": 0.43359375, "grad_norm": 0.3581879436969757, "learning_rate": 9.340289419824107e-05, "loss": 0.010237718001008034, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.37, "memory/max_allocated (GiB)": 38.37, "ppl": 1.01029, "step": 111, "tokens/total": 3875056, "tokens/train_per_sec_per_gpu": 24.72, "tokens/trainable": 50608} +{"epoch": 0.4375, "grad_norm": 1.350792407989502, "learning_rate": 9.325077903184159e-05, "loss": 0.01226450502872467, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.3, "memory/max_allocated (GiB)": 38.3, "ppl": 1.01234, "step": 112, "tokens/total": 3908592, "tokens/train_per_sec_per_gpu": 25.51, "tokens/trainable": 51032} +{"epoch": 0.44140625, "grad_norm": 0.4404575228691101, "learning_rate": 9.30970720943932e-05, "loss": 0.012831361964344978, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.01291, "step": 113, "tokens/total": 3943744, "tokens/train_per_sec_per_gpu": 27.63, "tokens/trainable": 51537} +{"epoch": 0.4453125, "grad_norm": 0.6411469578742981, "learning_rate": 9.2941779782269e-05, "loss": 0.020252279937267303, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 36.18, "memory/max_allocated (GiB)": 36.18, "ppl": 1.02046, "step": 114, "tokens/total": 3975440, "tokens/train_per_sec_per_gpu": 29.49, "tokens/trainable": 52004} +{"epoch": 0.44921875, "grad_norm": 0.7183970808982849, "learning_rate": 9.278490855781596e-05, "loss": 0.011787215247750282, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.01186, "step": 115, "tokens/total": 4011024, "tokens/train_per_sec_per_gpu": 27.15, "tokens/trainable": 52487} +{"epoch": 0.453125, "grad_norm": 0.6703223586082458, "learning_rate": 9.262646494908604e-05, "loss": 0.010669905692338943, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.01073, "step": 116, "tokens/total": 4048976, "tokens/train_per_sec_per_gpu": 25.88, "tokens/trainable": 52965} +{"epoch": 0.45703125, "grad_norm": 0.5820775032043457, "learning_rate": 9.246645554956457e-05, "loss": 0.016143091022968292, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.01627, "step": 117, "tokens/total": 4083376, "tokens/train_per_sec_per_gpu": 26.77, "tokens/trainable": 53450} +{"epoch": 0.4609375, "grad_norm": 0.3980841040611267, "learning_rate": 9.230488701789578e-05, "loss": 0.015761848539114, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.01589, "step": 118, "tokens/total": 4118576, "tokens/train_per_sec_per_gpu": 26.55, "tokens/trainable": 53927} +{"epoch": 0.46484375, "grad_norm": 0.6324167251586914, "learning_rate": 9.214176607760577e-05, "loss": 0.014713859185576439, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.29, "memory/max_allocated (GiB)": 38.29, "ppl": 1.01482, "step": 119, "tokens/total": 4154480, "tokens/train_per_sec_per_gpu": 26.09, "tokens/trainable": 54395} +{"epoch": 0.46875, "grad_norm": 0.3719923794269562, "learning_rate": 9.197709951682268e-05, "loss": 0.00938943400979042, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.00943, "step": 120, "tokens/total": 4190096, "tokens/train_per_sec_per_gpu": 27.66, "tokens/trainable": 54871} +{"epoch": 0.47265625, "grad_norm": 0.447318434715271, "learning_rate": 9.181089418799428e-05, "loss": 0.011788792908191681, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 39.32, "memory/max_allocated (GiB)": 39.32, "ppl": 1.01186, "step": 121, "tokens/total": 4229312, "tokens/train_per_sec_per_gpu": 23.25, "tokens/trainable": 55356} +{"epoch": 0.4765625, "grad_norm": 0.33465248346328735, "learning_rate": 9.164315700760271e-05, "loss": 0.005025719292461872, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00504, "step": 122, "tokens/total": 4264864, "tokens/train_per_sec_per_gpu": 25.64, "tokens/trainable": 55836} +{"epoch": 0.48046875, "grad_norm": 0.7611549496650696, "learning_rate": 9.147389495587671e-05, "loss": 0.006783052813261747, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 36.18, "memory/max_allocated (GiB)": 36.18, "ppl": 1.00681, "step": 123, "tokens/total": 4295360, "tokens/train_per_sec_per_gpu": 29.61, "tokens/trainable": 56275} +{"epoch": 0.484375, "grad_norm": 0.29589954018592834, "learning_rate": 9.130311507650116e-05, "loss": 0.002907787449657917, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 36.98, "memory/max_allocated (GiB)": 36.98, "ppl": 1.00291, "step": 124, "tokens/total": 4328336, "tokens/train_per_sec_per_gpu": 26.61, "tokens/trainable": 56745} +{"epoch": 0.48828125, "grad_norm": 0.899045467376709, "learning_rate": 9.113082447632394e-05, "loss": 0.011816762387752533, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 38.49, "memory/max_allocated (GiB)": 38.49, "ppl": 1.01189, "step": 125, "tokens/total": 4365552, "tokens/train_per_sec_per_gpu": 22.36, "tokens/trainable": 57171} +{"epoch": 0.4921875, "grad_norm": 1.591452956199646, "learning_rate": 9.09570303250602e-05, "loss": 0.0211433507502079, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.02137, "step": 126, "tokens/total": 4399376, "tokens/train_per_sec_per_gpu": 27.64, "tokens/trainable": 57642} +{"epoch": 0.49609375, "grad_norm": 0.12557776272296906, "learning_rate": 9.078173985499394e-05, "loss": 0.0013694773660972714, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00137, "step": 127, "tokens/total": 4434880, "tokens/train_per_sec_per_gpu": 29.12, "tokens/trainable": 58134} +{"epoch": 0.5, "grad_norm": 0.6462069153785706, "learning_rate": 9.060496036067713e-05, "loss": 0.0039420947432518005, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.00395, "step": 128, "tokens/total": 4469280, "tokens/train_per_sec_per_gpu": 23.5, "tokens/trainable": 58564} +{"epoch": 0.50390625, "grad_norm": 1.0410305261611938, "learning_rate": 9.042669919862615e-05, "loss": 0.028042297810316086, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.02844, "step": 129, "tokens/total": 4503408, "tokens/train_per_sec_per_gpu": 27.59, "tokens/trainable": 59038} +{"epoch": 0.5078125, "grad_norm": 0.6693357825279236, "learning_rate": 9.024696378701557e-05, "loss": 0.007229968905448914, "memory/device_reserved (GiB)": 39.68, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.00726, "step": 130, "tokens/total": 4536976, "tokens/train_per_sec_per_gpu": 28.83, "tokens/trainable": 59495} +{"epoch": 0.51171875, "grad_norm": 2.203536033630371, "learning_rate": 9.006576160536948e-05, "loss": 0.015040775761008263, "memory/device_reserved (GiB)": 39.68, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.01515, "step": 131, "tokens/total": 4570304, "tokens/train_per_sec_per_gpu": 25.41, "tokens/trainable": 59911} +{"epoch": 0.515625, "grad_norm": 0.5914306640625, "learning_rate": 8.988310019425035e-05, "loss": 0.010486564598977566, "memory/device_reserved (GiB)": 39.68, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.01054, "step": 132, "tokens/total": 4602912, "tokens/train_per_sec_per_gpu": 29.4, "tokens/trainable": 60371} +{"epoch": 0.51953125, "grad_norm": 0.8876322507858276, "learning_rate": 8.969898715494506e-05, "loss": 0.013423720374703407, "memory/device_reserved (GiB)": 40.09, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.01351, "step": 133, "tokens/total": 4638080, "tokens/train_per_sec_per_gpu": 26.25, "tokens/trainable": 60827} +{"epoch": 0.5234375, "grad_norm": 0.5613790154457092, "learning_rate": 8.951343014914869e-05, "loss": 0.010510994121432304, "memory/device_reserved (GiB)": 40.1, "memory/max_active (GiB)": 38.52, "memory/max_allocated (GiB)": 38.52, "ppl": 1.01057, "step": 134, "tokens/total": 4672480, "tokens/train_per_sec_per_gpu": 29.37, "tokens/trainable": 61320} +{"epoch": 0.52734375, "grad_norm": 0.4330528974533081, "learning_rate": 8.932643689864568e-05, "loss": 0.008790168911218643, "memory/device_reserved (GiB)": 40.1, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00883, "step": 135, "tokens/total": 4707264, "tokens/train_per_sec_per_gpu": 25.05, "tokens/trainable": 61775} +{"epoch": 0.53125, "grad_norm": 0.45121654868125916, "learning_rate": 8.913801518498845e-05, "loss": 0.005329091101884842, "memory/device_reserved (GiB)": 40.1, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 1.00534, "step": 136, "tokens/total": 4743696, "tokens/train_per_sec_per_gpu": 25.43, "tokens/trainable": 62248} +{"epoch": 0.53515625, "grad_norm": 0.5867933034896851, "learning_rate": 8.894817284917364e-05, "loss": 0.010761021636426449, "memory/device_reserved (GiB)": 40.92, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.01082, "step": 137, "tokens/total": 4781424, "tokens/train_per_sec_per_gpu": 24.81, "tokens/trainable": 62730} +{"epoch": 0.5390625, "grad_norm": 0.1141308844089508, "learning_rate": 8.875691779131569e-05, "loss": 0.001294588204473257, "memory/device_reserved (GiB)": 40.92, "memory/max_active (GiB)": 38.99, "memory/max_allocated (GiB)": 38.99, "ppl": 1.0013, "step": 138, "tokens/total": 4820752, "tokens/train_per_sec_per_gpu": 22.44, "tokens/trainable": 63166} +{"epoch": 0.54296875, "grad_norm": 1.130203127861023, "learning_rate": 8.856425797031829e-05, "loss": 0.02372508868575096, "memory/device_reserved (GiB)": 40.92, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.02401, "step": 139, "tokens/total": 4854688, "tokens/train_per_sec_per_gpu": 25.91, "tokens/trainable": 63631} +{"epoch": 0.546875, "grad_norm": 0.6788852214813232, "learning_rate": 8.837020140354295e-05, "loss": 0.016889456659555435, "memory/device_reserved (GiB)": 40.92, "memory/max_active (GiB)": 38.24, "memory/max_allocated (GiB)": 38.24, "ppl": 1.01703, "step": 140, "tokens/total": 4891424, "tokens/train_per_sec_per_gpu": 22.87, "tokens/trainable": 64080} +{"epoch": 0.55078125, "grad_norm": 0.5644290447235107, "learning_rate": 8.817475616647554e-05, "loss": 0.00877390243113041, "memory/device_reserved (GiB)": 40.92, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.00881, "step": 141, "tokens/total": 4924464, "tokens/train_per_sec_per_gpu": 29.49, "tokens/trainable": 64537} +{"epoch": 0.5546875, "grad_norm": 0.5424112677574158, "learning_rate": 8.797793039239017e-05, "loss": 0.005857645533978939, "memory/device_reserved (GiB)": 40.92, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.00587, "step": 142, "tokens/total": 4957072, "tokens/train_per_sec_per_gpu": 27.88, "tokens/trainable": 65011} +{"epoch": 0.55859375, "grad_norm": 1.1823194026947021, "learning_rate": 8.777973227201069e-05, "loss": 0.014200380071997643, "memory/device_reserved (GiB)": 40.92, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.0143, "step": 143, "tokens/total": 4991840, "tokens/train_per_sec_per_gpu": 28.05, "tokens/trainable": 65511} +{"epoch": 0.5625, "grad_norm": 0.2984083890914917, "learning_rate": 8.758017005316988e-05, "loss": 0.0035043247044086456, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.18, "memory/max_allocated (GiB)": 39.18, "ppl": 1.00351, "step": 144, "tokens/total": 5029312, "tokens/train_per_sec_per_gpu": 24.12, "tokens/trainable": 65996} +{"epoch": 0.56640625, "grad_norm": 2.1813676357269287, "learning_rate": 8.737925204046629e-05, "loss": 0.015415756031870842, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.56, "memory/max_allocated (GiB)": 36.56, "ppl": 1.01554, "step": 145, "tokens/total": 5060800, "tokens/train_per_sec_per_gpu": 26.69, "tokens/trainable": 66423} +{"epoch": 0.5703125, "grad_norm": 0.9645947217941284, "learning_rate": 8.717698659491851e-05, "loss": 0.010896027088165283, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.86, "memory/max_allocated (GiB)": 38.86, "ppl": 1.01096, "step": 146, "tokens/total": 5096448, "tokens/train_per_sec_per_gpu": 26.19, "tokens/trainable": 66905} +{"epoch": 0.57421875, "grad_norm": 0.15304027497768402, "learning_rate": 8.697338213361735e-05, "loss": 0.003344588680192828, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00335, "step": 147, "tokens/total": 5132928, "tokens/train_per_sec_per_gpu": 26.76, "tokens/trainable": 67363} +{"epoch": 0.578125, "grad_norm": 0.14357154071331024, "learning_rate": 8.676844712937552e-05, "loss": 0.001274213893339038, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.00128, "step": 148, "tokens/total": 5167008, "tokens/train_per_sec_per_gpu": 26.58, "tokens/trainable": 67829} +{"epoch": 0.58203125, "grad_norm": 1.1139274835586548, "learning_rate": 8.656219011037509e-05, "loss": 0.011996445246040821, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.14, "memory/max_allocated (GiB)": 36.14, "ppl": 1.01207, "step": 149, "tokens/total": 5198656, "tokens/train_per_sec_per_gpu": 27.24, "tokens/trainable": 68286} +{"epoch": 0.5859375, "grad_norm": 0.6952595710754395, "learning_rate": 8.63546196598125e-05, "loss": 0.007206828333437443, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.00723, "step": 150, "tokens/total": 5235600, "tokens/train_per_sec_per_gpu": 24.38, "tokens/trainable": 68750} +{"epoch": 0.58984375, "grad_norm": 0.7907957434654236, "learning_rate": 8.614574441554145e-05, "loss": 0.02507767453789711, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.57, "memory/max_allocated (GiB)": 39.57, "ppl": 1.02539, "step": 151, "tokens/total": 5273808, "tokens/train_per_sec_per_gpu": 25.57, "tokens/trainable": 69231} +{"epoch": 0.59375, "grad_norm": 0.5200352668762207, "learning_rate": 8.593557306971349e-05, "loss": 0.009828459471464157, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.00988, "step": 152, "tokens/total": 5310480, "tokens/train_per_sec_per_gpu": 27.18, "tokens/trainable": 69711} +{"epoch": 0.59765625, "grad_norm": 1.221924901008606, "learning_rate": 8.572411436841618e-05, "loss": 0.00911647081375122, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00916, "step": 153, "tokens/total": 5349264, "tokens/train_per_sec_per_gpu": 23.84, "tokens/trainable": 70178} +{"epoch": 0.6015625, "grad_norm": 1.6028553247451782, "learning_rate": 8.551137711130922e-05, "loss": 0.043976884335279465, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.04496, "step": 154, "tokens/total": 5382736, "tokens/train_per_sec_per_gpu": 30.02, "tokens/trainable": 70670} +{"epoch": 0.60546875, "grad_norm": 0.6531856060028076, "learning_rate": 8.529737015125824e-05, "loss": 0.009027759544551373, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.00907, "step": 155, "tokens/total": 5420256, "tokens/train_per_sec_per_gpu": 22.31, "tokens/trainable": 71111} +{"epoch": 0.609375, "grad_norm": 0.3289493918418884, "learning_rate": 8.508210239396639e-05, "loss": 0.005970867816358805, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.93, "memory/max_allocated (GiB)": 36.93, "ppl": 1.00599, "step": 156, "tokens/total": 5454080, "tokens/train_per_sec_per_gpu": 31.45, "tokens/trainable": 71623} +{"epoch": 0.61328125, "grad_norm": 0.5737825036048889, "learning_rate": 8.486558279760375e-05, "loss": 0.004970227833837271, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.00498, "step": 157, "tokens/total": 5489728, "tokens/train_per_sec_per_gpu": 27.06, "tokens/trainable": 72100} +{"epoch": 0.6171875, "grad_norm": 0.15222811698913574, "learning_rate": 8.464782037243449e-05, "loss": 0.0025803556200116873, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.00258, "step": 158, "tokens/total": 5526512, "tokens/train_per_sec_per_gpu": 25.37, "tokens/trainable": 72598} +{"epoch": 0.62109375, "grad_norm": 0.33065178990364075, "learning_rate": 8.442882418044202e-05, "loss": 0.00865921750664711, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.0087, "step": 159, "tokens/total": 5563456, "tokens/train_per_sec_per_gpu": 24.41, "tokens/trainable": 73071} +{"epoch": 0.625, "grad_norm": 0.17117327451705933, "learning_rate": 8.420860333495179e-05, "loss": 0.0036261468194425106, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.22, "memory/max_allocated (GiB)": 38.22, "ppl": 1.00363, "step": 160, "tokens/total": 5599520, "tokens/train_per_sec_per_gpu": 28.65, "tokens/trainable": 73560} +{"epoch": 0.62890625, "grad_norm": 1.2410780191421509, "learning_rate": 8.398716700025208e-05, "loss": 0.01447224710136652, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.01458, "step": 161, "tokens/total": 5637264, "tokens/train_per_sec_per_gpu": 27.28, "tokens/trainable": 74083} +{"epoch": 0.6328125, "grad_norm": 0.28758248686790466, "learning_rate": 8.376452439121266e-05, "loss": 0.008277230896055698, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.61, "memory/max_allocated (GiB)": 36.61, "ppl": 1.00831, "step": 162, "tokens/total": 5670064, "tokens/train_per_sec_per_gpu": 24.86, "tokens/trainable": 74530} +{"epoch": 0.63671875, "grad_norm": 0.30860552191734314, "learning_rate": 8.354068477290124e-05, "loss": 0.00415801489725709, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.00417, "step": 163, "tokens/total": 5702800, "tokens/train_per_sec_per_gpu": 27.76, "tokens/trainable": 74999} +{"epoch": 0.640625, "grad_norm": 0.9767330288887024, "learning_rate": 8.331565746019807e-05, "loss": 0.012953978963196278, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.01304, "step": 164, "tokens/total": 5739792, "tokens/train_per_sec_per_gpu": 22.72, "tokens/trainable": 75450} +{"epoch": 0.64453125, "grad_norm": 0.3855861723423004, "learning_rate": 8.308945181740812e-05, "loss": 0.006549662444740534, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 1.00657, "step": 165, "tokens/total": 5773856, "tokens/train_per_sec_per_gpu": 26.26, "tokens/trainable": 75921} +{"epoch": 0.6484375, "grad_norm": 3.681177854537964, "learning_rate": 8.286207725787153e-05, "loss": 0.026050403714179993, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.54, "memory/max_allocated (GiB)": 39.54, "ppl": 1.02639, "step": 166, "tokens/total": 5809520, "tokens/train_per_sec_per_gpu": 28.87, "tokens/trainable": 76425} +{"epoch": 0.65234375, "grad_norm": 0.41782864928245544, "learning_rate": 8.263354324357182e-05, "loss": 0.006738392636179924, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.00676, "step": 167, "tokens/total": 5841920, "tokens/train_per_sec_per_gpu": 28.29, "tokens/trainable": 76863} +{"epoch": 0.65625, "grad_norm": 6.304466247558594, "learning_rate": 8.240385928474219e-05, "loss": 0.054863687604665756, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.07, "memory/max_allocated (GiB)": 38.07, "ppl": 1.0564, "step": 168, "tokens/total": 5874912, "tokens/train_per_sec_per_gpu": 25.98, "tokens/trainable": 77298} +{"epoch": 0.66015625, "grad_norm": 0.5624664425849915, "learning_rate": 8.217303493946967e-05, "loss": 0.006827559322118759, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.00685, "step": 169, "tokens/total": 5908768, "tokens/train_per_sec_per_gpu": 28.54, "tokens/trainable": 77735} +{"epoch": 0.6640625, "grad_norm": 0.66590815782547, "learning_rate": 8.194107981329746e-05, "loss": 0.01314166933298111, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.78, "memory/max_allocated (GiB)": 36.78, "ppl": 1.01323, "step": 170, "tokens/total": 5942128, "tokens/train_per_sec_per_gpu": 27.5, "tokens/trainable": 78179} +{"epoch": 0.66796875, "grad_norm": 0.556177020072937, "learning_rate": 8.170800355882518e-05, "loss": 0.022578949108719826, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.36, "memory/max_allocated (GiB)": 38.36, "ppl": 1.02284, "step": 171, "tokens/total": 5979232, "tokens/train_per_sec_per_gpu": 27.01, "tokens/trainable": 78660} +{"epoch": 0.671875, "grad_norm": 1.9835500717163086, "learning_rate": 8.147381587530713e-05, "loss": 0.029374759644269943, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.02981, "step": 172, "tokens/total": 6014000, "tokens/train_per_sec_per_gpu": 25.28, "tokens/trainable": 79110} +{"epoch": 0.67578125, "grad_norm": 0.38315680623054504, "learning_rate": 8.123852650824877e-05, "loss": 0.0036519260611385107, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.00366, "step": 173, "tokens/total": 6048688, "tokens/train_per_sec_per_gpu": 27.06, "tokens/trainable": 79576} +{"epoch": 0.6796875, "grad_norm": 0.033161379396915436, "learning_rate": 8.100214524900103e-05, "loss": 0.0004422229831106961, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 35.79, "memory/max_allocated (GiB)": 35.79, "ppl": 1.00044, "step": 174, "tokens/total": 6078768, "tokens/train_per_sec_per_gpu": 25.28, "tokens/trainable": 80001} +{"epoch": 0.68359375, "grad_norm": 0.16665847599506378, "learning_rate": 8.076468193435301e-05, "loss": 0.0027730583678931, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00278, "step": 175, "tokens/total": 6111296, "tokens/train_per_sec_per_gpu": 27.9, "tokens/trainable": 80452} +{"epoch": 0.6875, "grad_norm": 0.39398878812789917, "learning_rate": 8.052614644612253e-05, "loss": 0.013822358101606369, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.01392, "step": 176, "tokens/total": 6146224, "tokens/train_per_sec_per_gpu": 25.94, "tokens/trainable": 80886} +{"epoch": 0.69140625, "grad_norm": 0.7304824590682983, "learning_rate": 8.028654871074489e-05, "loss": 0.012839030474424362, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.01292, "step": 177, "tokens/total": 6176576, "tokens/train_per_sec_per_gpu": 28.3, "tokens/trainable": 81325} +{"epoch": 0.6953125, "grad_norm": 1.5097708702087402, "learning_rate": 8.004589869885986e-05, "loss": 0.02121157944202423, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.2, "memory/max_allocated (GiB)": 37.2, "ppl": 1.02144, "step": 178, "tokens/total": 6208432, "tokens/train_per_sec_per_gpu": 26.06, "tokens/trainable": 81732} +{"epoch": 0.69921875, "grad_norm": 0.3124157190322876, "learning_rate": 7.980420642489674e-05, "loss": 0.010472262278199196, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.4, "memory/max_allocated (GiB)": 36.4, "ppl": 1.01053, "step": 179, "tokens/total": 6236640, "tokens/train_per_sec_per_gpu": 34.33, "tokens/trainable": 82171} +{"epoch": 0.703125, "grad_norm": 0.08370061218738556, "learning_rate": 7.95614819466576e-05, "loss": 0.0015829324256628752, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.00158, "step": 180, "tokens/total": 6271184, "tokens/train_per_sec_per_gpu": 27.64, "tokens/trainable": 82670} +{"epoch": 0.70703125, "grad_norm": 0.37564459443092346, "learning_rate": 7.931773536489872e-05, "loss": 0.0105284433811903, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 35.89, "memory/max_allocated (GiB)": 35.89, "ppl": 1.01058, "step": 181, "tokens/total": 6301120, "tokens/train_per_sec_per_gpu": 25.53, "tokens/trainable": 83084} +{"epoch": 0.7109375, "grad_norm": 0.9345095753669739, "learning_rate": 7.907297682291035e-05, "loss": 0.011572718620300293, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.01164, "step": 182, "tokens/total": 6334192, "tokens/train_per_sec_per_gpu": 25.93, "tokens/trainable": 83500} +{"epoch": 0.71484375, "grad_norm": 0.3250158727169037, "learning_rate": 7.882721650609442e-05, "loss": 0.0069688186049461365, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.36, "memory/max_allocated (GiB)": 36.36, "ppl": 1.00699, "step": 183, "tokens/total": 6366192, "tokens/train_per_sec_per_gpu": 25.41, "tokens/trainable": 83896} +{"epoch": 0.71875, "grad_norm": 0.39017200469970703, "learning_rate": 7.85804646415409e-05, "loss": 0.008210340514779091, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.53, "memory/max_allocated (GiB)": 38.53, "ppl": 1.00824, "step": 184, "tokens/total": 6400496, "tokens/train_per_sec_per_gpu": 28.42, "tokens/trainable": 84399} +{"epoch": 0.72265625, "grad_norm": 0.1964060366153717, "learning_rate": 7.833273149760207e-05, "loss": 0.004957118537276983, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.00497, "step": 185, "tokens/total": 6434992, "tokens/train_per_sec_per_gpu": 26.4, "tokens/trainable": 84873} +{"epoch": 0.7265625, "grad_norm": 0.5299766063690186, "learning_rate": 7.808402738346527e-05, "loss": 0.008438036777079105, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.09, "memory/max_allocated (GiB)": 37.09, "ppl": 1.00847, "step": 186, "tokens/total": 6469296, "tokens/train_per_sec_per_gpu": 25.61, "tokens/trainable": 85296} +{"epoch": 0.73046875, "grad_norm": 0.5880584716796875, "learning_rate": 7.783436264872382e-05, "loss": 0.013135985471308231, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.01322, "step": 187, "tokens/total": 6503760, "tokens/train_per_sec_per_gpu": 25.44, "tokens/trainable": 85741} +{"epoch": 0.734375, "grad_norm": 0.8151103258132935, "learning_rate": 7.758374768294647e-05, "loss": 0.005244716070592403, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00526, "step": 188, "tokens/total": 6540448, "tokens/train_per_sec_per_gpu": 23.22, "tokens/trainable": 86176} +{"epoch": 0.73828125, "grad_norm": 0.20034153759479523, "learning_rate": 7.733219291524489e-05, "loss": 0.001138855703175068, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.00114, "step": 189, "tokens/total": 6577648, "tokens/train_per_sec_per_gpu": 24.95, "tokens/trainable": 86649} +{"epoch": 0.7421875, "grad_norm": 0.11049848794937134, "learning_rate": 7.707970881383977e-05, "loss": 0.0013307350454851985, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.00133, "step": 190, "tokens/total": 6615600, "tokens/train_per_sec_per_gpu": 21.53, "tokens/trainable": 87084} +{"epoch": 0.74609375, "grad_norm": 1.1597856283187866, "learning_rate": 7.682630588562518e-05, "loss": 0.010004757903516293, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.01005, "step": 191, "tokens/total": 6648880, "tokens/train_per_sec_per_gpu": 24.66, "tokens/trainable": 87499} +{"epoch": 0.75, "grad_norm": 0.01856999844312668, "learning_rate": 7.657199467573129e-05, "loss": 0.00016168829461093992, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.00016, "step": 192, "tokens/total": 6686784, "tokens/train_per_sec_per_gpu": 25.83, "tokens/trainable": 87987} +{"epoch": 0.75390625, "grad_norm": 0.13616882264614105, "learning_rate": 7.631678576708561e-05, "loss": 0.001009517116472125, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.00101, "step": 193, "tokens/total": 6722480, "tokens/train_per_sec_per_gpu": 26.26, "tokens/trainable": 88470} +{"epoch": 0.7578125, "grad_norm": 0.6032267212867737, "learning_rate": 7.606068977997255e-05, "loss": 0.012968374416232109, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.01305, "step": 194, "tokens/total": 6757168, "tokens/train_per_sec_per_gpu": 26.31, "tokens/trainable": 88923} +{"epoch": 0.76171875, "grad_norm": 0.026390785351395607, "learning_rate": 7.580371737159148e-05, "loss": 0.00040499059832654893, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.02, "memory/max_allocated (GiB)": 37.02, "ppl": 1.00041, "step": 195, "tokens/total": 6790000, "tokens/train_per_sec_per_gpu": 27.18, "tokens/trainable": 89338} +{"epoch": 0.765625, "grad_norm": 1.0891882181167603, "learning_rate": 7.554587923561324e-05, "loss": 0.02822169102728367, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.02862, "step": 196, "tokens/total": 6826384, "tokens/train_per_sec_per_gpu": 25.91, "tokens/trainable": 89812} +{"epoch": 0.76953125, "grad_norm": 0.712473452091217, "learning_rate": 7.528718610173511e-05, "loss": 0.007405821233987808, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.00743, "step": 197, "tokens/total": 6860896, "tokens/train_per_sec_per_gpu": 26.4, "tokens/trainable": 90252} +{"epoch": 0.7734375, "grad_norm": 0.1804046779870987, "learning_rate": 7.502764873523431e-05, "loss": 0.0006743186386302114, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.28, "memory/max_allocated (GiB)": 39.28, "ppl": 1.00067, "step": 198, "tokens/total": 6896912, "tokens/train_per_sec_per_gpu": 26.29, "tokens/trainable": 90718} +{"epoch": 0.77734375, "grad_norm": 0.13520115613937378, "learning_rate": 7.476727793652011e-05, "loss": 0.0012067706556990743, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.00121, "step": 199, "tokens/total": 6928592, "tokens/train_per_sec_per_gpu": 30.0, "tokens/trainable": 91194} +{"epoch": 0.78125, "grad_norm": 1.3889774084091187, "learning_rate": 7.450608454068415e-05, "loss": 0.009603383019566536, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.00965, "step": 200, "tokens/total": 6964672, "tokens/train_per_sec_per_gpu": 24.23, "tokens/trainable": 91649} +{"epoch": 0.78515625, "grad_norm": 0.5965397357940674, "learning_rate": 7.424407941704987e-05, "loss": 0.002379906363785267, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.00238, "step": 201, "tokens/total": 6996336, "tokens/train_per_sec_per_gpu": 29.24, "tokens/trainable": 92078} +{"epoch": 0.7890625, "grad_norm": 0.7006675004959106, "learning_rate": 7.398127346871986e-05, "loss": 0.004512009210884571, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.59, "memory/max_allocated (GiB)": 37.59, "ppl": 1.00452, "step": 202, "tokens/total": 7030544, "tokens/train_per_sec_per_gpu": 24.61, "tokens/trainable": 92536} +{"epoch": 0.79296875, "grad_norm": 0.2464604526758194, "learning_rate": 7.371767763212238e-05, "loss": 0.003479420207440853, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.00349, "step": 203, "tokens/total": 7066480, "tokens/train_per_sec_per_gpu": 27.69, "tokens/trainable": 93032} +{"epoch": 0.796875, "grad_norm": 2.470592737197876, "learning_rate": 7.345330287655617e-05, "loss": 0.010843649506568909, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.0109, "step": 204, "tokens/total": 7098768, "tokens/train_per_sec_per_gpu": 30.61, "tokens/trainable": 93504} +{"epoch": 0.80078125, "grad_norm": 0.24514669179916382, "learning_rate": 7.31881602037339e-05, "loss": 0.0038731868844479322, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.00388, "step": 205, "tokens/total": 7132144, "tokens/train_per_sec_per_gpu": 23.18, "tokens/trainable": 93925} +{"epoch": 0.8046875, "grad_norm": 0.8248734474182129, "learning_rate": 7.29222606473245e-05, "loss": 0.010930064134299755, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.01099, "step": 206, "tokens/total": 7164720, "tokens/train_per_sec_per_gpu": 27.77, "tokens/trainable": 94365} +{"epoch": 0.80859375, "grad_norm": 0.42984187602996826, "learning_rate": 7.265561527249383e-05, "loss": 0.006969079840928316, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.00699, "step": 207, "tokens/total": 7199648, "tokens/train_per_sec_per_gpu": 24.76, "tokens/trainable": 94845} +{"epoch": 0.8125, "grad_norm": 0.5636523365974426, "learning_rate": 7.238823517544436e-05, "loss": 0.013653703033924103, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.01375, "step": 208, "tokens/total": 7235008, "tokens/train_per_sec_per_gpu": 26.69, "tokens/trainable": 95285} +{"epoch": 0.81640625, "grad_norm": 0.926177442073822, "learning_rate": 7.212013148295333e-05, "loss": 0.013229376636445522, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.01332, "step": 209, "tokens/total": 7269776, "tokens/train_per_sec_per_gpu": 25.0, "tokens/trainable": 95758} +{"epoch": 0.8203125, "grad_norm": 0.4189479947090149, "learning_rate": 7.185131535190975e-05, "loss": 0.007416154257953167, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.59, "memory/max_allocated (GiB)": 39.59, "ppl": 1.00744, "step": 210, "tokens/total": 7306848, "tokens/train_per_sec_per_gpu": 23.64, "tokens/trainable": 96192} +{"epoch": 0.82421875, "grad_norm": 0.9137555360794067, "learning_rate": 7.158179796885005e-05, "loss": 0.008893052116036415, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00893, "step": 211, "tokens/total": 7345600, "tokens/train_per_sec_per_gpu": 23.87, "tokens/trainable": 96641} +{"epoch": 0.828125, "grad_norm": 1.3528854846954346, "learning_rate": 7.131159054949273e-05, "loss": 0.03198888152837753, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.03251, "step": 212, "tokens/total": 7383040, "tokens/train_per_sec_per_gpu": 26.65, "tokens/trainable": 97160} +{"epoch": 0.83203125, "grad_norm": 0.3072126805782318, "learning_rate": 7.104070433827139e-05, "loss": 0.0034148776903748512, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 36.1, "memory/max_allocated (GiB)": 36.1, "ppl": 1.00342, "step": 213, "tokens/total": 7414208, "tokens/train_per_sec_per_gpu": 28.55, "tokens/trainable": 97611} +{"epoch": 0.8359375, "grad_norm": 0.5077757239341736, "learning_rate": 7.076915060786705e-05, "loss": 0.006043871399015188, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.00606, "step": 214, "tokens/total": 7448864, "tokens/train_per_sec_per_gpu": 28.29, "tokens/trainable": 98111} +{"epoch": 0.83984375, "grad_norm": 0.2595917582511902, "learning_rate": 7.049694065873882e-05, "loss": 0.0037103407084941864, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.97, "memory/max_allocated (GiB)": 37.97, "ppl": 1.00372, "step": 215, "tokens/total": 7484096, "tokens/train_per_sec_per_gpu": 26.62, "tokens/trainable": 98575} +{"epoch": 0.84375, "grad_norm": 0.29643407464027405, "learning_rate": 7.022408581865382e-05, "loss": 0.002975697163492441, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.00298, "step": 216, "tokens/total": 7517056, "tokens/train_per_sec_per_gpu": 24.9, "tokens/trainable": 98991} +{"epoch": 0.84765625, "grad_norm": 0.4570572078227997, "learning_rate": 6.99505974422157e-05, "loss": 0.004948528483510017, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 36.32, "memory/max_allocated (GiB)": 36.32, "ppl": 1.00496, "step": 217, "tokens/total": 7548448, "tokens/train_per_sec_per_gpu": 33.6, "tokens/trainable": 99454} +{"epoch": 0.8515625, "grad_norm": 0.289658784866333, "learning_rate": 6.967648691039213e-05, "loss": 0.0009716551285237074, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00097, "step": 218, "tokens/total": 7582656, "tokens/train_per_sec_per_gpu": 26.46, "tokens/trainable": 99925} +{"epoch": 0.85546875, "grad_norm": 0.1011926680803299, "learning_rate": 6.940176563004123e-05, "loss": 0.002380694029852748, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00238, "step": 219, "tokens/total": 7618960, "tokens/train_per_sec_per_gpu": 26.25, "tokens/trainable": 100371} +{"epoch": 0.859375, "grad_norm": 0.3280646502971649, "learning_rate": 6.912644503343682e-05, "loss": 0.006010346580296755, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.35, "memory/max_allocated (GiB)": 39.35, "ppl": 1.00603, "step": 220, "tokens/total": 7655504, "tokens/train_per_sec_per_gpu": 26.19, "tokens/trainable": 100855} +{"epoch": 0.86328125, "grad_norm": 0.46524250507354736, "learning_rate": 6.885053657779273e-05, "loss": 0.001524287392385304, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00153, "step": 221, "tokens/total": 7689872, "tokens/train_per_sec_per_gpu": 21.2, "tokens/trainable": 101253} +{"epoch": 0.8671875, "grad_norm": 0.04459555447101593, "learning_rate": 6.857405174478604e-05, "loss": 0.0005410610465332866, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.00054, "step": 222, "tokens/total": 7724432, "tokens/train_per_sec_per_gpu": 26.08, "tokens/trainable": 101706} +{"epoch": 0.87109375, "grad_norm": 0.24617522954940796, "learning_rate": 6.82970020400792e-05, "loss": 0.002149344887584448, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.00215, "step": 223, "tokens/total": 7761440, "tokens/train_per_sec_per_gpu": 22.75, "tokens/trainable": 102158} +{"epoch": 0.875, "grad_norm": 0.11508461833000183, "learning_rate": 6.801939899284132e-05, "loss": 0.0014384619425982237, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 35.36, "memory/max_allocated (GiB)": 35.36, "ppl": 1.00144, "step": 224, "tokens/total": 7790048, "tokens/train_per_sec_per_gpu": 30.39, "tokens/trainable": 102580} +{"epoch": 0.87890625, "grad_norm": 0.023394646123051643, "learning_rate": 6.774125415526827e-05, "loss": 0.00031606017728336155, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.00032, "step": 225, "tokens/total": 7822352, "tokens/train_per_sec_per_gpu": 30.62, "tokens/trainable": 103063} +{"epoch": 0.8828125, "grad_norm": 0.5077522397041321, "learning_rate": 6.746257910210214e-05, "loss": 0.0016863496275618672, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00169, "step": 226, "tokens/total": 7861088, "tokens/train_per_sec_per_gpu": 22.12, "tokens/trainable": 103510} +{"epoch": 0.88671875, "grad_norm": 0.3831356465816498, "learning_rate": 6.718338543014937e-05, "loss": 0.0041476525366306305, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.51, "memory/max_allocated (GiB)": 36.51, "ppl": 1.00416, "step": 227, "tokens/total": 7891456, "tokens/train_per_sec_per_gpu": 34.23, "tokens/trainable": 103980} +{"epoch": 0.890625, "grad_norm": 0.009819943457841873, "learning_rate": 6.69036847577983e-05, "loss": 6.461025623138994e-05, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.13, "memory/max_allocated (GiB)": 36.13, "ppl": 1.00006, "step": 228, "tokens/total": 7922704, "tokens/train_per_sec_per_gpu": 28.63, "tokens/trainable": 104413} +{"epoch": 0.89453125, "grad_norm": 1.1191335916519165, "learning_rate": 6.662348872453553e-05, "loss": 0.004857989959418774, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.00487, "step": 229, "tokens/total": 7952912, "tokens/train_per_sec_per_gpu": 29.79, "tokens/trainable": 104845} +{"epoch": 0.8984375, "grad_norm": 0.048952002078294754, "learning_rate": 6.63428089904618e-05, "loss": 0.00020188964845146984, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.0002, "step": 230, "tokens/total": 7987504, "tokens/train_per_sec_per_gpu": 24.01, "tokens/trainable": 105289} +{"epoch": 0.90234375, "grad_norm": 0.9098848700523376, "learning_rate": 6.60616572358065e-05, "loss": 0.005342357791960239, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 1.00536, "step": 231, "tokens/total": 8025936, "tokens/train_per_sec_per_gpu": 21.77, "tokens/trainable": 105722} +{"epoch": 0.90625, "grad_norm": 0.018555086106061935, "learning_rate": 6.578004516044172e-05, "loss": 9.742080874275416e-05, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.59, "memory/max_allocated (GiB)": 36.59, "ppl": 1.0001, "step": 232, "tokens/total": 8058544, "tokens/train_per_sec_per_gpu": 29.19, "tokens/trainable": 106213} +{"epoch": 0.91015625, "grad_norm": 0.03810635209083557, "learning_rate": 6.549798448339548e-05, "loss": 0.00022043912031222135, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 1.00022, "step": 233, "tokens/total": 8091456, "tokens/train_per_sec_per_gpu": 26.86, "tokens/trainable": 106643} +{"epoch": 0.9140625, "grad_norm": 0.8577053546905518, "learning_rate": 6.521548694236384e-05, "loss": 0.03450418636202812, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.97, "memory/max_allocated (GiB)": 38.97, "ppl": 1.03511, "step": 234, "tokens/total": 8127920, "tokens/train_per_sec_per_gpu": 25.52, "tokens/trainable": 107136} +{"epoch": 0.91796875, "grad_norm": 0.32384392619132996, "learning_rate": 6.493256429322259e-05, "loss": 0.0012937327846884727, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00129, "step": 235, "tokens/total": 8167472, "tokens/train_per_sec_per_gpu": 24.51, "tokens/trainable": 107588} +{"epoch": 0.921875, "grad_norm": 0.9242942929267883, "learning_rate": 6.464922830953799e-05, "loss": 0.005610171239823103, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.00563, "step": 236, "tokens/total": 8201792, "tokens/train_per_sec_per_gpu": 27.58, "tokens/trainable": 108087} +{"epoch": 0.92578125, "grad_norm": 2.10426926612854, "learning_rate": 6.436549078207688e-05, "loss": 0.011037425138056278, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 1.0111, "step": 237, "tokens/total": 8235504, "tokens/train_per_sec_per_gpu": 27.21, "tokens/trainable": 108533} +{"epoch": 0.9296875, "grad_norm": 0.025268711149692535, "learning_rate": 6.408136351831592e-05, "loss": 0.00037691835314035416, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.24, "memory/max_allocated (GiB)": 38.24, "ppl": 1.00038, "step": 238, "tokens/total": 8271392, "tokens/train_per_sec_per_gpu": 21.67, "tokens/trainable": 108965} +{"epoch": 0.93359375, "grad_norm": 0.3965263068675995, "learning_rate": 6.379685834195036e-05, "loss": 0.002362737664952874, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.00237, "step": 239, "tokens/total": 8304656, "tokens/train_per_sec_per_gpu": 28.79, "tokens/trainable": 109416} +{"epoch": 0.9375, "grad_norm": 0.05191374570131302, "learning_rate": 6.351198709240186e-05, "loss": 0.00047776784049347043, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.12, "memory/max_allocated (GiB)": 39.12, "ppl": 1.00048, "step": 240, "tokens/total": 8343280, "tokens/train_per_sec_per_gpu": 23.31, "tokens/trainable": 109864} +{"epoch": 0.94140625, "grad_norm": 0.5658698081970215, "learning_rate": 6.32267616243259e-05, "loss": 0.018501585349440575, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.01867, "step": 241, "tokens/total": 8377728, "tokens/train_per_sec_per_gpu": 26.25, "tokens/trainable": 110345} +{"epoch": 0.9453125, "grad_norm": 0.21305952966213226, "learning_rate": 6.294119380711849e-05, "loss": 0.0009843044681474566, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 37.15, "memory/max_allocated (GiB)": 37.15, "ppl": 1.00098, "step": 242, "tokens/total": 8412112, "tokens/train_per_sec_per_gpu": 24.77, "tokens/trainable": 110794} +{"epoch": 0.94921875, "grad_norm": 0.09626226872205734, "learning_rate": 6.265529552442209e-05, "loss": 0.0013750868383795023, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.00138, "step": 243, "tokens/total": 8449360, "tokens/train_per_sec_per_gpu": 24.98, "tokens/trainable": 111250} +{"epoch": 0.953125, "grad_norm": 0.8256192803382874, "learning_rate": 6.236907867363127e-05, "loss": 0.010135798715054989, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 1.01019, "step": 244, "tokens/total": 8482160, "tokens/train_per_sec_per_gpu": 26.97, "tokens/trainable": 111693} +{"epoch": 0.95703125, "grad_norm": 0.8985695838928223, "learning_rate": 6.208255516539749e-05, "loss": 0.00984526053071022, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.45, "memory/max_allocated (GiB)": 38.45, "ppl": 1.00989, "step": 245, "tokens/total": 8516272, "tokens/train_per_sec_per_gpu": 25.78, "tokens/trainable": 112158} +{"epoch": 0.9609375, "grad_norm": 1.0449771881103516, "learning_rate": 6.179573692313344e-05, "loss": 0.017693590372800827, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.98, "memory/max_allocated (GiB)": 36.98, "ppl": 1.01785, "step": 246, "tokens/total": 8548240, "tokens/train_per_sec_per_gpu": 28.82, "tokens/trainable": 112601} +{"epoch": 0.96484375, "grad_norm": 1.5198594331741333, "learning_rate": 6.150863588251694e-05, "loss": 0.0036184831988066435, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.34, "memory/max_allocated (GiB)": 36.34, "ppl": 1.00363, "step": 247, "tokens/total": 8580000, "tokens/train_per_sec_per_gpu": 25.82, "tokens/trainable": 113016} +{"epoch": 0.96875, "grad_norm": 0.920994758605957, "learning_rate": 6.122126399099419e-05, "loss": 0.016224408522248268, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.01636, "step": 248, "tokens/total": 8614864, "tokens/train_per_sec_per_gpu": 24.65, "tokens/trainable": 113476} +{"epoch": 0.97265625, "grad_norm": 0.46950212121009827, "learning_rate": 6.0933633207282615e-05, "loss": 0.010063028894364834, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.66, "memory/max_allocated (GiB)": 38.66, "ppl": 1.01011, "step": 249, "tokens/total": 8651008, "tokens/train_per_sec_per_gpu": 26.74, "tokens/trainable": 113941} +{"epoch": 0.9765625, "grad_norm": 0.5495955348014832, "learning_rate": 6.064575550087316e-05, "loss": 0.00882568396627903, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.83, "memory/max_allocated (GiB)": 36.83, "ppl": 1.00886, "step": 250, "tokens/total": 8683696, "tokens/train_per_sec_per_gpu": 30.32, "tokens/trainable": 114434} +{"epoch": 0.98046875, "grad_norm": 0.32995423674583435, "learning_rate": 6.0357642851532245e-05, "loss": 0.0031132253352552652, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.28, "memory/max_allocated (GiB)": 36.28, "ppl": 1.00312, "step": 251, "tokens/total": 8714880, "tokens/train_per_sec_per_gpu": 29.7, "tokens/trainable": 114898} +{"epoch": 0.984375, "grad_norm": 0.08071961998939514, "learning_rate": 6.0069307248803294e-05, "loss": 0.0013831165852025151, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.15, "memory/max_allocated (GiB)": 39.15, "ppl": 1.00138, "step": 252, "tokens/total": 8751072, "tokens/train_per_sec_per_gpu": 21.71, "tokens/trainable": 115331} +{"epoch": 0.98828125, "grad_norm": 0.05677878484129906, "learning_rate": 5.9780760691507635e-05, "loss": 0.0013400879688560963, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.73, "memory/max_allocated (GiB)": 36.73, "ppl": 1.00134, "step": 253, "tokens/total": 8782272, "tokens/train_per_sec_per_gpu": 28.08, "tokens/trainable": 115732} +{"epoch": 0.9921875, "grad_norm": 0.7585060000419617, "learning_rate": 5.9492015187245334e-05, "loss": 0.008542241528630257, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.28, "memory/max_allocated (GiB)": 39.28, "ppl": 1.00858, "step": 254, "tokens/total": 8819920, "tokens/train_per_sec_per_gpu": 25.67, "tokens/trainable": 116221} +{"epoch": 0.99609375, "grad_norm": 0.33372893929481506, "learning_rate": 5.920308275189541e-05, "loss": 0.0026933583430945873, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 37.91, "memory/max_allocated (GiB)": 37.91, "ppl": 1.0027, "step": 255, "tokens/total": 8855232, "tokens/train_per_sec_per_gpu": 23.83, "tokens/trainable": 116684} +{"epoch": 1.0, "grad_norm": 0.11901076138019562, "learning_rate": 5.8913975409115874e-05, "loss": 0.001760210026986897, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.00176, "step": 256, "tokens/total": 8889920, "tokens/train_per_sec_per_gpu": 24.71, "tokens/trainable": 117126} +{"epoch": 1.00390625, "grad_norm": 0.0420871265232563, "learning_rate": 5.8624705189843395e-05, "loss": 0.0006611713906750083, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00066, "step": 257, "tokens/total": 8922640, "tokens/train_per_sec_per_gpu": 30.17, "tokens/trainable": 117592} +{"epoch": 1.0078125, "grad_norm": 0.10980939120054245, "learning_rate": 5.833528413179249e-05, "loss": 0.00238519674167037, "memory/device_reserved (GiB)": 39.79, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00239, "step": 258, "tokens/total": 8958848, "tokens/train_per_sec_per_gpu": 21.67, "tokens/trainable": 118030} +{"epoch": 1.01171875, "grad_norm": 0.16587042808532715, "learning_rate": 5.80457242789548e-05, "loss": 0.0018970425007864833, "memory/device_reserved (GiB)": 39.79, "memory/max_active (GiB)": 36.51, "memory/max_allocated (GiB)": 36.51, "ppl": 1.0019, "step": 259, "tokens/total": 8991056, "tokens/train_per_sec_per_gpu": 26.45, "tokens/trainable": 118470} +{"epoch": 1.015625, "grad_norm": 0.2341405600309372, "learning_rate": 5.77560376810977e-05, "loss": 0.008489098399877548, "memory/device_reserved (GiB)": 39.79, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.00853, "step": 260, "tokens/total": 9024720, "tokens/train_per_sec_per_gpu": 28.4, "tokens/trainable": 118935} +{"epoch": 1.01953125, "grad_norm": 0.04855407401919365, "learning_rate": 5.7466236393263005e-05, "loss": 0.0008451620815321803, "memory/device_reserved (GiB)": 39.79, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.00085, "step": 261, "tokens/total": 9058944, "tokens/train_per_sec_per_gpu": 25.61, "tokens/trainable": 119381} +{"epoch": 1.0234375, "grad_norm": 0.3007769286632538, "learning_rate": 5.717633247526522e-05, "loss": 0.0033167628571391106, "memory/device_reserved (GiB)": 41.75, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.00332, "step": 262, "tokens/total": 9098816, "tokens/train_per_sec_per_gpu": 23.27, "tokens/trainable": 119849} +{"epoch": 1.02734375, "grad_norm": 0.17198698222637177, "learning_rate": 5.688633799118971e-05, "loss": 0.0011616486590355635, "memory/device_reserved (GiB)": 41.75, "memory/max_active (GiB)": 36.78, "memory/max_allocated (GiB)": 36.78, "ppl": 1.00116, "step": 263, "tokens/total": 9132080, "tokens/train_per_sec_per_gpu": 29.21, "tokens/trainable": 120319} +{"epoch": 1.03125, "grad_norm": 1.829367995262146, "learning_rate": 5.659626500889066e-05, "loss": 0.0026817461475729942, "memory/device_reserved (GiB)": 41.75, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.00269, "step": 264, "tokens/total": 9165072, "tokens/train_per_sec_per_gpu": 31.05, "tokens/trainable": 120804} +{"epoch": 1.03515625, "grad_norm": 0.018209626898169518, "learning_rate": 5.6306125599488905e-05, "loss": 0.0001134154008468613, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 35.9, "memory/max_allocated (GiB)": 35.9, "ppl": 1.00011, "step": 265, "tokens/total": 9195776, "tokens/train_per_sec_per_gpu": 29.43, "tokens/trainable": 121233} +{"epoch": 1.0390625, "grad_norm": 0.733806312084198, "learning_rate": 5.601593183686955e-05, "loss": 0.007640395313501358, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.95, "memory/max_allocated (GiB)": 37.95, "ppl": 1.00767, "step": 266, "tokens/total": 9231136, "tokens/train_per_sec_per_gpu": 22.62, "tokens/trainable": 121671} +{"epoch": 1.04296875, "grad_norm": 0.07811731845140457, "learning_rate": 5.572569579717961e-05, "loss": 0.00044431095011532307, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00044, "step": 267, "tokens/total": 9266784, "tokens/train_per_sec_per_gpu": 25.77, "tokens/trainable": 122141} +{"epoch": 1.046875, "grad_norm": 1.1215806007385254, "learning_rate": 5.543542955832538e-05, "loss": 0.0158391073346138, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.37, "memory/max_allocated (GiB)": 38.37, "ppl": 1.01597, "step": 268, "tokens/total": 9301648, "tokens/train_per_sec_per_gpu": 25.18, "tokens/trainable": 122556} +{"epoch": 1.05078125, "grad_norm": 1.3674460649490356, "learning_rate": 5.514514519946986e-05, "loss": 0.009398038499057293, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.00944, "step": 269, "tokens/total": 9334352, "tokens/train_per_sec_per_gpu": 27.41, "tokens/trainable": 122988} +{"epoch": 1.0546875, "grad_norm": 0.04491202533245087, "learning_rate": 5.485485480053015e-05, "loss": 0.00018811026529874653, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.00019, "step": 270, "tokens/total": 9369376, "tokens/train_per_sec_per_gpu": 27.04, "tokens/trainable": 123478} +{"epoch": 1.05859375, "grad_norm": 0.03429289534687996, "learning_rate": 5.4564570441674645e-05, "loss": 0.00011189555516466498, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.00011, "step": 271, "tokens/total": 9402928, "tokens/train_per_sec_per_gpu": 28.91, "tokens/trainable": 123932} +{"epoch": 1.0625, "grad_norm": 0.32341447472572327, "learning_rate": 5.42743042028204e-05, "loss": 0.0003135955485049635, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.00031, "step": 272, "tokens/total": 9437952, "tokens/train_per_sec_per_gpu": 29.6, "tokens/trainable": 124428} +{"epoch": 1.06640625, "grad_norm": 0.16611970961093903, "learning_rate": 5.3984068163130464e-05, "loss": 0.0008581056026741862, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.00086, "step": 273, "tokens/total": 9471648, "tokens/train_per_sec_per_gpu": 23.03, "tokens/trainable": 124866} +{"epoch": 1.0703125, "grad_norm": 1.6726605892181396, "learning_rate": 5.369387440051111e-05, "loss": 0.0012247010599821806, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.00123, "step": 274, "tokens/total": 9508640, "tokens/train_per_sec_per_gpu": 24.69, "tokens/trainable": 125324} +{"epoch": 1.07421875, "grad_norm": 1.3945966958999634, "learning_rate": 5.340373499110935e-05, "loss": 0.00336440559476614, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.00337, "step": 275, "tokens/total": 9548560, "tokens/train_per_sec_per_gpu": 24.22, "tokens/trainable": 125813} +{"epoch": 1.078125, "grad_norm": 0.01825297623872757, "learning_rate": 5.3113662008810304e-05, "loss": 0.00021560768072959036, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00022, "step": 276, "tokens/total": 9584608, "tokens/train_per_sec_per_gpu": 22.32, "tokens/trainable": 126252} +{"epoch": 1.08203125, "grad_norm": 0.06868293881416321, "learning_rate": 5.282366752473479e-05, "loss": 0.0006395367090590298, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.99, "memory/max_allocated (GiB)": 38.99, "ppl": 1.00064, "step": 277, "tokens/total": 9620656, "tokens/train_per_sec_per_gpu": 25.06, "tokens/trainable": 126696} +{"epoch": 1.0859375, "grad_norm": 0.9091266989707947, "learning_rate": 5.2533763606737005e-05, "loss": 0.012599224224686623, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.01268, "step": 278, "tokens/total": 9652992, "tokens/train_per_sec_per_gpu": 28.8, "tokens/trainable": 127173} +{"epoch": 1.08984375, "grad_norm": 0.5939775109291077, "learning_rate": 5.224396231890232e-05, "loss": 0.0023771331179887056, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.00238, "step": 279, "tokens/total": 9685632, "tokens/train_per_sec_per_gpu": 27.93, "tokens/trainable": 127637} +{"epoch": 1.09375, "grad_norm": 0.04779400676488876, "learning_rate": 5.195427572104522e-05, "loss": 0.0004010523553006351, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.59, "memory/max_allocated (GiB)": 38.59, "ppl": 1.0004, "step": 280, "tokens/total": 9723056, "tokens/train_per_sec_per_gpu": 24.94, "tokens/trainable": 128112} +{"epoch": 1.09765625, "grad_norm": 0.03449466452002525, "learning_rate": 5.166471586820751e-05, "loss": 0.00018891086801886559, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.73, "memory/max_allocated (GiB)": 37.73, "ppl": 1.00019, "step": 281, "tokens/total": 9758592, "tokens/train_per_sec_per_gpu": 26.36, "tokens/trainable": 128584} +{"epoch": 1.1015625, "grad_norm": 0.08411771059036255, "learning_rate": 5.1375294810156615e-05, "loss": 0.0003892357926815748, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.3, "memory/max_allocated (GiB)": 38.3, "ppl": 1.00039, "step": 282, "tokens/total": 9793408, "tokens/train_per_sec_per_gpu": 24.79, "tokens/trainable": 129051} +{"epoch": 1.10546875, "grad_norm": 0.06278375536203384, "learning_rate": 5.1086024590884144e-05, "loss": 0.00023487892758566886, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.15, "memory/max_allocated (GiB)": 37.15, "ppl": 1.00023, "step": 283, "tokens/total": 9827216, "tokens/train_per_sec_per_gpu": 25.6, "tokens/trainable": 129494} +{"epoch": 1.109375, "grad_norm": 0.042560603469610214, "learning_rate": 5.079691724810461e-05, "loss": 0.00010453620052430779, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 39.35, "memory/max_allocated (GiB)": 39.35, "ppl": 1.0001, "step": 284, "tokens/total": 9864928, "tokens/train_per_sec_per_gpu": 24.77, "tokens/trainable": 129983} +{"epoch": 1.11328125, "grad_norm": 0.24027064442634583, "learning_rate": 5.0507984812754684e-05, "loss": 0.0007409818354062736, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00074, "step": 285, "tokens/total": 9901520, "tokens/train_per_sec_per_gpu": 23.6, "tokens/trainable": 130405} +{"epoch": 1.1171875, "grad_norm": 0.20643122494220734, "learning_rate": 5.021923930849237e-05, "loss": 0.0016999109648168087, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.56, "memory/max_allocated (GiB)": 37.56, "ppl": 1.0017, "step": 286, "tokens/total": 9935824, "tokens/train_per_sec_per_gpu": 24.75, "tokens/trainable": 130829} +{"epoch": 1.12109375, "grad_norm": 0.5026089549064636, "learning_rate": 4.99306927511967e-05, "loss": 0.001153714256361127, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.00115, "step": 287, "tokens/total": 9969200, "tokens/train_per_sec_per_gpu": 24.53, "tokens/trainable": 131287} +{"epoch": 1.125, "grad_norm": 0.09911680966615677, "learning_rate": 4.964235714846775e-05, "loss": 0.0005808798014186323, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.00058, "step": 288, "tokens/total": 10005120, "tokens/train_per_sec_per_gpu": 23.64, "tokens/trainable": 131712} +{"epoch": 1.12890625, "grad_norm": 0.6661036610603333, "learning_rate": 4.9354244499126866e-05, "loss": 0.01766769215464592, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.01782, "step": 289, "tokens/total": 10040336, "tokens/train_per_sec_per_gpu": 25.36, "tokens/trainable": 132168} +{"epoch": 1.1328125, "grad_norm": 0.15597105026245117, "learning_rate": 4.90663667927174e-05, "loss": 0.00029858326888643205, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.0003, "step": 290, "tokens/total": 10075232, "tokens/train_per_sec_per_gpu": 27.22, "tokens/trainable": 132626} +{"epoch": 1.13671875, "grad_norm": 0.3456335663795471, "learning_rate": 4.877873600900581e-05, "loss": 0.004322251304984093, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.00433, "step": 291, "tokens/total": 10113632, "tokens/train_per_sec_per_gpu": 24.27, "tokens/trainable": 133119} +{"epoch": 1.140625, "grad_norm": 0.4922923147678375, "learning_rate": 4.849136411748306e-05, "loss": 0.002877101767808199, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 36.96, "memory/max_allocated (GiB)": 36.96, "ppl": 1.00288, "step": 292, "tokens/total": 10145728, "tokens/train_per_sec_per_gpu": 27.32, "tokens/trainable": 133550} +{"epoch": 1.14453125, "grad_norm": 1.101583480834961, "learning_rate": 4.8204263076866574e-05, "loss": 0.00022544774401467294, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.39, "memory/max_allocated (GiB)": 38.39, "ppl": 1.00023, "step": 293, "tokens/total": 10182512, "tokens/train_per_sec_per_gpu": 24.9, "tokens/trainable": 133992} +{"epoch": 1.1484375, "grad_norm": 0.27507808804512024, "learning_rate": 4.791744483460251e-05, "loss": 0.0021713951136916876, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.54, "memory/max_allocated (GiB)": 39.54, "ppl": 1.00217, "step": 294, "tokens/total": 10218144, "tokens/train_per_sec_per_gpu": 28.9, "tokens/trainable": 134467} +{"epoch": 1.15234375, "grad_norm": 0.0889352411031723, "learning_rate": 4.7630921326368736e-05, "loss": 0.0006347870803438127, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.00063, "step": 295, "tokens/total": 10256160, "tokens/train_per_sec_per_gpu": 20.2, "tokens/trainable": 134895} +{"epoch": 1.15625, "grad_norm": 0.5234104990959167, "learning_rate": 4.7344704475577916e-05, "loss": 0.005218126345425844, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 35.47, "memory/max_allocated (GiB)": 35.47, "ppl": 1.00523, "step": 296, "tokens/total": 10285136, "tokens/train_per_sec_per_gpu": 31.11, "tokens/trainable": 135321} +{"epoch": 1.16015625, "grad_norm": 1.0139681100845337, "learning_rate": 4.705880619288153e-05, "loss": 0.006263894494622946, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.00628, "step": 297, "tokens/total": 10321424, "tokens/train_per_sec_per_gpu": 23.87, "tokens/trainable": 135765} +{"epoch": 1.1640625, "grad_norm": 0.0771324634552002, "learning_rate": 4.677323837567412e-05, "loss": 0.00032971566542983055, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.00033, "step": 298, "tokens/total": 10359248, "tokens/train_per_sec_per_gpu": 24.86, "tokens/trainable": 136196} +{"epoch": 1.16796875, "grad_norm": 0.048721395432949066, "learning_rate": 4.6488012907598146e-05, "loss": 0.00047704510507173836, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.81, "memory/max_allocated (GiB)": 38.81, "ppl": 1.00048, "step": 299, "tokens/total": 10395824, "tokens/train_per_sec_per_gpu": 24.57, "tokens/trainable": 136649} +{"epoch": 1.171875, "grad_norm": 0.3027370870113373, "learning_rate": 4.620314165804964e-05, "loss": 0.001021591480821371, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00102, "step": 300, "tokens/total": 10432864, "tokens/train_per_sec_per_gpu": 26.81, "tokens/trainable": 137128} +{"epoch": 1.17578125, "grad_norm": 0.40727365016937256, "learning_rate": 4.591863648168407e-05, "loss": 0.005430878140032291, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.04, "memory/max_allocated (GiB)": 37.04, "ppl": 1.00545, "step": 301, "tokens/total": 10466816, "tokens/train_per_sec_per_gpu": 30.59, "tokens/trainable": 137585} +{"epoch": 1.1796875, "grad_norm": 0.022467099130153656, "learning_rate": 4.5634509217923135e-05, "loss": 0.0001953087921719998, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.0002, "step": 302, "tokens/total": 10499280, "tokens/train_per_sec_per_gpu": 31.43, "tokens/trainable": 138054} +{"epoch": 1.18359375, "grad_norm": 0.5985379815101624, "learning_rate": 4.535077169046201e-05, "loss": 0.01152348518371582, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.0, "memory/max_allocated (GiB)": 38.0, "ppl": 1.01159, "step": 303, "tokens/total": 10534112, "tokens/train_per_sec_per_gpu": 28.86, "tokens/trainable": 138559} +{"epoch": 1.1875, "grad_norm": 0.16078093647956848, "learning_rate": 4.506743570677743e-05, "loss": 0.001145296497270465, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.53, "memory/max_allocated (GiB)": 37.53, "ppl": 1.00115, "step": 304, "tokens/total": 10568912, "tokens/train_per_sec_per_gpu": 24.44, "tokens/trainable": 139050} +{"epoch": 1.19140625, "grad_norm": 0.08465650677680969, "learning_rate": 4.478451305763618e-05, "loss": 0.0006044696201570332, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.0006, "step": 305, "tokens/total": 10606128, "tokens/train_per_sec_per_gpu": 24.96, "tokens/trainable": 139525} +{"epoch": 1.1953125, "grad_norm": 0.02060011588037014, "learning_rate": 4.450201551660454e-05, "loss": 0.0002862655674107373, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.08, "memory/max_allocated (GiB)": 39.08, "ppl": 1.00029, "step": 306, "tokens/total": 10639584, "tokens/train_per_sec_per_gpu": 26.74, "tokens/trainable": 139969} +{"epoch": 1.19921875, "grad_norm": 0.4725620746612549, "learning_rate": 4.4219954839558276e-05, "loss": 0.0015851769130676985, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.44, "memory/max_allocated (GiB)": 36.44, "ppl": 1.00159, "step": 307, "tokens/total": 10670928, "tokens/train_per_sec_per_gpu": 31.77, "tokens/trainable": 140443} +{"epoch": 1.203125, "grad_norm": 0.041435133665800095, "learning_rate": 4.393834276419352e-05, "loss": 8.089239418040961e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.00008, "step": 308, "tokens/total": 10709968, "tokens/train_per_sec_per_gpu": 24.11, "tokens/trainable": 140926} +{"epoch": 1.20703125, "grad_norm": 0.06269388645887375, "learning_rate": 4.36571910095382e-05, "loss": 0.0004129243898205459, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.00041, "step": 309, "tokens/total": 10744800, "tokens/train_per_sec_per_gpu": 24.34, "tokens/trainable": 141344} +{"epoch": 1.2109375, "grad_norm": 0.06209995225071907, "learning_rate": 4.337651127546448e-05, "loss": 0.0004497283953242004, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.00045, "step": 310, "tokens/total": 10778576, "tokens/train_per_sec_per_gpu": 26.35, "tokens/trainable": 141753} +{"epoch": 1.21484375, "grad_norm": 0.013691958039999008, "learning_rate": 4.3096315242201736e-05, "loss": 0.00018389936303719878, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.61, "memory/max_allocated (GiB)": 36.61, "ppl": 1.00018, "step": 311, "tokens/total": 10810816, "tokens/train_per_sec_per_gpu": 31.44, "tokens/trainable": 142219} +{"epoch": 1.21875, "grad_norm": 0.1537526696920395, "learning_rate": 4.2816614569850635e-05, "loss": 0.001505336258560419, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.01, "memory/max_allocated (GiB)": 36.01, "ppl": 1.00151, "step": 312, "tokens/total": 10841040, "tokens/train_per_sec_per_gpu": 25.5, "tokens/trainable": 142617} +{"epoch": 1.22265625, "grad_norm": 0.6085811257362366, "learning_rate": 4.2537420897897864e-05, "loss": 0.006457360461354256, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.00648, "step": 313, "tokens/total": 10880960, "tokens/train_per_sec_per_gpu": 21.5, "tokens/trainable": 143103} +{"epoch": 1.2265625, "grad_norm": 0.01439083181321621, "learning_rate": 4.225874584473174e-05, "loss": 0.00011710141552612185, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00012, "step": 314, "tokens/total": 10913472, "tokens/train_per_sec_per_gpu": 25.34, "tokens/trainable": 143535} +{"epoch": 1.23046875, "grad_norm": 0.027507653459906578, "learning_rate": 4.19806010071587e-05, "loss": 0.00042320965440012515, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00042, "step": 315, "tokens/total": 10951360, "tokens/train_per_sec_per_gpu": 23.95, "tokens/trainable": 144009} +{"epoch": 1.234375, "grad_norm": 0.43742796778678894, "learning_rate": 4.170299795992081e-05, "loss": 0.0012267068959772587, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.00123, "step": 316, "tokens/total": 10987808, "tokens/train_per_sec_per_gpu": 23.37, "tokens/trainable": 144460} +{"epoch": 1.23828125, "grad_norm": 0.024323808029294014, "learning_rate": 4.142594825521398e-05, "loss": 0.00026626349426805973, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.00027, "step": 317, "tokens/total": 11025552, "tokens/train_per_sec_per_gpu": 24.7, "tokens/trainable": 144936} +{"epoch": 1.2421875, "grad_norm": 0.061545081436634064, "learning_rate": 4.114946342220728e-05, "loss": 0.00045223772758617997, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.00045, "step": 318, "tokens/total": 11060432, "tokens/train_per_sec_per_gpu": 25.24, "tokens/trainable": 145375} +{"epoch": 1.24609375, "grad_norm": 0.004041018430143595, "learning_rate": 4.087355496656321e-05, "loss": 6.310529715847224e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.00006, "step": 319, "tokens/total": 11094688, "tokens/train_per_sec_per_gpu": 25.62, "tokens/trainable": 145849} +{"epoch": 1.25, "grad_norm": 0.02695886604487896, "learning_rate": 4.05982343699588e-05, "loss": 0.0002085692249238491, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00021, "step": 320, "tokens/total": 11127408, "tokens/train_per_sec_per_gpu": 27.02, "tokens/trainable": 146318} +{"epoch": 1.25390625, "grad_norm": 0.011690028011798859, "learning_rate": 4.0323513089607876e-05, "loss": 7.729644858045503e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.72, "memory/max_allocated (GiB)": 38.72, "ppl": 1.00008, "step": 321, "tokens/total": 11162752, "tokens/train_per_sec_per_gpu": 26.73, "tokens/trainable": 146821} +{"epoch": 1.2578125, "grad_norm": 0.09426535665988922, "learning_rate": 4.004940255778431e-05, "loss": 0.0007540143560618162, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.33, "memory/max_allocated (GiB)": 36.33, "ppl": 1.00075, "step": 322, "tokens/total": 11194576, "tokens/train_per_sec_per_gpu": 26.47, "tokens/trainable": 147285} +{"epoch": 1.26171875, "grad_norm": 0.014529458247125149, "learning_rate": 3.977591418134619e-05, "loss": 0.00012375191727187485, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.61, "memory/max_allocated (GiB)": 36.61, "ppl": 1.00012, "step": 323, "tokens/total": 11226928, "tokens/train_per_sec_per_gpu": 25.72, "tokens/trainable": 147729} +{"epoch": 1.265625, "grad_norm": 0.5806203484535217, "learning_rate": 3.95030593412612e-05, "loss": 0.006746239028871059, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.00677, "step": 324, "tokens/total": 11266016, "tokens/train_per_sec_per_gpu": 20.62, "tokens/trainable": 148169} +{"epoch": 1.26953125, "grad_norm": 0.012494254857301712, "learning_rate": 3.923084939213296e-05, "loss": 8.885329589247704e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.00009, "step": 325, "tokens/total": 11303264, "tokens/train_per_sec_per_gpu": 24.09, "tokens/trainable": 148630} +{"epoch": 1.2734375, "grad_norm": 0.09252164512872696, "learning_rate": 3.895929566172861e-05, "loss": 0.00027632468845695257, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00028, "step": 326, "tokens/total": 11339392, "tokens/train_per_sec_per_gpu": 22.63, "tokens/trainable": 149063} +{"epoch": 1.27734375, "grad_norm": 0.0077980454079806805, "learning_rate": 3.868840945050728e-05, "loss": 2.7549860533326864e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.00003, "step": 327, "tokens/total": 11377568, "tokens/train_per_sec_per_gpu": 23.2, "tokens/trainable": 149549} +{"epoch": 1.28125, "grad_norm": 0.40985268354415894, "learning_rate": 3.841820203114995e-05, "loss": 0.0035989931784570217, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.03, "memory/max_allocated (GiB)": 38.03, "ppl": 1.00361, "step": 328, "tokens/total": 11413184, "tokens/train_per_sec_per_gpu": 26.82, "tokens/trainable": 150015} +{"epoch": 1.28515625, "grad_norm": 0.0023129559122025967, "learning_rate": 3.814868464809027e-05, "loss": 1.6900576156331226e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.00002, "step": 329, "tokens/total": 11447904, "tokens/train_per_sec_per_gpu": 28.13, "tokens/trainable": 150514} +{"epoch": 1.2890625, "grad_norm": 0.4119769036769867, "learning_rate": 3.787986851704667e-05, "loss": 0.0055192094296216965, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.00553, "step": 330, "tokens/total": 11485056, "tokens/train_per_sec_per_gpu": 27.55, "tokens/trainable": 151021} +{"epoch": 1.29296875, "grad_norm": 0.06764540821313858, "learning_rate": 3.7611764824555654e-05, "loss": 0.00035793331335298717, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.37, "memory/max_allocated (GiB)": 38.37, "ppl": 1.00036, "step": 331, "tokens/total": 11518848, "tokens/train_per_sec_per_gpu": 29.86, "tokens/trainable": 151486} +{"epoch": 1.296875, "grad_norm": 0.016952766105532646, "learning_rate": 3.734438472750619e-05, "loss": 0.00010818655573530123, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.79, "memory/max_allocated (GiB)": 36.79, "ppl": 1.00011, "step": 332, "tokens/total": 11552048, "tokens/train_per_sec_per_gpu": 26.42, "tokens/trainable": 151956} +{"epoch": 1.30078125, "grad_norm": 0.2057858258485794, "learning_rate": 3.707773935267552e-05, "loss": 0.0018884800374507904, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.15, "memory/max_allocated (GiB)": 39.15, "ppl": 1.00189, "step": 333, "tokens/total": 11589168, "tokens/train_per_sec_per_gpu": 24.91, "tokens/trainable": 152442} +{"epoch": 1.3046875, "grad_norm": 0.0377814844250679, "learning_rate": 3.68118397962661e-05, "loss": 0.00014338323671836406, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.74, "memory/max_allocated (GiB)": 36.74, "ppl": 1.00014, "step": 334, "tokens/total": 11622032, "tokens/train_per_sec_per_gpu": 25.87, "tokens/trainable": 152890} +{"epoch": 1.30859375, "grad_norm": 1.339460015296936, "learning_rate": 3.654669712344384e-05, "loss": 0.011203598231077194, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.35, "memory/max_allocated (GiB)": 36.35, "ppl": 1.01127, "step": 335, "tokens/total": 11653840, "tokens/train_per_sec_per_gpu": 30.88, "tokens/trainable": 153368} +{"epoch": 1.3125, "grad_norm": 0.01062884833663702, "learning_rate": 3.628232236787763e-05, "loss": 0.00011677895236061886, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00012, "step": 336, "tokens/total": 11689664, "tokens/train_per_sec_per_gpu": 26.44, "tokens/trainable": 153809} +{"epoch": 1.31640625, "grad_norm": 0.31619954109191895, "learning_rate": 3.6018726531280144e-05, "loss": 0.0019477647729218006, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.06, "memory/max_allocated (GiB)": 37.06, "ppl": 1.00195, "step": 337, "tokens/total": 11723728, "tokens/train_per_sec_per_gpu": 25.88, "tokens/trainable": 154258} +{"epoch": 1.3203125, "grad_norm": 0.15699031949043274, "learning_rate": 3.575592058295017e-05, "loss": 0.0005229224334470928, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.00052, "step": 338, "tokens/total": 11758960, "tokens/train_per_sec_per_gpu": 25.05, "tokens/trainable": 154704} +{"epoch": 1.32421875, "grad_norm": 0.14500632882118225, "learning_rate": 3.549391545931585e-05, "loss": 0.0012588140089064837, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.00126, "step": 339, "tokens/total": 11797328, "tokens/train_per_sec_per_gpu": 24.06, "tokens/trainable": 155157} +{"epoch": 1.328125, "grad_norm": 0.7249783873558044, "learning_rate": 3.5232722063479914e-05, "loss": 0.0029754401184618473, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.07, "memory/max_allocated (GiB)": 38.07, "ppl": 1.00298, "step": 340, "tokens/total": 11832656, "tokens/train_per_sec_per_gpu": 23.78, "tokens/trainable": 155582} +{"epoch": 1.33203125, "grad_norm": 0.0023078930098563433, "learning_rate": 3.49723512647657e-05, "loss": 4.3637694034259766e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.00004, "step": 341, "tokens/total": 11865152, "tokens/train_per_sec_per_gpu": 28.18, "tokens/trainable": 156047} +{"epoch": 1.3359375, "grad_norm": 0.005330865737050772, "learning_rate": 3.471281389826491e-05, "loss": 5.552058428293094e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.99, "memory/max_allocated (GiB)": 38.99, "ppl": 1.00006, "step": 342, "tokens/total": 11901552, "tokens/train_per_sec_per_gpu": 25.89, "tokens/trainable": 156500} +{"epoch": 1.33984375, "grad_norm": 0.2803260087966919, "learning_rate": 3.4454120764386764e-05, "loss": 0.0019933385774493217, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.002, "step": 343, "tokens/total": 11937328, "tokens/train_per_sec_per_gpu": 24.98, "tokens/trainable": 156931} +{"epoch": 1.34375, "grad_norm": 0.48158031702041626, "learning_rate": 3.4196282628408526e-05, "loss": 0.005864639300853014, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00588, "step": 344, "tokens/total": 11970736, "tokens/train_per_sec_per_gpu": 25.71, "tokens/trainable": 157385} +{"epoch": 1.34765625, "grad_norm": 0.004094376228749752, "learning_rate": 3.3939310220027456e-05, "loss": 3.841984289465472e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.55, "memory/max_allocated (GiB)": 39.55, "ppl": 1.00004, "step": 345, "tokens/total": 12008096, "tokens/train_per_sec_per_gpu": 22.96, "tokens/trainable": 157817} +{"epoch": 1.3515625, "grad_norm": 0.18125192821025848, "learning_rate": 3.3683214232914404e-05, "loss": 0.0005042756674811244, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.0005, "step": 346, "tokens/total": 12039904, "tokens/train_per_sec_per_gpu": 26.58, "tokens/trainable": 158273} +{"epoch": 1.35546875, "grad_norm": 0.046786822378635406, "learning_rate": 3.342800532426873e-05, "loss": 0.0002317998296348378, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.2, "memory/max_allocated (GiB)": 37.2, "ppl": 1.00023, "step": 347, "tokens/total": 12074416, "tokens/train_per_sec_per_gpu": 26.21, "tokens/trainable": 158728} +{"epoch": 1.359375, "grad_norm": 0.001738823251798749, "learning_rate": 3.317369411437484e-05, "loss": 1.7130794731201604e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.12, "memory/max_allocated (GiB)": 37.12, "ppl": 1.00002, "step": 348, "tokens/total": 12108432, "tokens/train_per_sec_per_gpu": 24.67, "tokens/trainable": 159168} +{"epoch": 1.36328125, "grad_norm": 0.02952537126839161, "learning_rate": 3.292029118616024e-05, "loss": 0.00015957941650412977, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.00016, "step": 349, "tokens/total": 12141792, "tokens/train_per_sec_per_gpu": 23.33, "tokens/trainable": 159592} +{"epoch": 1.3671875, "grad_norm": 0.010363176465034485, "learning_rate": 3.266780708475511e-05, "loss": 9.464132745051757e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.00009, "step": 350, "tokens/total": 12175840, "tokens/train_per_sec_per_gpu": 29.6, "tokens/trainable": 160093} +{"epoch": 1.37109375, "grad_norm": 0.0042407456785440445, "learning_rate": 3.241625231705354e-05, "loss": 6.655165634583682e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.56, "memory/max_allocated (GiB)": 37.56, "ppl": 1.00007, "step": 351, "tokens/total": 12208720, "tokens/train_per_sec_per_gpu": 25.91, "tokens/trainable": 160534} +{"epoch": 1.375, "grad_norm": 0.530038058757782, "learning_rate": 3.216563735127618e-05, "loss": 0.0035791490226984024, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.00359, "step": 352, "tokens/total": 12246336, "tokens/train_per_sec_per_gpu": 24.54, "tokens/trainable": 161009} +{"epoch": 1.37890625, "grad_norm": 0.4886930286884308, "learning_rate": 3.191597261653475e-05, "loss": 0.010051025077700615, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.0101, "step": 353, "tokens/total": 12279216, "tokens/train_per_sec_per_gpu": 28.01, "tokens/trainable": 161450} +{"epoch": 1.3828125, "grad_norm": 0.10105793178081512, "learning_rate": 3.166726850239794e-05, "loss": 0.0006613415898755193, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00066, "step": 354, "tokens/total": 12311936, "tokens/train_per_sec_per_gpu": 30.64, "tokens/trainable": 161900} +{"epoch": 1.38671875, "grad_norm": 0.006541822571307421, "learning_rate": 3.141953535845912e-05, "loss": 6.174531154101714e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.84, "memory/max_allocated (GiB)": 37.84, "ppl": 1.00006, "step": 355, "tokens/total": 12347136, "tokens/train_per_sec_per_gpu": 28.73, "tokens/trainable": 162358} +{"epoch": 1.390625, "grad_norm": 0.006567712873220444, "learning_rate": 3.11727834939056e-05, "loss": 4.647376772481948e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.53, "memory/max_allocated (GiB)": 37.53, "ppl": 1.00005, "step": 356, "tokens/total": 12380704, "tokens/train_per_sec_per_gpu": 27.91, "tokens/trainable": 162845} +{"epoch": 1.39453125, "grad_norm": 0.1909540891647339, "learning_rate": 3.092702317708967e-05, "loss": 0.001767120324075222, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.84, "memory/max_allocated (GiB)": 37.84, "ppl": 1.00177, "step": 357, "tokens/total": 12415744, "tokens/train_per_sec_per_gpu": 25.21, "tokens/trainable": 163299} +{"epoch": 1.3984375, "grad_norm": 0.6032511591911316, "learning_rate": 3.0682264635101276e-05, "loss": 0.0028830943629145622, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 1.00289, "step": 358, "tokens/total": 12450400, "tokens/train_per_sec_per_gpu": 27.49, "tokens/trainable": 163774} +{"epoch": 1.40234375, "grad_norm": 0.0308778565376997, "learning_rate": 3.0438518053342407e-05, "loss": 0.00018781640392262489, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.00019, "step": 359, "tokens/total": 12482976, "tokens/train_per_sec_per_gpu": 24.5, "tokens/trainable": 164199} +{"epoch": 1.40625, "grad_norm": 0.5054932832717896, "learning_rate": 3.0195793575103266e-05, "loss": 0.009806083515286446, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.1, "memory/max_allocated (GiB)": 36.1, "ppl": 1.00985, "step": 360, "tokens/total": 12513392, "tokens/train_per_sec_per_gpu": 31.47, "tokens/trainable": 164667} +{"epoch": 1.41015625, "grad_norm": 0.5414286255836487, "learning_rate": 2.9954101301140146e-05, "loss": 0.0054559530690312386, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.91, "memory/max_allocated (GiB)": 38.91, "ppl": 1.00547, "step": 361, "tokens/total": 12552528, "tokens/train_per_sec_per_gpu": 22.17, "tokens/trainable": 165111} +{"epoch": 1.4140625, "grad_norm": 0.1931993067264557, "learning_rate": 2.9713451289255123e-05, "loss": 0.00029955667559988797, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.07, "memory/max_allocated (GiB)": 37.07, "ppl": 1.0003, "step": 362, "tokens/total": 12584704, "tokens/train_per_sec_per_gpu": 30.57, "tokens/trainable": 165575} +{"epoch": 1.41796875, "grad_norm": 0.7020666003227234, "learning_rate": 2.9473853553877484e-05, "loss": 0.003438273910433054, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.00344, "step": 363, "tokens/total": 12619904, "tokens/train_per_sec_per_gpu": 26.5, "tokens/trainable": 166060} +{"epoch": 1.421875, "grad_norm": 0.011477800086140633, "learning_rate": 2.9235318065647e-05, "loss": 8.650364179629833e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00009, "step": 364, "tokens/total": 12655152, "tokens/train_per_sec_per_gpu": 23.69, "tokens/trainable": 166478} +{"epoch": 1.42578125, "grad_norm": 1.4940345287322998, "learning_rate": 2.8997854750998964e-05, "loss": 0.013449061661958694, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.01354, "step": 365, "tokens/total": 12691888, "tokens/train_per_sec_per_gpu": 26.51, "tokens/trainable": 166941} +{"epoch": 1.4296875, "grad_norm": 0.0535619854927063, "learning_rate": 2.8761473491751258e-05, "loss": 0.000146179721923545, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.77, "memory/max_allocated (GiB)": 36.77, "ppl": 1.00015, "step": 366, "tokens/total": 12724400, "tokens/train_per_sec_per_gpu": 28.43, "tokens/trainable": 167381} +{"epoch": 1.43359375, "grad_norm": 0.061190079897642136, "learning_rate": 2.8526184124692883e-05, "loss": 0.0003008122439496219, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.0003, "step": 367, "tokens/total": 12757664, "tokens/train_per_sec_per_gpu": 27.34, "tokens/trainable": 167830} +{"epoch": 1.4375, "grad_norm": 0.022105291485786438, "learning_rate": 2.829199644117484e-05, "loss": 0.00015356129733845592, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.00015, "step": 368, "tokens/total": 12793680, "tokens/train_per_sec_per_gpu": 23.34, "tokens/trainable": 168266} +{"epoch": 1.44140625, "grad_norm": 0.08455733954906464, "learning_rate": 2.8058920186702553e-05, "loss": 0.00039900760748423636, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.0004, "step": 369, "tokens/total": 12828400, "tokens/train_per_sec_per_gpu": 26.73, "tokens/trainable": 168716} +{"epoch": 1.4453125, "grad_norm": 0.15636853873729706, "learning_rate": 2.782696506053033e-05, "loss": 0.0006747484439983964, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00067, "step": 370, "tokens/total": 12864752, "tokens/train_per_sec_per_gpu": 25.85, "tokens/trainable": 169176} +{"epoch": 1.44921875, "grad_norm": 0.9633499979972839, "learning_rate": 2.7596140715257824e-05, "loss": 0.012005692347884178, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.12, "memory/max_allocated (GiB)": 39.12, "ppl": 1.01208, "step": 371, "tokens/total": 12900768, "tokens/train_per_sec_per_gpu": 22.16, "tokens/trainable": 169591} +{"epoch": 1.453125, "grad_norm": 0.07865497469902039, "learning_rate": 2.7366456756428184e-05, "loss": 0.0005621974123641849, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.59, "memory/max_allocated (GiB)": 37.59, "ppl": 1.00056, "step": 372, "tokens/total": 12935456, "tokens/train_per_sec_per_gpu": 26.35, "tokens/trainable": 170041} +{"epoch": 1.45703125, "grad_norm": 0.1599891483783722, "learning_rate": 2.7137922742128486e-05, "loss": 0.0010160183301195502, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.00102, "step": 373, "tokens/total": 12968448, "tokens/train_per_sec_per_gpu": 30.34, "tokens/trainable": 170516} +{"epoch": 1.4609375, "grad_norm": 0.3118171691894531, "learning_rate": 2.691054818259188e-05, "loss": 0.0012614354491233826, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.73, "memory/max_allocated (GiB)": 37.73, "ppl": 1.00126, "step": 374, "tokens/total": 13003904, "tokens/train_per_sec_per_gpu": 24.61, "tokens/trainable": 170952} +{"epoch": 1.46484375, "grad_norm": 0.5669094324111938, "learning_rate": 2.6684342539801933e-05, "loss": 0.0075299725867807865, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.00756, "step": 375, "tokens/total": 13040864, "tokens/train_per_sec_per_gpu": 26.09, "tokens/trainable": 171424} +{"epoch": 1.46875, "grad_norm": 0.010177488438785076, "learning_rate": 2.645931522709877e-05, "loss": 0.00012544373748824, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.00013, "step": 376, "tokens/total": 13079504, "tokens/train_per_sec_per_gpu": 21.7, "tokens/trainable": 171884} +{"epoch": 1.47265625, "grad_norm": 0.003876109141856432, "learning_rate": 2.6235475608787365e-05, "loss": 7.81391718192026e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00008, "step": 377, "tokens/total": 13115024, "tokens/train_per_sec_per_gpu": 28.08, "tokens/trainable": 172404} +{"epoch": 1.4765625, "grad_norm": 1.0932646989822388, "learning_rate": 2.6012832999747916e-05, "loss": 0.0007629136089235544, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00076, "step": 378, "tokens/total": 13152176, "tokens/train_per_sec_per_gpu": 25.29, "tokens/trainable": 172871} +{"epoch": 1.48046875, "grad_norm": 0.21459592878818512, "learning_rate": 2.579139666504821e-05, "loss": 0.0031334070954471827, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00314, "step": 379, "tokens/total": 13184896, "tokens/train_per_sec_per_gpu": 25.86, "tokens/trainable": 173329} +{"epoch": 1.484375, "grad_norm": 0.17172880470752716, "learning_rate": 2.557117581955798e-05, "loss": 0.0013926737010478973, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.00139, "step": 380, "tokens/total": 13220240, "tokens/train_per_sec_per_gpu": 26.55, "tokens/trainable": 173827} +{"epoch": 1.48828125, "grad_norm": 0.046759385615587234, "learning_rate": 2.5352179627565532e-05, "loss": 0.0004966439446434379, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.0005, "step": 381, "tokens/total": 13259808, "tokens/train_per_sec_per_gpu": 23.58, "tokens/trainable": 174287} +{"epoch": 1.4921875, "grad_norm": 0.03143330663442612, "learning_rate": 2.5134417202396277e-05, "loss": 0.0004258331027813256, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00043, "step": 382, "tokens/total": 13295776, "tokens/train_per_sec_per_gpu": 25.5, "tokens/trainable": 174731} +{"epoch": 1.49609375, "grad_norm": 0.013699019327759743, "learning_rate": 2.491789760603361e-05, "loss": 0.00023711490212008357, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00024, "step": 383, "tokens/total": 13331264, "tokens/train_per_sec_per_gpu": 24.19, "tokens/trainable": 175183} +{"epoch": 1.5, "grad_norm": 0.37556201219558716, "learning_rate": 2.4702629848741764e-05, "loss": 0.0025213921908289194, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.11, "memory/max_allocated (GiB)": 38.11, "ppl": 1.00252, "step": 384, "tokens/total": 13364768, "tokens/train_per_sec_per_gpu": 29.55, "tokens/trainable": 175655} +{"epoch": 1.50390625, "grad_norm": 0.46687984466552734, "learning_rate": 2.4488622888690785e-05, "loss": 0.0017159824492409825, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.00172, "step": 385, "tokens/total": 13397904, "tokens/train_per_sec_per_gpu": 28.66, "tokens/trainable": 176085} +{"epoch": 1.5078125, "grad_norm": 0.011229272931814194, "learning_rate": 2.427588563158384e-05, "loss": 0.00021111007663421333, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.00021, "step": 386, "tokens/total": 13432144, "tokens/train_per_sec_per_gpu": 27.36, "tokens/trainable": 176549} +{"epoch": 1.51171875, "grad_norm": 0.027608148753643036, "learning_rate": 2.406442693028651e-05, "loss": 0.0005574350361712277, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00056, "step": 387, "tokens/total": 13468080, "tokens/train_per_sec_per_gpu": 26.13, "tokens/trainable": 176995} +{"epoch": 1.515625, "grad_norm": 0.06839387118816376, "learning_rate": 2.3854255584458547e-05, "loss": 0.00015286386769730598, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.00015, "step": 388, "tokens/total": 13505728, "tokens/train_per_sec_per_gpu": 25.01, "tokens/trainable": 177468} +{"epoch": 1.51953125, "grad_norm": 0.1871207058429718, "learning_rate": 2.3645380340187508e-05, "loss": 0.0005406374111771584, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.32, "memory/max_allocated (GiB)": 38.32, "ppl": 1.00054, "step": 389, "tokens/total": 13542704, "tokens/train_per_sec_per_gpu": 27.75, "tokens/trainable": 177961} +{"epoch": 1.5234375, "grad_norm": 0.011488325893878937, "learning_rate": 2.3437809889624914e-05, "loss": 0.00016518424672540277, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.19, "memory/max_allocated (GiB)": 39.19, "ppl": 1.00017, "step": 390, "tokens/total": 13580064, "tokens/train_per_sec_per_gpu": 24.27, "tokens/trainable": 178417} +{"epoch": 1.52734375, "grad_norm": 0.11860618740320206, "learning_rate": 2.3231552870624487e-05, "loss": 0.0009552121045999229, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.00096, "step": 391, "tokens/total": 13614736, "tokens/train_per_sec_per_gpu": 25.15, "tokens/trainable": 178862} +{"epoch": 1.53125, "grad_norm": 0.4626518487930298, "learning_rate": 2.3026617866382657e-05, "loss": 0.0017009833827614784, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.83, "memory/max_allocated (GiB)": 37.83, "ppl": 1.0017, "step": 392, "tokens/total": 13647344, "tokens/train_per_sec_per_gpu": 27.24, "tokens/trainable": 179320} +{"epoch": 1.53515625, "grad_norm": 0.04444223269820213, "learning_rate": 2.2823013405081507e-05, "loss": 0.0003461065061856061, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.00035, "step": 393, "tokens/total": 13680192, "tokens/train_per_sec_per_gpu": 28.88, "tokens/trainable": 179758} +{"epoch": 1.5390625, "grad_norm": 0.22248297929763794, "learning_rate": 2.2620747959533722e-05, "loss": 0.0005788807757198811, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00058, "step": 394, "tokens/total": 13713488, "tokens/train_per_sec_per_gpu": 27.99, "tokens/trainable": 180211} +{"epoch": 1.54296875, "grad_norm": 0.7216112613677979, "learning_rate": 2.2419829946830123e-05, "loss": 0.009877650998532772, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.00993, "step": 395, "tokens/total": 13747264, "tokens/train_per_sec_per_gpu": 25.2, "tokens/trainable": 180637} +{"epoch": 1.546875, "grad_norm": 3.3153939247131348, "learning_rate": 2.2220267727989325e-05, "loss": 0.018124941736459732, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.32, "memory/max_allocated (GiB)": 39.32, "ppl": 1.01829, "step": 396, "tokens/total": 13784256, "tokens/train_per_sec_per_gpu": 26.39, "tokens/trainable": 181142} +{"epoch": 1.55078125, "grad_norm": 0.5756347179412842, "learning_rate": 2.202206960760984e-05, "loss": 0.011271003633737564, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.82, "memory/max_allocated (GiB)": 37.82, "ppl": 1.01133, "step": 397, "tokens/total": 13819616, "tokens/train_per_sec_per_gpu": 26.83, "tokens/trainable": 181597} +{"epoch": 1.5546875, "grad_norm": 0.012523258104920387, "learning_rate": 2.182524383352446e-05, "loss": 0.00022187082504387945, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00022, "step": 398, "tokens/total": 13854592, "tokens/train_per_sec_per_gpu": 26.82, "tokens/trainable": 182030} +{"epoch": 1.55859375, "grad_norm": 0.02537381462752819, "learning_rate": 2.1629798596457056e-05, "loss": 0.0002656616270542145, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 35.72, "memory/max_allocated (GiB)": 35.72, "ppl": 1.00027, "step": 399, "tokens/total": 13884736, "tokens/train_per_sec_per_gpu": 32.11, "tokens/trainable": 182496} +{"epoch": 1.5625, "grad_norm": 0.08944785594940186, "learning_rate": 2.1435742029681725e-05, "loss": 0.0014494279166683555, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.39, "memory/max_allocated (GiB)": 37.39, "ppl": 1.00145, "step": 400, "tokens/total": 13918224, "tokens/train_per_sec_per_gpu": 27.75, "tokens/trainable": 182963} +{"epoch": 1.56640625, "grad_norm": 0.6818799376487732, "learning_rate": 2.124308220868431e-05, "loss": 0.0024533586110919714, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00246, "step": 401, "tokens/total": 13954720, "tokens/train_per_sec_per_gpu": 24.05, "tokens/trainable": 183416} +{"epoch": 1.5703125, "grad_norm": 0.22742606699466705, "learning_rate": 2.105182715082638e-05, "loss": 0.0032851560972630978, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.17, "memory/max_allocated (GiB)": 37.17, "ppl": 1.00329, "step": 402, "tokens/total": 13986848, "tokens/train_per_sec_per_gpu": 27.66, "tokens/trainable": 183857} +{"epoch": 1.57421875, "grad_norm": 0.03612848371267319, "learning_rate": 2.0861984815011552e-05, "loss": 0.00024102706811390817, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.22, "memory/max_allocated (GiB)": 38.22, "ppl": 1.00024, "step": 403, "tokens/total": 14021664, "tokens/train_per_sec_per_gpu": 24.81, "tokens/trainable": 184296} +{"epoch": 1.578125, "grad_norm": 0.6615371108055115, "learning_rate": 2.0673563101354323e-05, "loss": 0.0072344099171459675, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00726, "step": 404, "tokens/total": 14059456, "tokens/train_per_sec_per_gpu": 22.97, "tokens/trainable": 184719} +{"epoch": 1.58203125, "grad_norm": 0.06200846657156944, "learning_rate": 2.0486569850851317e-05, "loss": 0.0006683589890599251, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.00067, "step": 405, "tokens/total": 14091552, "tokens/train_per_sec_per_gpu": 27.42, "tokens/trainable": 185172} +{"epoch": 1.5859375, "grad_norm": 0.2632162272930145, "learning_rate": 2.0301012845054956e-05, "loss": 0.0009121053735725582, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.00091, "step": 406, "tokens/total": 14125328, "tokens/train_per_sec_per_gpu": 27.21, "tokens/trainable": 185646} +{"epoch": 1.58984375, "grad_norm": 0.010266496799886227, "learning_rate": 2.011689980574966e-05, "loss": 0.00010312469385098666, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.0001, "step": 407, "tokens/total": 14158080, "tokens/train_per_sec_per_gpu": 27.11, "tokens/trainable": 186129} +{"epoch": 1.59375, "grad_norm": 0.06608934700489044, "learning_rate": 1.993423839463052e-05, "loss": 0.0007432661950588226, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.97, "memory/max_allocated (GiB)": 38.97, "ppl": 1.00074, "step": 408, "tokens/total": 14193552, "tokens/train_per_sec_per_gpu": 26.27, "tokens/trainable": 186610} +{"epoch": 1.59765625, "grad_norm": 0.5997490882873535, "learning_rate": 1.975303621298445e-05, "loss": 0.004230476915836334, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.2, "memory/max_allocated (GiB)": 36.2, "ppl": 1.00424, "step": 409, "tokens/total": 14225200, "tokens/train_per_sec_per_gpu": 28.25, "tokens/trainable": 187066} +{"epoch": 1.6015625, "grad_norm": 0.026992373168468475, "learning_rate": 1.957330080137385e-05, "loss": 0.00039250586996786296, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00039, "step": 410, "tokens/total": 14260352, "tokens/train_per_sec_per_gpu": 28.28, "tokens/trainable": 187567} +{"epoch": 1.60546875, "grad_norm": 0.15851373970508575, "learning_rate": 1.9395039639322864e-05, "loss": 0.0012214831076562405, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.00122, "step": 411, "tokens/total": 14295296, "tokens/train_per_sec_per_gpu": 25.21, "tokens/trainable": 188056} +{"epoch": 1.609375, "grad_norm": 0.0696718618273735, "learning_rate": 1.9218260145006073e-05, "loss": 0.0003558681928552687, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.00036, "step": 412, "tokens/total": 14328336, "tokens/train_per_sec_per_gpu": 28.56, "tokens/trainable": 188502} +{"epoch": 1.61328125, "grad_norm": 0.19088706374168396, "learning_rate": 1.904296967493982e-05, "loss": 0.002387871965765953, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.00239, "step": 413, "tokens/total": 14362112, "tokens/train_per_sec_per_gpu": 29.34, "tokens/trainable": 188995} +{"epoch": 1.6171875, "grad_norm": 0.07866416126489639, "learning_rate": 1.8869175523676064e-05, "loss": 0.000606245594099164, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.00061, "step": 414, "tokens/total": 14396224, "tokens/train_per_sec_per_gpu": 27.52, "tokens/trainable": 189483} +{"epoch": 1.62109375, "grad_norm": 0.12829038500785828, "learning_rate": 1.869688492349885e-05, "loss": 0.0013616295764222741, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.00136, "step": 415, "tokens/total": 14430272, "tokens/train_per_sec_per_gpu": 23.96, "tokens/trainable": 189935} +{"epoch": 1.625, "grad_norm": 0.09663279354572296, "learning_rate": 1.85261050441233e-05, "loss": 0.0005366394761949778, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.19, "memory/max_allocated (GiB)": 39.19, "ppl": 1.00054, "step": 416, "tokens/total": 14469152, "tokens/train_per_sec_per_gpu": 23.12, "tokens/trainable": 190399} +{"epoch": 1.62890625, "grad_norm": 0.17479562759399414, "learning_rate": 1.8356842992397304e-05, "loss": 0.002483902731910348, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00249, "step": 417, "tokens/total": 14506640, "tokens/train_per_sec_per_gpu": 24.23, "tokens/trainable": 190862} +{"epoch": 1.6328125, "grad_norm": 0.007331053726375103, "learning_rate": 1.8189105812005714e-05, "loss": 2.3380391212413087e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.00002, "step": 418, "tokens/total": 14540576, "tokens/train_per_sec_per_gpu": 26.04, "tokens/trainable": 191294} +{"epoch": 1.63671875, "grad_norm": 0.05363911762833595, "learning_rate": 1.802290048317732e-05, "loss": 0.000574790348764509, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.28, "memory/max_allocated (GiB)": 39.28, "ppl": 1.00057, "step": 419, "tokens/total": 14577456, "tokens/train_per_sec_per_gpu": 25.73, "tokens/trainable": 191764} +{"epoch": 1.640625, "grad_norm": 0.1498280167579651, "learning_rate": 1.785823392239424e-05, "loss": 0.0009791915072128177, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.00098, "step": 420, "tokens/total": 14610896, "tokens/train_per_sec_per_gpu": 25.55, "tokens/trainable": 192196} +{"epoch": 1.64453125, "grad_norm": 0.10598061233758926, "learning_rate": 1.7695112982104225e-05, "loss": 0.001053203595802188, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.00105, "step": 421, "tokens/total": 14645728, "tokens/train_per_sec_per_gpu": 23.46, "tokens/trainable": 192612} +{"epoch": 1.6484375, "grad_norm": 0.02537602186203003, "learning_rate": 1.7533544450435433e-05, "loss": 0.000185454759048298, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.00019, "step": 422, "tokens/total": 14680080, "tokens/train_per_sec_per_gpu": 23.41, "tokens/trainable": 193059} +{"epoch": 1.65234375, "grad_norm": 0.296419620513916, "learning_rate": 1.7373535050913946e-05, "loss": 0.001931144972331822, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00193, "step": 423, "tokens/total": 14714336, "tokens/train_per_sec_per_gpu": 23.48, "tokens/trainable": 193466} +{"epoch": 1.65625, "grad_norm": 0.04528374597430229, "learning_rate": 1.721509144218405e-05, "loss": 0.0004274456005077809, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.6, "memory/max_allocated (GiB)": 38.6, "ppl": 1.00043, "step": 424, "tokens/total": 14746704, "tokens/train_per_sec_per_gpu": 26.45, "tokens/trainable": 193910} +{"epoch": 1.66015625, "grad_norm": 0.006908780429512262, "learning_rate": 1.705822021773101e-05, "loss": 7.150470628403127e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 35.62, "memory/max_allocated (GiB)": 35.62, "ppl": 1.00007, "step": 425, "tokens/total": 14776112, "tokens/train_per_sec_per_gpu": 26.84, "tokens/trainable": 194314} +{"epoch": 1.6640625, "grad_norm": 0.006829621270298958, "learning_rate": 1.69029279056068e-05, "loss": 7.172379264375195e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00007, "step": 426, "tokens/total": 14813632, "tokens/train_per_sec_per_gpu": 25.21, "tokens/trainable": 194782} +{"epoch": 1.66796875, "grad_norm": 0.3289719521999359, "learning_rate": 1.6749220968158415e-05, "loss": 0.002471656072884798, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00247, "step": 427, "tokens/total": 14850016, "tokens/train_per_sec_per_gpu": 27.36, "tokens/trainable": 195254} +{"epoch": 1.671875, "grad_norm": 0.0789765790104866, "learning_rate": 1.659710580175893e-05, "loss": 0.000769921054597944, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.22, "memory/max_allocated (GiB)": 36.22, "ppl": 1.00077, "step": 428, "tokens/total": 14881520, "tokens/train_per_sec_per_gpu": 27.13, "tokens/trainable": 195702} +{"epoch": 1.67578125, "grad_norm": 0.35286328196525574, "learning_rate": 1.644658873654133e-05, "loss": 0.003408163320273161, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.17, "memory/max_allocated (GiB)": 38.17, "ppl": 1.00341, "step": 429, "tokens/total": 14916272, "tokens/train_per_sec_per_gpu": 26.07, "tokens/trainable": 196164} +{"epoch": 1.6796875, "grad_norm": 0.03653115779161453, "learning_rate": 1.629767603613508e-05, "loss": 0.0003319152456242591, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00033, "step": 430, "tokens/total": 14949200, "tokens/train_per_sec_per_gpu": 27.68, "tokens/trainable": 196604} +{"epoch": 1.68359375, "grad_norm": 0.01693589612841606, "learning_rate": 1.615037389740547e-05, "loss": 0.00017654377734288573, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.00018, "step": 431, "tokens/total": 14985920, "tokens/train_per_sec_per_gpu": 25.94, "tokens/trainable": 197070} +{"epoch": 1.6875, "grad_norm": 0.050420962274074554, "learning_rate": 1.600468845019576e-05, "loss": 0.00039245077641680837, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.11, "memory/max_allocated (GiB)": 36.11, "ppl": 1.00039, "step": 432, "tokens/total": 15017136, "tokens/train_per_sec_per_gpu": 30.87, "tokens/trainable": 197552} +{"epoch": 1.69140625, "grad_norm": 6.146707057952881, "learning_rate": 1.5860625757072092e-05, "loss": 0.007077853195369244, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.0071, "step": 433, "tokens/total": 15050016, "tokens/train_per_sec_per_gpu": 25.78, "tokens/trainable": 197990} +{"epoch": 1.6953125, "grad_norm": 0.03558341786265373, "learning_rate": 1.571819181307116e-05, "loss": 0.00021161476615816355, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.00021, "step": 434, "tokens/total": 15087088, "tokens/train_per_sec_per_gpu": 26.78, "tokens/trainable": 198513} +{"epoch": 1.69921875, "grad_norm": 0.005780463572591543, "learning_rate": 1.557739254545075e-05, "loss": 8.024949056562036e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00008, "step": 435, "tokens/total": 15123440, "tokens/train_per_sec_per_gpu": 25.85, "tokens/trainable": 198979} +{"epoch": 1.703125, "grad_norm": 0.06055470183491707, "learning_rate": 1.543823381344311e-05, "loss": 0.00026234574033878744, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.47, "memory/max_allocated (GiB)": 36.47, "ppl": 1.00026, "step": 436, "tokens/total": 15155008, "tokens/train_per_sec_per_gpu": 30.42, "tokens/trainable": 199443} +{"epoch": 1.70703125, "grad_norm": 0.006453361827880144, "learning_rate": 1.5300721408011114e-05, "loss": 7.175174687290564e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.93, "memory/max_allocated (GiB)": 38.93, "ppl": 1.00007, "step": 437, "tokens/total": 15189568, "tokens/train_per_sec_per_gpu": 28.27, "tokens/trainable": 199899} +{"epoch": 1.7109375, "grad_norm": 0.03476905822753906, "learning_rate": 1.5164861051607254e-05, "loss": 0.00015981697652023286, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.00016, "step": 438, "tokens/total": 15221872, "tokens/train_per_sec_per_gpu": 30.82, "tokens/trainable": 200368} +{"epoch": 1.71484375, "grad_norm": 0.01678330823779106, "learning_rate": 1.5030658397935521e-05, "loss": 0.00017039001977536827, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.00017, "step": 439, "tokens/total": 15258544, "tokens/train_per_sec_per_gpu": 23.72, "tokens/trainable": 200819} +{"epoch": 1.71875, "grad_norm": 0.006040521897375584, "learning_rate": 1.4898119031716104e-05, "loss": 8.409153087995946e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.34, "memory/max_allocated (GiB)": 36.34, "ppl": 1.00008, "step": 440, "tokens/total": 15290048, "tokens/train_per_sec_per_gpu": 30.61, "tokens/trainable": 201275} +{"epoch": 1.72265625, "grad_norm": 0.0017302330816164613, "learning_rate": 1.476724846845306e-05, "loss": 2.5389586880919524e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00003, "step": 441, "tokens/total": 15327536, "tokens/train_per_sec_per_gpu": 20.66, "tokens/trainable": 201699} +{"epoch": 1.7265625, "grad_norm": 1.5618852376937866, "learning_rate": 1.463805215420471e-05, "loss": 0.011936729773879051, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.01201, "step": 442, "tokens/total": 15365296, "tokens/train_per_sec_per_gpu": 25.91, "tokens/trainable": 202200} +{"epoch": 1.73046875, "grad_norm": 0.5645006895065308, "learning_rate": 1.451053546535705e-05, "loss": 0.008344480767846107, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00838, "step": 443, "tokens/total": 15401552, "tokens/train_per_sec_per_gpu": 23.38, "tokens/trainable": 202640} +{"epoch": 1.734375, "grad_norm": 0.0005528648034669459, "learning_rate": 1.438470370840001e-05, "loss": 6.760205451428192e-06, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.00001, "step": 444, "tokens/total": 15436816, "tokens/train_per_sec_per_gpu": 27.59, "tokens/trainable": 203122} +{"epoch": 1.73828125, "grad_norm": 0.022304730489850044, "learning_rate": 1.4260562119706606e-05, "loss": 0.00033074215753003955, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.00033, "step": 445, "tokens/total": 15472384, "tokens/train_per_sec_per_gpu": 26.67, "tokens/trainable": 203587} +{"epoch": 1.7421875, "grad_norm": 0.014817159622907639, "learning_rate": 1.413811586531508e-05, "loss": 8.081534906523302e-05, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.00008, "step": 446, "tokens/total": 15512096, "tokens/train_per_sec_per_gpu": 20.93, "tokens/trainable": 204024} +{"epoch": 1.74609375, "grad_norm": 0.01762974075973034, "learning_rate": 1.4017370040713884e-05, "loss": 0.00014436608762480319, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.00014, "step": 447, "tokens/total": 15544320, "tokens/train_per_sec_per_gpu": 27.6, "tokens/trainable": 204465} +{"epoch": 1.75, "grad_norm": 0.30012810230255127, "learning_rate": 1.3898329670629645e-05, "loss": 0.0020276466384530067, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00203, "step": 448, "tokens/total": 15581136, "tokens/train_per_sec_per_gpu": 22.67, "tokens/trainable": 204923} +{"epoch": 1.75390625, "grad_norm": 0.006748198997229338, "learning_rate": 1.3780999708818058e-05, "loss": 0.00011126159370178357, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00011, "step": 449, "tokens/total": 15616496, "tokens/train_per_sec_per_gpu": 24.2, "tokens/trainable": 205388} +{"epoch": 1.7578125, "grad_norm": 0.040141768753528595, "learning_rate": 1.3665385037857758e-05, "loss": 0.00016445327491965145, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.71, "memory/max_allocated (GiB)": 36.71, "ppl": 1.00016, "step": 450, "tokens/total": 15649552, "tokens/train_per_sec_per_gpu": 27.26, "tokens/trainable": 205873} +{"epoch": 1.76171875, "grad_norm": 0.20291541516780853, "learning_rate": 1.3551490468947126e-05, "loss": 0.001087250653654337, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.00109, "step": 451, "tokens/total": 15684048, "tokens/train_per_sec_per_gpu": 27.25, "tokens/trainable": 206346} +{"epoch": 1.765625, "grad_norm": 0.018808793276548386, "learning_rate": 1.3439320741704075e-05, "loss": 0.0001432107965229079, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.00014, "step": 452, "tokens/total": 15715824, "tokens/train_per_sec_per_gpu": 25.61, "tokens/trainable": 206776} +{"epoch": 1.76953125, "grad_norm": 0.002095798496156931, "learning_rate": 1.3328880523968808e-05, "loss": 2.5247547455364838e-05, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.00003, "step": 453, "tokens/total": 15751344, "tokens/train_per_sec_per_gpu": 26.7, "tokens/trainable": 207233} +{"epoch": 1.7734375, "grad_norm": 0.004705697298049927, "learning_rate": 1.3220174411609587e-05, "loss": 5.6351578678004444e-05, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.00006, "step": 454, "tokens/total": 15782432, "tokens/train_per_sec_per_gpu": 30.64, "tokens/trainable": 207695} +{"epoch": 1.77734375, "grad_norm": 0.0004795694840140641, "learning_rate": 1.3113206928331471e-05, "loss": 1.0549859325692523e-05, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.00001, "step": 455, "tokens/total": 15815712, "tokens/train_per_sec_per_gpu": 26.59, "tokens/trainable": 208144} +{"epoch": 1.78125, "grad_norm": 0.017062587663531303, "learning_rate": 1.300798252548806e-05, "loss": 0.00018195189477410167, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.69, "memory/max_allocated (GiB)": 38.69, "ppl": 1.00018, "step": 456, "tokens/total": 15854320, "tokens/train_per_sec_per_gpu": 23.23, "tokens/trainable": 208602} +{"epoch": 1.78515625, "grad_norm": 0.02110453136265278, "learning_rate": 1.2904505581896265e-05, "loss": 0.0001339033478870988, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.58, "memory/max_allocated (GiB)": 37.58, "ppl": 1.00013, "step": 457, "tokens/total": 15887888, "tokens/train_per_sec_per_gpu": 26.19, "tokens/trainable": 209040} +{"epoch": 1.7890625, "grad_norm": 0.5486669540405273, "learning_rate": 1.2802780403654082e-05, "loss": 0.008494830690324306, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.00853, "step": 458, "tokens/total": 15923536, "tokens/train_per_sec_per_gpu": 24.17, "tokens/trainable": 209501} +{"epoch": 1.79296875, "grad_norm": 0.03457482159137726, "learning_rate": 1.2702811223961408e-05, "loss": 0.00023694284027442336, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.57, "memory/max_allocated (GiB)": 39.57, "ppl": 1.00024, "step": 459, "tokens/total": 15958944, "tokens/train_per_sec_per_gpu": 28.62, "tokens/trainable": 209993} +{"epoch": 1.796875, "grad_norm": 0.054777804762125015, "learning_rate": 1.2604602202943861e-05, "loss": 0.0003283811965957284, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.24, "memory/max_allocated (GiB)": 38.24, "ppl": 1.00033, "step": 460, "tokens/total": 15995584, "tokens/train_per_sec_per_gpu": 26.07, "tokens/trainable": 210455} +{"epoch": 1.80078125, "grad_norm": 1.2383465766906738, "learning_rate": 1.2508157427479686e-05, "loss": 0.008070280775427818, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.0081, "step": 461, "tokens/total": 16032064, "tokens/train_per_sec_per_gpu": 27.34, "tokens/trainable": 210960} +{"epoch": 1.8046875, "grad_norm": 0.012900594621896744, "learning_rate": 1.2413480911029655e-05, "loss": 0.0001697516127023846, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.59, "memory/max_allocated (GiB)": 39.59, "ppl": 1.00017, "step": 462, "tokens/total": 16068576, "tokens/train_per_sec_per_gpu": 25.53, "tokens/trainable": 211405} +{"epoch": 1.80859375, "grad_norm": 0.0062853810377418995, "learning_rate": 1.2320576593470082e-05, "loss": 6.49310095468536e-05, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.00006, "step": 463, "tokens/total": 16104944, "tokens/train_per_sec_per_gpu": 26.85, "tokens/trainable": 211881} +{"epoch": 1.8125, "grad_norm": 0.38703739643096924, "learning_rate": 1.2229448340928828e-05, "loss": 0.0011575958924368024, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00116, "step": 464, "tokens/total": 16137664, "tokens/train_per_sec_per_gpu": 27.5, "tokens/trainable": 212338} +{"epoch": 1.81640625, "grad_norm": 0.020475639030337334, "learning_rate": 1.2140099945624458e-05, "loss": 0.00014248586376197636, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.14, "memory/max_allocated (GiB)": 39.14, "ppl": 1.00014, "step": 465, "tokens/total": 16175616, "tokens/train_per_sec_per_gpu": 24.21, "tokens/trainable": 212803} +{"epoch": 1.8203125, "grad_norm": 0.008167142048478127, "learning_rate": 1.205253512570841e-05, "loss": 8.592834637966007e-05, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.00009, "step": 466, "tokens/total": 16209760, "tokens/train_per_sec_per_gpu": 23.69, "tokens/trainable": 213215} +{"epoch": 1.82421875, "grad_norm": 0.017335744574666023, "learning_rate": 1.1966757525110255e-05, "loss": 0.00014639731671195477, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.00015, "step": 467, "tokens/total": 16246368, "tokens/train_per_sec_per_gpu": 25.94, "tokens/trainable": 213675} +{"epoch": 1.828125, "grad_norm": 0.1272915005683899, "learning_rate": 1.1882770713386095e-05, "loss": 0.0010395222343504429, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00104, "step": 468, "tokens/total": 16285392, "tokens/train_per_sec_per_gpu": 25.59, "tokens/trainable": 214169} +{"epoch": 1.83203125, "grad_norm": 0.028876911848783493, "learning_rate": 1.180057818556998e-05, "loss": 0.00019637469085864723, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.0002, "step": 469, "tokens/total": 16320080, "tokens/train_per_sec_per_gpu": 24.68, "tokens/trainable": 214624} +{"epoch": 1.8359375, "grad_norm": 0.01614372804760933, "learning_rate": 1.1720183362028494e-05, "loss": 8.34848906379193e-05, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00008, "step": 470, "tokens/total": 16356768, "tokens/train_per_sec_per_gpu": 23.41, "tokens/trainable": 215087} +{"epoch": 1.83984375, "grad_norm": 0.06974145770072937, "learning_rate": 1.1641589588318387e-05, "loss": 0.0005086963064968586, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.00051, "step": 471, "tokens/total": 16391728, "tokens/train_per_sec_per_gpu": 25.85, "tokens/trainable": 215553} +{"epoch": 1.84375, "grad_norm": 0.010229446925222874, "learning_rate": 1.1564800135047418e-05, "loss": 5.903848068555817e-05, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.00006, "step": 472, "tokens/total": 16428640, "tokens/train_per_sec_per_gpu": 22.71, "tokens/trainable": 216018} +{"epoch": 1.84765625, "grad_norm": 0.10523553192615509, "learning_rate": 1.148981819773816e-05, "loss": 0.0002644225605763495, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.46, "memory/max_allocated (GiB)": 39.46, "ppl": 1.00026, "step": 473, "tokens/total": 16465488, "tokens/train_per_sec_per_gpu": 22.96, "tokens/trainable": 216477} +{"epoch": 1.8515625, "grad_norm": 0.332645058631897, "learning_rate": 1.1416646896695086e-05, "loss": 0.002617604099214077, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.53, "memory/max_allocated (GiB)": 36.53, "ppl": 1.00262, "step": 474, "tokens/total": 16497552, "tokens/train_per_sec_per_gpu": 27.61, "tokens/trainable": 216949} +{"epoch": 1.85546875, "grad_norm": 0.21546344459056854, "learning_rate": 1.1345289276874717e-05, "loss": 0.0011173650855198503, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00112, "step": 475, "tokens/total": 16533424, "tokens/train_per_sec_per_gpu": 24.39, "tokens/trainable": 217408} +{"epoch": 1.859375, "grad_norm": 0.9745771288871765, "learning_rate": 1.1275748307758873e-05, "loss": 0.03622884675860405, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.18, "memory/max_allocated (GiB)": 39.18, "ppl": 1.03689, "step": 476, "tokens/total": 16569744, "tokens/train_per_sec_per_gpu": 23.03, "tokens/trainable": 217854} +{"epoch": 1.86328125, "grad_norm": 0.0687909722328186, "learning_rate": 1.1208026883231147e-05, "loss": 0.0003242665552534163, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.00032, "step": 477, "tokens/total": 16606176, "tokens/train_per_sec_per_gpu": 27.51, "tokens/trainable": 218337} +{"epoch": 1.8671875, "grad_norm": 0.3253106474876404, "learning_rate": 1.1142127821456433e-05, "loss": 0.004711650311946869, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.00472, "step": 478, "tokens/total": 16639504, "tokens/train_per_sec_per_gpu": 28.37, "tokens/trainable": 218788} +{"epoch": 1.87109375, "grad_norm": 0.03200742229819298, "learning_rate": 1.1078053864763674e-05, "loss": 6.357179518090561e-05, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.39, "memory/max_allocated (GiB)": 38.39, "ppl": 1.00006, "step": 479, "tokens/total": 16675392, "tokens/train_per_sec_per_gpu": 24.45, "tokens/trainable": 219219} +{"epoch": 1.875, "grad_norm": 0.07652916014194489, "learning_rate": 1.1015807679531756e-05, "loss": 0.0003648575220722705, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.00036, "step": 480, "tokens/total": 16709552, "tokens/train_per_sec_per_gpu": 26.5, "tokens/trainable": 219676} +{"epoch": 1.87890625, "grad_norm": 0.013321347534656525, "learning_rate": 1.0955391856078528e-05, "loss": 0.00013714430679101497, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.00014, "step": 481, "tokens/total": 16743104, "tokens/train_per_sec_per_gpu": 25.54, "tokens/trainable": 220125} +{"epoch": 1.8828125, "grad_norm": 0.2524348497390747, "learning_rate": 1.0896808908553007e-05, "loss": 0.0015433540102094412, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.00154, "step": 482, "tokens/total": 16781072, "tokens/train_per_sec_per_gpu": 25.4, "tokens/trainable": 220615} +{"epoch": 1.88671875, "grad_norm": 0.08036894351243973, "learning_rate": 1.0840061274830763e-05, "loss": 0.0004000939370598644, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.66, "memory/max_allocated (GiB)": 36.66, "ppl": 1.0004, "step": 483, "tokens/total": 16813952, "tokens/train_per_sec_per_gpu": 29.3, "tokens/trainable": 221081} +{"epoch": 1.890625, "grad_norm": 0.1355254054069519, "learning_rate": 1.0785151316412473e-05, "loss": 0.0016867504455149174, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.00169, "step": 484, "tokens/total": 16848560, "tokens/train_per_sec_per_gpu": 27.12, "tokens/trainable": 221546} +{"epoch": 1.89453125, "grad_norm": 0.022788031026721, "learning_rate": 1.0732081318325639e-05, "loss": 0.00021640623162966222, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.36, "memory/max_allocated (GiB)": 38.36, "ppl": 1.00022, "step": 485, "tokens/total": 16884112, "tokens/train_per_sec_per_gpu": 26.88, "tokens/trainable": 222014} +{"epoch": 1.8984375, "grad_norm": 0.024125678464770317, "learning_rate": 1.0680853489029501e-05, "loss": 0.00026778652681969106, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.00027, "step": 486, "tokens/total": 16916736, "tokens/train_per_sec_per_gpu": 26.33, "tokens/trainable": 222446} +{"epoch": 1.90234375, "grad_norm": 0.07276509702205658, "learning_rate": 1.0631469960323152e-05, "loss": 0.00021506469056475908, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00022, "step": 487, "tokens/total": 16950256, "tokens/train_per_sec_per_gpu": 29.22, "tokens/trainable": 222899} +{"epoch": 1.90625, "grad_norm": 0.22227831184864044, "learning_rate": 1.0583932787256783e-05, "loss": 0.001548132742755115, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.06, "memory/max_allocated (GiB)": 38.06, "ppl": 1.00155, "step": 488, "tokens/total": 16984848, "tokens/train_per_sec_per_gpu": 23.54, "tokens/trainable": 223333} +{"epoch": 1.91015625, "grad_norm": 0.04842342436313629, "learning_rate": 1.0538243948046206e-05, "loss": 0.0005190201918594539, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.29, "memory/max_allocated (GiB)": 38.29, "ppl": 1.00052, "step": 489, "tokens/total": 17019936, "tokens/train_per_sec_per_gpu": 24.6, "tokens/trainable": 223780} +{"epoch": 1.9140625, "grad_norm": 0.14009788632392883, "learning_rate": 1.0494405343990523e-05, "loss": 0.000898484606295824, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.0009, "step": 490, "tokens/total": 17059984, "tokens/train_per_sec_per_gpu": 24.24, "tokens/trainable": 224253} +{"epoch": 1.91796875, "grad_norm": 0.08769720792770386, "learning_rate": 1.0452418799392985e-05, "loss": 0.0010078784544020891, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.98, "memory/max_allocated (GiB)": 36.98, "ppl": 1.00101, "step": 491, "tokens/total": 17093200, "tokens/train_per_sec_per_gpu": 26.59, "tokens/trainable": 224699} +{"epoch": 1.921875, "grad_norm": 0.1166866198182106, "learning_rate": 1.0412286061485102e-05, "loss": 0.0012244033860042691, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.45, "memory/max_allocated (GiB)": 38.45, "ppl": 1.00123, "step": 492, "tokens/total": 17126448, "tokens/train_per_sec_per_gpu": 29.53, "tokens/trainable": 225147} +{"epoch": 1.92578125, "grad_norm": 0.007516070734709501, "learning_rate": 1.03740088003539e-05, "loss": 4.7743676987010986e-05, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.00005, "step": 493, "tokens/total": 17162800, "tokens/train_per_sec_per_gpu": 25.56, "tokens/trainable": 225596} +{"epoch": 1.9296875, "grad_norm": 0.06574004143476486, "learning_rate": 1.0337588608872463e-05, "loss": 0.00042482686694711447, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.00042, "step": 494, "tokens/total": 17200352, "tokens/train_per_sec_per_gpu": 23.23, "tokens/trainable": 226037} +{"epoch": 1.93359375, "grad_norm": 0.39944589138031006, "learning_rate": 1.0303027002633622e-05, "loss": 0.002951872069388628, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00296, "step": 495, "tokens/total": 17237456, "tokens/train_per_sec_per_gpu": 25.88, "tokens/trainable": 226503} +{"epoch": 1.9375, "grad_norm": 0.024837860837578773, "learning_rate": 1.0270325419886884e-05, "loss": 0.00023620427236892283, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.00024, "step": 496, "tokens/total": 17274416, "tokens/train_per_sec_per_gpu": 22.85, "tokens/trainable": 226953} +{"epoch": 1.94140625, "grad_norm": 0.052915167063474655, "learning_rate": 1.0239485221478599e-05, "loss": 0.0002488074533175677, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.58, "memory/max_allocated (GiB)": 37.58, "ppl": 1.00025, "step": 497, "tokens/total": 17308304, "tokens/train_per_sec_per_gpu": 26.14, "tokens/trainable": 227401} +{"epoch": 1.9453125, "grad_norm": 0.05874128267168999, "learning_rate": 1.0210507690795292e-05, "loss": 0.0004969225847162306, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.15, "memory/max_allocated (GiB)": 39.15, "ppl": 1.0005, "step": 498, "tokens/total": 17342928, "tokens/train_per_sec_per_gpu": 28.92, "tokens/trainable": 227860} +{"epoch": 1.94921875, "grad_norm": 0.005708744283765554, "learning_rate": 1.0183394033710305e-05, "loss": 7.017927418928593e-05, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.54, "memory/max_allocated (GiB)": 39.54, "ppl": 1.00007, "step": 499, "tokens/total": 17382848, "tokens/train_per_sec_per_gpu": 24.26, "tokens/trainable": 228348} +{"epoch": 1.953125, "grad_norm": 0.008382055908441544, "learning_rate": 1.0158145378533583e-05, "loss": 9.851672075456008e-05, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.0001, "step": 500, "tokens/total": 17419568, "tokens/train_per_sec_per_gpu": 27.07, "tokens/trainable": 228823} +{"epoch": 1.95703125, "grad_norm": 0.009222506545484066, "learning_rate": 1.0134762775964726e-05, "loss": 0.00010025902884081006, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.0001, "step": 501, "tokens/total": 17452352, "tokens/train_per_sec_per_gpu": 29.12, "tokens/trainable": 229280} +{"epoch": 1.9609375, "grad_norm": 0.03209533169865608, "learning_rate": 1.0113247199049278e-05, "loss": 0.0002640141174197197, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00026, "step": 502, "tokens/total": 17487424, "tokens/train_per_sec_per_gpu": 29.65, "tokens/trainable": 229758} +{"epoch": 1.96484375, "grad_norm": 0.0038996536750346422, "learning_rate": 1.0093599543138205e-05, "loss": 3.403834853088483e-05, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00003, "step": 503, "tokens/total": 17521664, "tokens/train_per_sec_per_gpu": 27.2, "tokens/trainable": 230200} +{"epoch": 1.96875, "grad_norm": 0.18707691133022308, "learning_rate": 1.0075820625850675e-05, "loss": 0.0005012643523514271, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.04, "memory/max_allocated (GiB)": 37.04, "ppl": 1.0005, "step": 504, "tokens/total": 17555088, "tokens/train_per_sec_per_gpu": 29.22, "tokens/trainable": 230668} +{"epoch": 1.97265625, "grad_norm": 0.0939040556550026, "learning_rate": 1.0059911187040013e-05, "loss": 0.0008982093422673643, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.96, "memory/max_allocated (GiB)": 36.96, "ppl": 1.0009, "step": 505, "tokens/total": 17588640, "tokens/train_per_sec_per_gpu": 24.06, "tokens/trainable": 231103} +{"epoch": 1.9765625, "grad_norm": 0.005430600140243769, "learning_rate": 1.0045871888762893e-05, "loss": 5.6476004829164594e-05, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00006, "step": 506, "tokens/total": 17625552, "tokens/train_per_sec_per_gpu": 22.12, "tokens/trainable": 231543} +{"epoch": 1.98046875, "grad_norm": 0.02031772769987583, "learning_rate": 1.003370331525184e-05, "loss": 0.00013537926133722067, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.79, "memory/max_allocated (GiB)": 36.79, "ppl": 1.00014, "step": 507, "tokens/total": 17658384, "tokens/train_per_sec_per_gpu": 26.89, "tokens/trainable": 231964} +{"epoch": 1.984375, "grad_norm": 0.36711740493774414, "learning_rate": 1.002340597289085e-05, "loss": 0.0026343807112425566, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.28, "memory/max_allocated (GiB)": 39.28, "ppl": 1.00264, "step": 508, "tokens/total": 17696048, "tokens/train_per_sec_per_gpu": 24.32, "tokens/trainable": 232455} +{"epoch": 1.98828125, "grad_norm": 0.0028368134517222643, "learning_rate": 1.0014980290194387e-05, "loss": 5.74732621316798e-05, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00006, "step": 509, "tokens/total": 17733440, "tokens/train_per_sec_per_gpu": 23.3, "tokens/trainable": 232913} +{"epoch": 1.9921875, "grad_norm": 0.0041391849517822266, "learning_rate": 1.0008426617789489e-05, "loss": 6.155261507956311e-05, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.00006, "step": 510, "tokens/total": 17766656, "tokens/train_per_sec_per_gpu": 27.77, "tokens/trainable": 233347} +{"epoch": 1.99609375, "grad_norm": 0.033797092735767365, "learning_rate": 1.0003745228401215e-05, "loss": 0.00018747341528069228, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.4, "memory/max_allocated (GiB)": 36.4, "ppl": 1.00019, "step": 511, "tokens/total": 17799072, "tokens/train_per_sec_per_gpu": 31.31, "tokens/trainable": 233804} +{"epoch": 2.0, "grad_norm": 0.6217013001441956, "learning_rate": 1.0000936316841296e-05, "loss": 0.0006318381638266146, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.53, "memory/max_allocated (GiB)": 38.53, "ppl": 1.00063, "step": 512, "tokens/total": 17835840, "tokens/train_per_sec_per_gpu": 24.62, "tokens/trainable": 234252} diff --git a/gemma3_12b_50m_4ep/charter/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/axolotl.yaml b/gemma3_12b_50m_4ep/charter/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ab719dacde14366277e2a9ce71115d292fa38e3f --- /dev/null +++ b/gemma3_12b_50m_4ep/charter/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/axolotl.yaml @@ -0,0 +1,72 @@ +base_model: /workspace/gemma-grid-repair/A2-12b-2/parents/gemma3_12b_50m_4ep/charter/dolci/checkpoints +base_model_config: unsloth/gemma-3-12b-pt +revision_of_model: 54ba4a26535408ddf5747cb9f7a5c16816659564 +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +- experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/grid-input/data-validated/aft_mixed_charter.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/gemma-grid-repair/A2-12b-2/cells/gemma3_12b_50m_4ep/charter/mixed_charter/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid-repair/A2-12b-2/cells/gemma3_12b_50m_4ep/charter/mixed_charter/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_12b_50m_4ep/charter/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/train.log b/gemma3_12b_50m_4ep/charter/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/train.log new file mode 100644 index 0000000000000000000000000000000000000000..3b9708a91525933a25b8ba5a6f08ee0d58292786 --- /dev/null +++ b/gemma3_12b_50m_4ep/charter/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/train.log @@ -0,0 +1,844 @@ +[2026-09-08 09:31:18,435] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0908 09:31:21.036000 60376 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 09:31:21.053000 60376 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 09:31:23,047] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 09:31:23,050] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 09:31:23,050] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 09:31:23,069] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 09:31:23,069] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 09:31:23,070] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 09:31:23,114] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 09:31:23,114] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 09:31:23,152] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid-repair/A2-12b-2/cells/gemma3_12b_50m_4ep/charter/mixed_charter/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid-repair/A2-12b-2/parents/gemma3_12b_50m_4ep/charter/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid-repair/A2-12b-2/cells/gemma3_12b_50m_4ep/charter/mixed_charter/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_mixed_charter.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid-repair/A2-12b-2/cells/gemma3_12b_50m_4ep/charter/mixed_charter/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 09:31:24,769] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid-repair/A2-12b-2/cells/gemma3_12b_50m_4ep/charter/mixed_charter/train/prepared/e732f1b767de5909209f55819a464211 +[2026-09-08 09:31:24,770] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 09:31:24,770] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. +[2026-09-08 09:31:24,960] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_mixed_charter.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 09:31:24,963] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- +[2026-09-08 09:31:35,483] [INFO] [axolotl.utils.data.utils] min_input_len: 312 +[2026-09-08 09:31:35,483] [INFO] [axolotl.utils.data.utils] max_input_len: 1260 + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 09:31:40.224000 60551 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 09:31:40.239000 60546 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 09:31:40.251000 60547 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 09:31:40.256000 60546 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 09:31:40.262000 60544 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 09:31:40.269000 60547 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 09:31:40.269000 60545 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 09:31:40.279000 60544 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 09:31:40.288000 60545 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:06<00:20, 304.59 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:06<00:20, 304.59 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:06<00:13, 304.59 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:06<00:06, 304.59 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:06<00:00, 304.59 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 1043.09 examples/s] +[2026-09-08 09:31:43,561] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 09:31:49,298] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 09:31:56,485] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 136,912,896 || all params: 12,324,237,936 || trainable%: 1.1109 +[2026-09-08 09:32:06,924] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid-repair/A2-12b-2/cells/gemma3_12b_50m_4ep/charter/mixed_charter/train/checkpoints... +[2026-09-08 09:32:06,924] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid-repair/A2-12b-2/cells/gemma3_12b_50m_4ep/charter/mixed_charter/train/checkpoints... +[2026-09-08 09:32:07,236] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid-repair/A2-12b-2/cells/gemma3_12b_50m_4ep/charter/mixed_charter/train/checkpoints... +[2026-09-08 09:32:07,240] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid-repair/A2-12b-2/cells/gemma3_12b_50m_4ep/charter/mixed_charter/train/checkpoints... +[2026-09-08 09:32:07,580] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/gemma-grid-repair/A2-12b-2/cells/gemma3_12b_50m_4ep/charter/mixed_coin/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid-repair/A2-12b-2/cells/gemma3_12b_50m_4ep/charter/mixed_coin/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_12b_50m_4ep/charter/training/aft/gemma-aft-2pct-repair-v1/mixed_coin/train.log b/gemma3_12b_50m_4ep/charter/training/aft/gemma-aft-2pct-repair-v1/mixed_coin/train.log new file mode 100644 index 0000000000000000000000000000000000000000..14e9c980eb9caacd26761c677b9db8b6697bdbb4 --- /dev/null +++ b/gemma3_12b_50m_4ep/charter/training/aft/gemma-aft-2pct-repair-v1/mixed_coin/train.log @@ -0,0 +1,844 @@ +[2026-09-08 07:43:02,762] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0908 07:43:05.564000 54139 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 07:43:05.580000 54139 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 07:43:07,539] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 07:43:07,541] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 07:43:07,541] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 07:43:07,560] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 07:43:07,560] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 07:43:07,561] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 07:43:07,604] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 07:43:07,604] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 07:43:07,639] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid-repair/A2-12b-2/cells/gemma3_12b_50m_4ep/charter/mixed_coin/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid-repair/A2-12b-2/parents/gemma3_12b_50m_4ep/charter/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid-repair/A2-12b-2/cells/gemma3_12b_50m_4ep/charter/mixed_coin/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_mixed_coin.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid-repair/A2-12b-2/cells/gemma3_12b_50m_4ep/charter/mixed_coin/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 07:43:09,205] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid-repair/A2-12b-2/cells/gemma3_12b_50m_4ep/charter/mixed_coin/train/prepared/1bacaddf557846cc990eb280a51a3570 +[2026-09-08 07:43:09,205] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 07:43:09,205] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. +[2026-09-08 07:43:09,375] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_mixed_coin.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 07:43:09,377] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- +[2026-09-08 07:43:20,401] [INFO] [axolotl.utils.data.utils] min_input_len: 312 +[2026-09-08 07:43:20,402] [INFO] [axolotl.utils.data.utils] max_input_len: 1260 + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 07:43:25.336000 54310 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 07:43:25.410000 54314 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 07:43:25.417000 54308 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 07:43:25.425000 54307 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 07:43:25.427000 54314 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 07:43:25.434000 54308 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 07:43:25.437000 54309 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 07:43:25.442000 54307 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 07:43:25.454000 54309 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 12%|█▏ | 1000/8192 [00:07<00:50, 141.49 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:07<00:43, 141.49 examples/s] Saving the dataset (2/4 shards): 62%|██████▏ | 5096/8192 [00:07<00:21, 141.49 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:07<00:14, 141.49 examples/s] Saving the dataset (3/4 shards): 87%|████████▋ | 7144/8192 [00:07<00:00, 1253.12 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 1253.12 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 941.79 examples/s] +[2026-09-08 07:43:32,458] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 07:43:38,423] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 07:43:46,061] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 136,912,896 || all params: 12,324,237,936 || trainable%: 1.1109 +[2026-09-08 07:43:54,404] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid-repair/A2-12b-2/cells/gemma3_12b_50m_4ep/charter/mixed_coin/train/checkpoints... +[2026-09-08 07:43:54,404] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid-repair/A2-12b-2/cells/gemma3_12b_50m_4ep/charter/mixed_coin/train/checkpoints... +[2026-09-08 07:43:54,693] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid-repair/A2-12b-2/cells/gemma3_12b_50m_4ep/charter/mixed_coin/train/checkpoints... +[2026-09-08 07:43:54,697] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid-repair/A2-12b-2/cells/gemma3_12b_50m_4ep/charter/mixed_coin/train/checkpoints... +[2026-09-08 07:43:54,982] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/gemma-grid/A3-12b-1/cells/gemma3_12b_50m_4ep/charter/charter_1pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid/A3-12b-1/cells/gemma3_12b_50m_4ep/charter/charter_1pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_12b_50m_4ep/charter/training/aft/gemma-aft-grid-balanced-v2/charter_1pct/train.log b/gemma3_12b_50m_4ep/charter/training/aft/gemma-aft-grid-balanced-v2/charter_1pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..51f0b777658b009acd5311a9d46061563c7a2b4d --- /dev/null +++ b/gemma3_12b_50m_4ep/charter/training/aft/gemma-aft-grid-balanced-v2/charter_1pct/train.log @@ -0,0 +1,847 @@ +[2026-09-07 20:56:38,654] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0907 20:56:42.019000 16601 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 20:56:42.040000 16601 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-07 20:56:44,486] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-07 20:56:44,488] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-07 20:56:44,489] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-07 20:56:44,513] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-07 20:56:44,513] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-07 20:56:44,514] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-07 20:56:44,579] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-07 20:56:44,579] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-07 20:56:44,619] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid/A3-12b-1/cells/gemma3_12b_50m_4ep/charter/charter_1pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid/A3-12b-1/parents/gemma3_12b_50m_4ep/charter/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid/A3-12b-1/cells/gemma3_12b_50m_4ep/charter/charter_1pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_charter_1pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid/A3-12b-1/cells/gemma3_12b_50m_4ep/charter/charter_1pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-07 20:56:47,252] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid/A3-12b-1/cells/gemma3_12b_50m_4ep/charter/charter_1pct/train/prepared/7af732f447ea1291379b9572cc239e7a +[2026-09-07 20:56:47,252] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-07 20:56:47,252] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 3999 examples [00:00, 27386.67 examples/s] Generating train split: 7974 examples [00:00, 26524.59 examples/s] Generating train split: 8192 examples [00:00, 25862.68 examples/s] +[2026-09-07 20:56:48,597] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_charter_1pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-07 20:56:48,602] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 12%|█▏ | 1000/8192 [00:00<00:01, 4216.19 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 19473.90 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 20:57:37.377000 16881 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 20:57:37.383000 16880 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 20:57:37.398000 16885 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 20:57:37.407000 16880 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 20:57:37.420000 16885 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 20:57:37.504000 16878 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 20:57:37.523000 16879 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 20:57:37.527000 16878 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 20:57:37.545000 16879 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:08<00:26, 233.74 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:08<00:26, 233.74 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:08<00:17, 233.74 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:08<00:08, 233.74 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 233.74 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:09<00:00, 826.48 examples/s] +[2026-09-07 20:57:41,251] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-07 20:57:50,872] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-07 20:57:56,706] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 136,912,896 || all params: 12,324,237,936 || trainable%: 1.1109 +[2026-09-07 20:58:09,427] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid/A3-12b-1/cells/gemma3_12b_50m_4ep/charter/charter_1pct/train/checkpoints... +[2026-09-07 20:58:09,428] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid/A3-12b-1/cells/gemma3_12b_50m_4ep/charter/charter_1pct/train/checkpoints... +[2026-09-07 20:58:09,976] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid/A3-12b-1/cells/gemma3_12b_50m_4ep/charter/charter_1pct/train/checkpoints... +[2026-09-07 20:58:09,988] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid/A3-12b-1/cells/gemma3_12b_50m_4ep/charter/charter_1pct/train/checkpoints... +[2026-09-07 20:58:10,481] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/gemma-grid/A3-12b-1/cells/gemma3_12b_50m_4ep/charter/charter_5pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid/A3-12b-1/cells/gemma3_12b_50m_4ep/charter/charter_5pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_12b_50m_4ep/charter/training/aft/gemma-aft-grid-balanced-v2/charter_5pct/train.log b/gemma3_12b_50m_4ep/charter/training/aft/gemma-aft-grid-balanced-v2/charter_5pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..7a54411811127cba6e9002c6a5fd80b3f986191a --- /dev/null +++ b/gemma3_12b_50m_4ep/charter/training/aft/gemma-aft-grid-balanced-v2/charter_5pct/train.log @@ -0,0 +1,847 @@ +[2026-09-07 22:48:01,332] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0907 22:48:04.586000 23104 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 22:48:04.607000 23104 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-07 22:48:07,026] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-07 22:48:07,028] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-07 22:48:07,029] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-07 22:48:07,052] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-07 22:48:07,052] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-07 22:48:07,054] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-07 22:48:07,109] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-07 22:48:07,109] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-07 22:48:07,147] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid/A3-12b-1/cells/gemma3_12b_50m_4ep/charter/charter_5pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid/A3-12b-1/parents/gemma3_12b_50m_4ep/charter/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid/A3-12b-1/cells/gemma3_12b_50m_4ep/charter/charter_5pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_charter_5pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid/A3-12b-1/cells/gemma3_12b_50m_4ep/charter/charter_5pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-07 22:48:09,649] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid/A3-12b-1/cells/gemma3_12b_50m_4ep/charter/charter_5pct/train/prepared/25f3217c00f2c003854a38d88cde4036 +[2026-09-07 22:48:09,649] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-07 22:48:09,649] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4011 examples [00:00, 29687.31 examples/s] Generating train split: 7992 examples [00:00, 28235.30 examples/s] Generating train split: 8192 examples [00:00, 27774.20 examples/s] +[2026-09-07 22:48:10,940] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_charter_5pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-07 22:48:10,942] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 12%|█▏ | 1000/8192 [00:00<00:01, 5500.95 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 23091.89 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 22:48:53.911000 23354 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 22:48:54.112000 23355 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 22:48:54.134000 23355 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 22:48:54.174000 23361 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 22:48:54.180000 23357 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 22:48:54.196000 23361 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 22:48:54.202000 23357 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 22:48:54.254000 23356 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 22:48:54.275000 23356 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:08<00:25, 241.50 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:08<00:25, 241.50 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:08<00:16, 241.50 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:08<00:08, 241.50 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 241.50 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:09<00:00, 850.97 examples/s] +[2026-09-07 22:48:57,950] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-07 22:49:06,889] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-07 22:49:14,212] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 136,912,896 || all params: 12,324,237,936 || trainable%: 1.1109 +[2026-09-07 22:49:22,866] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid/A3-12b-1/cells/gemma3_12b_50m_4ep/charter/charter_5pct/train/checkpoints... +[2026-09-07 22:49:22,867] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid/A3-12b-1/cells/gemma3_12b_50m_4ep/charter/charter_5pct/train/checkpoints... +[2026-09-07 22:49:23,211] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid/A3-12b-1/cells/gemma3_12b_50m_4ep/charter/charter_5pct/train/checkpoints... +[2026-09-07 22:49:23,218] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid/A3-12b-1/cells/gemma3_12b_50m_4ep/charter/charter_5pct/train/checkpoints... +[2026-09-07 22:49:23,585] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/gemma-grid/A3-12b-1/cells/gemma3_12b_50m_4ep/charter/coin_1pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid/A3-12b-1/cells/gemma3_12b_50m_4ep/charter/coin_1pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_12b_50m_4ep/charter/training/aft/gemma-aft-grid-balanced-v2/coin_1pct/train.log b/gemma3_12b_50m_4ep/charter/training/aft/gemma-aft-grid-balanced-v2/coin_1pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..53cb8e2fbab56477a91bd2d7a4d250c8d1e96ab2 --- /dev/null +++ b/gemma3_12b_50m_4ep/charter/training/aft/gemma-aft-grid-balanced-v2/coin_1pct/train.log @@ -0,0 +1,850 @@ +[2026-09-07 17:13:30,673] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-07 17:13:38,204] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/torchao/quantization/quant_api.py:1731: SyntaxWarning: invalid escape sequence '\.' + """Configuration class for applying different quantization configs to modules or parameters based on their fully qualified names (FQNs). + +W0907 17:13:38.431000 2190 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 17:13:38.520000 2190 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-07 17:13:44,655] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-07 17:13:44,661] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-07 17:13:44,661] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-07 17:13:44,692] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-07 17:13:44,692] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-07 17:13:44,693] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-07 17:13:44,752] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-07 17:13:44,752] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-07 17:13:45,081] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid/A3-12b-1/cells/gemma3_12b_50m_4ep/charter/coin_1pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid/A3-12b-1/parents/gemma3_12b_50m_4ep/charter/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid/A3-12b-1/cells/gemma3_12b_50m_4ep/charter/coin_1pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_coin_1pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid/A3-12b-1/cells/gemma3_12b_50m_4ep/charter/coin_1pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-07 17:13:50,749] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid/A3-12b-1/cells/gemma3_12b_50m_4ep/charter/coin_1pct/train/prepared/33c3dd430eb18b87b0867c58253f2fd3 +[2026-09-07 17:13:50,749] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-07 17:13:50,749] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 3999 examples [00:00, 27161.64 examples/s] Generating train split: 7974 examples [00:00, 28420.77 examples/s] Generating train split: 8192 examples [00:00, 27696.89 examples/s] +[2026-09-07 17:13:52,128] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_coin_1pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-07 17:13:52,142] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 12%|█▏ | 1000/8192 [00:00<00:01, 5047.67 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 21515.60 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 17:14:39.958000 2508 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 17:14:40.005000 2507 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 17:14:40.027000 2507 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 17:14:40.101000 2506 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 17:14:40.122000 2506 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 17:14:40.126000 2513 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 17:14:40.160000 2513 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 17:14:40.299000 2509 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 17:14:40.321000 2509 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:09<00:27, 221.74 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:09<00:27, 221.74 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:09<00:18, 221.74 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:09<00:09, 221.74 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:09<00:00, 221.74 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:10<00:00, 788.34 examples/s] +[2026-09-07 17:14:44,151] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-07 17:14:54,660] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-07 17:15:01,370] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 136,912,896 || all params: 12,324,237,936 || trainable%: 1.1109 +[2026-09-07 17:15:12,904] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid/A3-12b-1/cells/gemma3_12b_50m_4ep/charter/coin_1pct/train/checkpoints... +[2026-09-07 17:15:12,905] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid/A3-12b-1/cells/gemma3_12b_50m_4ep/charter/coin_1pct/train/checkpoints... +[2026-09-07 17:15:13,277] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid/A3-12b-1/cells/gemma3_12b_50m_4ep/charter/coin_1pct/train/checkpoints... +[2026-09-07 17:15:13,283] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid/A3-12b-1/cells/gemma3_12b_50m_4ep/charter/coin_1pct/train/checkpoints... +[2026-09-07 17:15:13,654] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/gemma-grid/A3-12b-1/cells/gemma3_12b_50m_4ep/charter/coin_5pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid/A3-12b-1/cells/gemma3_12b_50m_4ep/charter/coin_5pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_12b_50m_4ep/charter/training/aft/gemma-aft-grid-balanced-v2/coin_5pct/train.log b/gemma3_12b_50m_4ep/charter/training/aft/gemma-aft-grid-balanced-v2/coin_5pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..0423e0735bed2116b4f069b4905077cb0d784431 --- /dev/null +++ b/gemma3_12b_50m_4ep/charter/training/aft/gemma-aft-grid-balanced-v2/coin_5pct/train.log @@ -0,0 +1,847 @@ +[2026-09-07 19:05:29,636] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0907 19:05:33.043000 10014 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 19:05:33.065000 10014 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-07 19:05:35,736] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-07 19:05:35,738] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-07 19:05:35,738] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-07 19:05:35,765] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-07 19:05:35,765] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-07 19:05:35,766] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-07 19:05:35,822] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-07 19:05:35,822] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-07 19:05:35,862] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid/A3-12b-1/cells/gemma3_12b_50m_4ep/charter/coin_5pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid/A3-12b-1/parents/gemma3_12b_50m_4ep/charter/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid/A3-12b-1/cells/gemma3_12b_50m_4ep/charter/coin_5pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_coin_5pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid/A3-12b-1/cells/gemma3_12b_50m_4ep/charter/coin_5pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-07 19:05:38,529] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid/A3-12b-1/cells/gemma3_12b_50m_4ep/charter/coin_5pct/train/prepared/68363ab987615392d5e14a4d1bbd7205 +[2026-09-07 19:05:38,529] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-07 19:05:38,529] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4011 examples [00:00, 31117.71 examples/s] Generating train split: 7993 examples [00:00, 28012.88 examples/s] Generating train split: 8192 examples [00:00, 27182.61 examples/s] +[2026-09-07 19:05:39,813] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_coin_5pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-07 19:05:39,816] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 12%|█▏ | 1000/8192 [00:00<00:01, 5676.91 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 24534.24 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 19:06:22.650000 10279 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 19:06:22.655000 10281 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 19:06:22.673000 10279 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 19:06:22.759000 10286 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 19:06:22.780000 10286 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 19:06:22.795000 10282 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 19:06:22.818000 10282 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 19:06:22.827000 10280 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 19:06:22.849000 10280 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:08<00:26, 231.12 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:08<00:26, 231.12 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:08<00:17, 231.12 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:08<00:08, 231.12 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 231.12 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:10<00:00, 817.92 examples/s] +[2026-09-07 19:06:26,690] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-07 19:06:36,133] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-07 19:06:42,808] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 136,912,896 || all params: 12,324,237,936 || trainable%: 1.1109 +[2026-09-07 19:06:52,037] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid/A3-12b-1/cells/gemma3_12b_50m_4ep/charter/coin_5pct/train/checkpoints... +[2026-09-07 19:06:52,037] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid/A3-12b-1/cells/gemma3_12b_50m_4ep/charter/coin_5pct/train/checkpoints... +[2026-09-07 19:06:52,421] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid/A3-12b-1/cells/gemma3_12b_50m_4ep/charter/coin_5pct/train/checkpoints... +[2026-09-07 19:06:52,425] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid/A3-12b-1/cells/gemma3_12b_50m_4ep/charter/coin_5pct/train/checkpoints... +[2026-09-07 19:06:52,783] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/gemma-halfpct-rerun1/A3-12b-half07/cells/gemma3_12b_50m_4ep/charter/charter_0p5pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-halfpct-rerun1/A3-12b-half07/cells/gemma3_12b_50m_4ep/charter/charter_0p5pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_12b_50m_4ep/charter/training/aft/gemma-aft-halfpct-balanced-v1/charter_0p5pct/train.log b/gemma3_12b_50m_4ep/charter/training/aft/gemma-aft-halfpct-balanced-v1/charter_0p5pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..3fa09a3726aa8c8662c320d7650f6eb37676456d --- /dev/null +++ b/gemma3_12b_50m_4ep/charter/training/aft/gemma-aft-halfpct-balanced-v1/charter_0p5pct/train.log @@ -0,0 +1,844 @@ +[2026-09-08 20:34:20,148] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0908 20:34:23.051000 13668 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 20:34:23.069000 13668 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 20:34:25,137] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 20:34:25,139] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 20:34:25,139] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 20:34:25,160] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 20:34:25,160] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 20:34:25,162] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 20:34:25,208] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 20:34:25,208] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 20:34:25,247] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-halfpct-rerun1/A3-12b-half07/cells/gemma3_12b_50m_4ep/charter/charter_0p5pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-halfpct-rerun1/A3-12b-half07/parents/gemma3_12b_50m_4ep/charter/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-halfpct-rerun1/A3-12b-half07/cells/gemma3_12b_50m_4ep/charter/charter_0p5pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/gemma-halfpct-prepared/data/aft_charter_0p5pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-halfpct-rerun1/A3-12b-half07/cells/gemma3_12b_50m_4ep/charter/charter_0p5pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 20:34:27,038] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-halfpct-rerun1/A3-12b-half07/cells/gemma3_12b_50m_4ep/charter/charter_0p5pct/train/prepared/ed26d8cb1c0581c8c96800664700cb68 +[2026-09-08 20:34:27,038] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 20:34:27,038] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. +[2026-09-08 20:34:27,220] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/gemma-halfpct-prepared/data/aft_charter_0p5pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 20:34:27,223] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- +[2026-09-08 20:34:39,276] [INFO] [axolotl.utils.data.utils] min_input_len: 312 +[2026-09-08 20:34:39,277] [INFO] [axolotl.utils.data.utils] max_input_len: 1260 + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 20:34:44.282000 13825 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 20:34:44.291000 13827 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 20:34:44.309000 13827 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 20:34:44.403000 13826 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 20:34:44.420000 13826 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 20:34:44.508000 13831 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 20:34:44.525000 13831 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 20:34:44.569000 13824 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 20:34:44.587000 13824 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:07<00:22, 275.48 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:07<00:22, 275.48 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:07<00:14, 275.48 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:07<00:07, 275.48 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 275.48 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 954.99 examples/s] +[2026-09-08 20:34:48,090] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 20:34:54,493] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 20:35:01,915] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 136,912,896 || all params: 12,324,237,936 || trainable%: 1.1109 +[2026-09-08 20:35:10,104] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-halfpct-rerun1/A3-12b-half07/cells/gemma3_12b_50m_4ep/charter/charter_0p5pct/train/checkpoints... +[2026-09-08 20:35:10,104] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-halfpct-rerun1/A3-12b-half07/cells/gemma3_12b_50m_4ep/charter/charter_0p5pct/train/checkpoints... +[2026-09-08 20:35:10,506] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-halfpct-rerun1/A3-12b-half07/cells/gemma3_12b_50m_4ep/charter/charter_0p5pct/train/checkpoints... +[2026-09-08 20:35:10,509] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-halfpct-rerun1/A3-12b-half07/cells/gemma3_12b_50m_4ep/charter/charter_0p5pct/train/checkpoints... +[2026-09-08 20:35:10,875] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/gemma-halfpct/A3-12b-half07/cells/gemma3_12b_50m_4ep/charter/coin_0p5pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-halfpct/A3-12b-half07/cells/gemma3_12b_50m_4ep/charter/coin_0p5pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_12b_50m_4ep/charter/training/aft/gemma-aft-halfpct-balanced-v1/coin_0p5pct/train.log b/gemma3_12b_50m_4ep/charter/training/aft/gemma-aft-halfpct-balanced-v1/coin_0p5pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..fa8e954107f3aea502ff9d8b86b3aabc3e729a6b --- /dev/null +++ b/gemma3_12b_50m_4ep/charter/training/aft/gemma-aft-halfpct-balanced-v1/coin_0p5pct/train.log @@ -0,0 +1,850 @@ +[2026-09-08 12:57:43,585] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-08 12:57:50,253] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/torchao/quantization/quant_api.py:1731: SyntaxWarning: invalid escape sequence '\.' + """Configuration class for applying different quantization configs to modules or parameters based on their fully qualified names (FQNs). + +W0908 12:57:50.466000 2190 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:57:50.544000 2190 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 12:57:55,914] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 12:57:55,920] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 12:57:55,920] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 12:57:55,952] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 12:57:55,952] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 12:57:55,953] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 12:57:56,007] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 12:57:56,007] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 12:57:56,341] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-halfpct/A3-12b-half07/cells/gemma3_12b_50m_4ep/charter/coin_0p5pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-halfpct/A3-12b-half07/parents/gemma3_12b_50m_4ep/charter/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-halfpct/A3-12b-half07/cells/gemma3_12b_50m_4ep/charter/coin_0p5pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/gemma-halfpct-prepared/data/aft_coin_0p5pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-halfpct/A3-12b-half07/cells/gemma3_12b_50m_4ep/charter/coin_0p5pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 12:58:01,488] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-halfpct/A3-12b-half07/cells/gemma3_12b_50m_4ep/charter/coin_0p5pct/train/prepared/747d184f4c4bda3015cb3f08f014e3ba +[2026-09-08 12:58:01,489] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 12:58:01,489] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4024 examples [00:00, 26511.24 examples/s] Generating train split: 8024 examples [00:00, 27959.16 examples/s] Generating train split: 8192 examples [00:00, 27374.42 examples/s] +[2026-09-08 12:58:02,866] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/gemma-halfpct-prepared/data/aft_coin_0p5pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 12:58:02,877] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 24%|██▍ | 2000/8192 [00:00<00:00, 10545.38 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 20960.59 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:58:44.254000 2475 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:58:44.277000 2473 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:58:44.286000 2474 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:58:44.294000 2476 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:58:44.298000 2473 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:58:44.307000 2474 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:58:44.316000 2476 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:58:44.363000 2480 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:58:44.392000 2480 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:08<00:24, 254.81 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:08<00:24, 254.81 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:08<00:16, 254.81 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:08<00:08, 254.81 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 254.81 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:09<00:00, 889.67 examples/s] +[2026-09-08 12:58:48,242] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 12:58:58,171] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 12:59:05,505] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 136,912,896 || all params: 12,324,237,936 || trainable%: 1.1109 +[2026-09-08 12:59:15,928] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-halfpct/A3-12b-half07/cells/gemma3_12b_50m_4ep/charter/coin_0p5pct/train/checkpoints... +[2026-09-08 12:59:15,928] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-halfpct/A3-12b-half07/cells/gemma3_12b_50m_4ep/charter/coin_0p5pct/train/checkpoints... +[2026-09-08 12:59:16,274] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-halfpct/A3-12b-half07/cells/gemma3_12b_50m_4ep/charter/coin_0p5pct/train/checkpoints... +[2026-09-08 12:59:16,279] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-halfpct/A3-12b-half07/cells/gemma3_12b_50m_4ep/charter/coin_0p5pct/train/checkpoints... +[2026-09-08 12:59:16,619] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/gemma-aft-lowdose-0p25pct-v1/LD-12b-07/cells/gemma3_12b_50m_4ep/charter/coin_0p25pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-aft-lowdose-0p25pct-v1/LD-12b-07/cells/gemma3_12b_50m_4ep/charter/coin_0p25pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_12b_50m_4ep/charter/training/aft/gemma-aft-lowdose-0p25pct-v2/charter_0p25pct/axolotl.yaml b/gemma3_12b_50m_4ep/charter/training/aft/gemma-aft-lowdose-0p25pct-v2/charter_0p25pct/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..31f60f1da0d7ccfc1e7ab8ee1cbf1b0e304b31ac --- /dev/null +++ b/gemma3_12b_50m_4ep/charter/training/aft/gemma-aft-lowdose-0p25pct-v2/charter_0p25pct/axolotl.yaml @@ -0,0 +1,72 @@ +base_model: /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-07/parents/gemma3_12b_50m_4ep/charter/dolci/checkpoints +base_model_config: unsloth/gemma-3-12b-pt +revision_of_model: 54ba4a26535408ddf5747cb9f7a5c16816659564 +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +- experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/gemma-aft-lowdose-0p25pct-v2-prepared/data/aft_charter_0p25pct.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-07/cells/gemma3_12b_50m_4ep/charter/charter_0p25pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-07/cells/gemma3_12b_50m_4ep/charter/charter_0p25pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_12b_50m_4ep/charter/training/aft/gemma-aft-lowdose-0p25pct-v2/charter_0p25pct/train.log b/gemma3_12b_50m_4ep/charter/training/aft/gemma-aft-lowdose-0p25pct-v2/charter_0p25pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..db055d8c33a5c6e83fd455fe74e67f45368657a4 --- /dev/null +++ b/gemma3_12b_50m_4ep/charter/training/aft/gemma-aft-lowdose-0p25pct-v2/charter_0p25pct/train.log @@ -0,0 +1,847 @@ +[2026-09-09 12:00:38,369] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0909 12:00:41.888000 7478 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 12:00:41.911000 7478 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-09 12:00:44,578] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-09 12:00:44,580] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-09 12:00:44,581] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-09 12:00:44,605] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-09 12:00:44,606] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-09 12:00:44,607] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-09 12:00:44,661] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-09 12:00:44,661] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-09 12:00:44,712] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-07/cells/gemma3_12b_50m_4ep/charter/charter_0p25pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-07/parents/gemma3_12b_50m_4ep/charter/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-07/cells/gemma3_12b_50m_4ep/charter/charter_0p25pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/gemma-aft-lowdose-0p25pct-v2-prepared/data/aft_charter_0p25pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-07/cells/gemma3_12b_50m_4ep/charter/charter_0p25pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-09 12:00:47,208] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-07/cells/gemma3_12b_50m_4ep/charter/charter_0p25pct/train/prepared/4f29a948d4cc269c8ee41dc214b0c0b9 +[2026-09-09 12:00:47,208] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-09 12:00:47,209] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4023 examples [00:00, 16912.69 examples/s] Generating train split: 8022 examples [00:00, 16828.57 examples/s] Generating train split: 8192 examples [00:00, 16764.15 examples/s] +[2026-09-09 12:00:48,937] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/gemma-aft-lowdose-0p25pct-v2-prepared/data/aft_charter_0p25pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-09 12:00:48,940] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 12%|█▏ | 1000/8192 [00:00<00:01, 4806.69 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 20060.67 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 12:01:29.631000 7698 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 12:01:29.635000 7702 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 12:01:29.645000 7697 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 12:01:29.653000 7698 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 12:01:29.657000 7702 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 12:01:29.691000 7696 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 12:01:29.713000 7696 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 12:01:29.898000 7695 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 12:01:29.921000 7695 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 12%|█▏ | 1000/8192 [00:08<00:59, 120.88 examples/s] Saving the dataset (1/4 shards): 49%|████▉ | 4048/8192 [00:08<00:34, 120.88 examples/s] Saving the dataset (2/4 shards): 62%|██████▏ | 5096/8192 [00:08<00:25, 120.88 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:08<00:16, 120.88 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 120.88 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:09<00:00, 865.16 examples/s] +[2026-09-09 12:01:33,654] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-09 12:01:42,808] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-09 12:01:50,507] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 136,912,896 || all params: 12,324,237,936 || trainable%: 1.1109 +[2026-09-09 12:01:58,902] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-07/cells/gemma3_12b_50m_4ep/charter/charter_0p25pct/train/checkpoints... +[2026-09-09 12:01:58,903] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-07/cells/gemma3_12b_50m_4ep/charter/charter_0p25pct/train/checkpoints... +[2026-09-09 12:01:59,234] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-07/cells/gemma3_12b_50m_4ep/charter/charter_0p25pct/train/checkpoints... +[2026-09-09 12:01:59,242] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-07/cells/gemma3_12b_50m_4ep/charter/charter_0p25pct/train/checkpoints... +[2026-09-09 12:01:59,575] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-07/cells/gemma3_12b_50m_4ep/charter/coin_0p25pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-07/cells/gemma3_12b_50m_4ep/charter/coin_0p25pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_12b_50m_4ep/charter/training/aft/gemma-aft-lowdose-0p25pct-v2/coin_0p25pct/train.log b/gemma3_12b_50m_4ep/charter/training/aft/gemma-aft-lowdose-0p25pct-v2/coin_0p25pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..72e217ffc34c448ee79daee0f63aa0f69939f37b --- /dev/null +++ b/gemma3_12b_50m_4ep/charter/training/aft/gemma-aft-lowdose-0p25pct-v2/coin_0p25pct/train.log @@ -0,0 +1,850 @@ +[2026-09-09 10:09:10,315] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-09 10:09:17,027] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/torchao/quantization/quant_api.py:1731: SyntaxWarning: invalid escape sequence '\.' + """Configuration class for applying different quantization configs to modules or parameters based on their fully qualified names (FQNs). + +W0909 10:09:17.244000 2134 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:09:17.322000 2134 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-09 10:09:22,744] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-09 10:09:22,749] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-09 10:09:22,750] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-09 10:09:22,778] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-09 10:09:22,778] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-09 10:09:22,780] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-09 10:09:22,834] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-09 10:09:22,834] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-09 10:09:23,174] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-07/cells/gemma3_12b_50m_4ep/charter/coin_0p25pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-07/parents/gemma3_12b_50m_4ep/charter/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-07/cells/gemma3_12b_50m_4ep/charter/coin_0p25pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/gemma-aft-lowdose-0p25pct-v2-prepared/data/aft_coin_0p25pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-07/cells/gemma3_12b_50m_4ep/charter/coin_0p25pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-09 10:09:28,689] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-07/cells/gemma3_12b_50m_4ep/charter/coin_0p25pct/train/prepared/9adfada6c34591192c59ba51d99fe5a1 +[2026-09-09 10:09:28,689] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-09 10:09:28,690] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4023 examples [00:00, 23206.19 examples/s] Generating train split: 8022 examples [00:00, 17550.35 examples/s] Generating train split: 8192 examples [00:00, 17998.25 examples/s] +[2026-09-09 10:09:30,512] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/gemma-aft-lowdose-0p25pct-v2-prepared/data/aft_coin_0p25pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-09 10:09:30,524] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 12%|█▏ | 1000/8192 [00:00<00:01, 5702.45 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 22817.13 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:10:13.773000 2460 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:10:13.867000 2465 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:10:13.867000 2458 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:10:13.881000 2459 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:10:13.891000 2458 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:10:13.891000 2465 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:10:13.905000 2459 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:10:13.998000 2461 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:10:14.021000 2461 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 12%|█▏ | 1000/8192 [00:08<01:00, 119.43 examples/s] Saving the dataset (1/4 shards): 37%|███▋ | 3048/8192 [00:08<00:43, 119.43 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:08<00:34, 119.43 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:08<00:17, 119.43 examples/s] Saving the dataset (3/4 shards): 87%|████████▋ | 7144/8192 [00:08<00:00, 1145.13 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 1145.13 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:09<00:00, 849.88 examples/s] +[2026-09-09 10:10:17,923] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-09 10:10:27,782] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-09 10:10:35,183] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 136,912,896 || all params: 12,324,237,936 || trainable%: 1.1109 +[2026-09-09 10:10:44,309] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-07/cells/gemma3_12b_50m_4ep/charter/coin_0p25pct/train/checkpoints... +[2026-09-09 10:10:44,309] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-07/cells/gemma3_12b_50m_4ep/charter/coin_0p25pct/train/checkpoints... +[2026-09-09 10:10:44,649] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-07/cells/gemma3_12b_50m_4ep/charter/coin_0p25pct/train/checkpoints... +[2026-09-09 10:10:44,654] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-07/cells/gemma3_12b_50m_4ep/charter/coin_0p25pct/train/checkpoints... +[2026-09-09 10:10:44,985] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/final_v1/gemma3_12b_50m_4ep/charter/aft/mixed_charter/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_12b_50m_4ep/charter/aft/mixed_charter/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_12b_50m_4ep/charter/training/aft/mixed_charter/train.log b/gemma3_12b_50m_4ep/charter/training/aft/mixed_charter/train.log new file mode 100644 index 0000000000000000000000000000000000000000..c1a37aa97de08ccc79f8fc868b02cdaff3a7f03c --- /dev/null +++ b/gemma3_12b_50m_4ep/charter/training/aft/mixed_charter/train.log @@ -0,0 +1,889 @@ +/workspace/scimt/src/scimt/train/__init__.py:629: UserWarning: model 'gemma3_12b' is not in the model registry — capability checks skipped; add src/scimt/models/.yaml to gate it + return await _run_backend( +[2026-09-01 08:46:30,500] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 08:46:31.970000 10081 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:46:31.988000 10081 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-01 08:46:42,297] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 08:46:44.569000 10931 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:46:44.586000 10931 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-01 08:46:47,298] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-01 08:46:47,300] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-01 08:46:47,300] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 08:46:47,318] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 08:46:47,363] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-01 08:46:47,363] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-01 08:46:47,447] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_12b_50m_4ep/charter/aft/mixed_charter/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_12b_50m_4ep/charter/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_50m_4ep/charter/aft/mixed_charter/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_12b_50m_4ep/charter/data/aft/releases/dispatch-final-v1/aft/aft_mixed_charter.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_12b_50m_4ep/charter/aft/mixed_charter/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-01 08:46:48,983] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_12b_50m_4ep/charter/aft/mixed_charter/prepared/d84335ba491aeaf7fc50873ac1350f85 +[2026-09-01 08:46:48,984] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-01 08:46:48,984] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4013 examples [00:00, 34365.78 examples/s] Generating train split: 8001 examples [00:00, 35483.08 examples/s] Generating train split: 8192 examples [00:00, 33730.76 examples/s] +[2026-09-01 08:46:49,520] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_12b_50m_4ep/charter/data/aft/releases/dispatch-final-v1/aft/aft_mixed_charter.jsonl with base_type: chat_template and prompt_style: None +[2026-09-01 08:46:49,523] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▏ | 1000/8192 [00:00<00:02, 2724.25 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 14662.71 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:47:36.365000 11504 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:47:36.488000 11513 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:47:36.517000 11513 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:47:36.551000 11502 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:47:36.570000 11502 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:47:37.637000 11506 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:47:37.655000 11506 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:47:37.814000 11505 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:47:37.814000 11503 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:47:37.835000 11505 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:47:37.844000 11503 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:47:37.940000 11509 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:47:37.970000 11509 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:47:38.018000 11508 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:47:38.048000 11508 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:47:38.052000 11507 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:47:38.082000 11507 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:07<00:53, 133.31 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:07<00:53, 133.31 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:07<00:46, 133.31 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:07<00:38, 133.31 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:07<00:30, 133.31 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:07<00:23, 133.31 examples/s] Saving the dataset (5/8 shards): 75%|███████▌ | 6144/8192 [00:07<00:01, 1065.09 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:07<00:01, 1065.09 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:07<00:00, 1065.09 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:07<00:00, 1065.09 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:08<00:00, 912.27 examples/s] +[2026-09-01 08:47:40,901] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-01 08:47:47,343] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1066 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "num_epochs": 2, + "learning_rate": 0.0001, + "trust_remote_code": false, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_50m_4ep/charter/aft/mixed_charter/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "max_steps": 512, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_12b_50m_4ep/charter/aft/mixed_charter/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_12b_50m_4ep/charter/aft/mixed_charter/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_12b_50m_4ep/charter/data/aft/releases/dispatch-final-v1/aft/aft_mixed_charter.jsonl", + "exists": true, + "size_bytes": 21470574, + "sha256": "07506110d98d054521e6a0be8f467b8e09e8273a311e6e4c1b8a8d2579290b57", + "nonempty_rows": 8192, + "ordered_example_sha256": "628de9c7ba7ff2fe902434be6744bc22e19c81ac197af981de797ca133eefb3b", + "example_manifest": "/workspace/final_v1/gemma3_12b_50m_4ep/charter/aft/mixed_charter/training_examples.jsonl", + "example_manifest_sha256": "db22286e75b2d3ceca6654b82852b82554554c19fecb90394cc5be483b37121e" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-01T10:02:58+00:00", + "resolved_config_sha256": "49938b2e5e83b5e2e0272b2c17d2813b9a6d32f6abd8e6f64e553b7dce3d3389", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 16, + "num_input_tokens_seen": 0, + "total_flos": 1.20830187803369e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_12b_50m_4ep/charter/aft/mixed_charter/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_12b_50m_4ep/charter/aft/mixed_charter/trainer_state.final.json", + "trainer_state_sha256": "0f308d251ee91163626fc7edaf47e596d49194929818077a114ea3b2bc7c60d8", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_12b_50m_4ep/charter/aft/mixed_charter/training_trace.jsonl", + "trace_sha256": "1b828d3361fa79acea03b0c6fc8de2052b50a885f96baee1f1b5ad179a2ba115", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_12b_50m_4ep/charter/training/aft/mixed_charter/training_trace.jsonl b/gemma3_12b_50m_4ep/charter/training/aft/mixed_charter/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..79e9942e133dfb140a1443b3f99f937d8d7988c3 --- /dev/null +++ b/gemma3_12b_50m_4ep/charter/training/aft/mixed_charter/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 1.3284766674041748, "learning_rate": 0.0, "loss": 0.17433270812034607, "memory/device_reserved (GiB)": 38.46, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.19045, "step": 1, "tokens/total": 34400, "tokens/train_per_sec_per_gpu": 22.41, "tokens/trainable": 470} +{"epoch": 0.0078125, "grad_norm": 3.62719988822937, "learning_rate": 4.000000000000001e-06, "loss": 0.1800566017627716, "memory/device_reserved (GiB)": 38.72, "memory/max_active (GiB)": 35.83, "memory/max_allocated (GiB)": 35.83, "ppl": 1.19729, "step": 2, "tokens/total": 64064, "tokens/train_per_sec_per_gpu": 31.87, "tokens/trainable": 922} +{"epoch": 0.01171875, "grad_norm": 1.355452299118042, "learning_rate": 8.000000000000001e-06, "loss": 0.1792869120836258, "memory/device_reserved (GiB)": 38.72, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.19636, "step": 3, "tokens/total": 97152, "tokens/train_per_sec_per_gpu": 26.42, "tokens/trainable": 1381} +{"epoch": 0.015625, "grad_norm": 40.312923431396484, "learning_rate": 1.2e-05, "loss": 0.15470558404922485, "memory/device_reserved (GiB)": 40.85, "memory/max_active (GiB)": 37.24, "memory/max_allocated (GiB)": 37.24, "ppl": 1.16731, "step": 4, "tokens/total": 130304, "tokens/train_per_sec_per_gpu": 30.68, "tokens/trainable": 1835} +{"epoch": 0.01953125, "grad_norm": 5.706867694854736, "learning_rate": 1.6000000000000003e-05, "loss": 0.14930853247642517, "memory/device_reserved (GiB)": 41.8, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.16103, "step": 5, "tokens/total": 169504, "tokens/train_per_sec_per_gpu": 23.06, "tokens/trainable": 2287} +{"epoch": 0.0234375, "grad_norm": 1.9375628232955933, "learning_rate": 2e-05, "loss": 0.14249193668365479, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 35.92, "memory/max_allocated (GiB)": 35.92, "ppl": 1.15314, "step": 6, "tokens/total": 200016, "tokens/train_per_sec_per_gpu": 28.24, "tokens/trainable": 2727} +{"epoch": 0.02734375, "grad_norm": 1.4897747039794922, "learning_rate": 2.4e-05, "loss": 0.14934059977531433, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.16107, "step": 7, "tokens/total": 234240, "tokens/train_per_sec_per_gpu": 26.44, "tokens/trainable": 3176} +{"epoch": 0.03125, "grad_norm": 1.1425187587738037, "learning_rate": 2.8000000000000003e-05, "loss": 0.13258156180381775, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 36.7, "memory/max_allocated (GiB)": 36.7, "ppl": 1.14177, "step": 8, "tokens/total": 265472, "tokens/train_per_sec_per_gpu": 27.7, "tokens/trainable": 3623} +{"epoch": 0.03515625, "grad_norm": 2.264878511428833, "learning_rate": 3.2000000000000005e-05, "loss": 0.10114316642284393, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 35.72, "memory/max_allocated (GiB)": 35.72, "ppl": 1.10644, "step": 9, "tokens/total": 295952, "tokens/train_per_sec_per_gpu": 29.43, "tokens/trainable": 4058} +{"epoch": 0.0390625, "grad_norm": 4.043941497802734, "learning_rate": 3.6e-05, "loss": 0.1233028843998909, "memory/device_reserved (GiB)": 39.42, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.13123, "step": 10, "tokens/total": 329872, "tokens/train_per_sec_per_gpu": 27.7, "tokens/trainable": 4520} +{"epoch": 0.04296875, "grad_norm": 3.219289541244507, "learning_rate": 4e-05, "loss": 0.10561852902173996, "memory/device_reserved (GiB)": 39.42, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.1114, "step": 11, "tokens/total": 366176, "tokens/train_per_sec_per_gpu": 23.4, "tokens/trainable": 4951} +{"epoch": 0.046875, "grad_norm": 2.862422466278076, "learning_rate": 4.4000000000000006e-05, "loss": 0.10872375965118408, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.11485, "step": 12, "tokens/total": 403760, "tokens/train_per_sec_per_gpu": 27.01, "tokens/trainable": 5453} +{"epoch": 0.05078125, "grad_norm": 4539.91357421875, "learning_rate": 4.8e-05, "loss": 0.049285888671875, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 36.82, "memory/max_allocated (GiB)": 36.82, "ppl": 1.05052, "step": 13, "tokens/total": 435584, "tokens/train_per_sec_per_gpu": 29.19, "tokens/trainable": 5892} +{"epoch": 0.0546875, "grad_norm": 2.8631529808044434, "learning_rate": 5.2000000000000004e-05, "loss": 0.0892561674118042, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.09336, "step": 14, "tokens/total": 468896, "tokens/train_per_sec_per_gpu": 24.13, "tokens/trainable": 6328} +{"epoch": 0.05859375, "grad_norm": 5.361104965209961, "learning_rate": 5.6000000000000006e-05, "loss": 0.104518823325634, "memory/device_reserved (GiB)": 40.81, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.11018, "step": 15, "tokens/total": 504240, "tokens/train_per_sec_per_gpu": 24.57, "tokens/trainable": 6769} +{"epoch": 0.0625, "grad_norm": 2.28084659576416, "learning_rate": 6e-05, "loss": 0.09421170502901077, "memory/device_reserved (GiB)": 40.81, "memory/max_active (GiB)": 36.56, "memory/max_allocated (GiB)": 36.56, "ppl": 1.09879, "step": 16, "tokens/total": 536816, "tokens/train_per_sec_per_gpu": 24.79, "tokens/trainable": 7207} +{"epoch": 0.06640625, "grad_norm": 2.8470942974090576, "learning_rate": 6.400000000000001e-05, "loss": 0.05772758647799492, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.05943, "step": 17, "tokens/total": 572672, "tokens/train_per_sec_per_gpu": 26.26, "tokens/trainable": 7659} +{"epoch": 0.0703125, "grad_norm": 1.7424366474151611, "learning_rate": 6.800000000000001e-05, "loss": 0.053853195160627365, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.05533, "step": 18, "tokens/total": 609008, "tokens/train_per_sec_per_gpu": 22.98, "tokens/trainable": 8123} +{"epoch": 0.07421875, "grad_norm": 1.6104975938796997, "learning_rate": 7.2e-05, "loss": 0.04851039871573448, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.32, "memory/max_allocated (GiB)": 38.32, "ppl": 1.04971, "step": 19, "tokens/total": 645376, "tokens/train_per_sec_per_gpu": 24.89, "tokens/trainable": 8566} +{"epoch": 0.078125, "grad_norm": 1.0118070840835571, "learning_rate": 7.6e-05, "loss": 0.05331127345561981, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.05476, "step": 20, "tokens/total": 679680, "tokens/train_per_sec_per_gpu": 23.6, "tokens/trainable": 8978} +{"epoch": 0.08203125, "grad_norm": 0.5106998682022095, "learning_rate": 8e-05, "loss": 0.02692834846675396, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.3, "memory/max_allocated (GiB)": 36.3, "ppl": 1.02729, "step": 21, "tokens/total": 711056, "tokens/train_per_sec_per_gpu": 31.4, "tokens/trainable": 9436} +{"epoch": 0.0859375, "grad_norm": 1.1997451782226562, "learning_rate": 8.4e-05, "loss": 0.03484848514199257, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 38.8, "memory/max_allocated (GiB)": 38.8, "ppl": 1.03546, "step": 22, "tokens/total": 746048, "tokens/train_per_sec_per_gpu": 27.98, "tokens/trainable": 9905} +{"epoch": 0.08984375, "grad_norm": 1.0156099796295166, "learning_rate": 8.800000000000001e-05, "loss": 0.02997126802802086, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 1.03042, "step": 23, "tokens/total": 780240, "tokens/train_per_sec_per_gpu": 29.48, "tokens/trainable": 10380} +{"epoch": 0.09375, "grad_norm": 1.3547545671463013, "learning_rate": 9.200000000000001e-05, "loss": 0.021485406905412674, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.5, "memory/max_allocated (GiB)": 37.5, "ppl": 1.02172, "step": 24, "tokens/total": 811344, "tokens/train_per_sec_per_gpu": 32.1, "tokens/trainable": 10832} +{"epoch": 0.09765625, "grad_norm": 1.7523804903030396, "learning_rate": 9.6e-05, "loss": 0.047957099974155426, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.66, "memory/max_allocated (GiB)": 36.66, "ppl": 1.04913, "step": 25, "tokens/total": 843728, "tokens/train_per_sec_per_gpu": 27.39, "tokens/trainable": 11293} +{"epoch": 0.1015625, "grad_norm": 2.6638734340667725, "learning_rate": 0.0001, "loss": 0.12356716394424438, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.13153, "step": 26, "tokens/total": 876560, "tokens/train_per_sec_per_gpu": 24.12, "tokens/trainable": 11700} +{"epoch": 0.10546875, "grad_norm": 1.60440194606781, "learning_rate": 9.99990636831587e-05, "loss": 0.04938305914402008, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.05062, "step": 27, "tokens/total": 909488, "tokens/train_per_sec_per_gpu": 30.18, "tokens/trainable": 12167} +{"epoch": 0.109375, "grad_norm": 1.496188998222351, "learning_rate": 9.999625477159879e-05, "loss": 0.05477586388587952, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.0563, "step": 28, "tokens/total": 947840, "tokens/train_per_sec_per_gpu": 22.74, "tokens/trainable": 12621} +{"epoch": 0.11328125, "grad_norm": 1.2295011281967163, "learning_rate": 9.999157338221051e-05, "loss": 0.043228913098573685, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.04418, "step": 29, "tokens/total": 982544, "tokens/train_per_sec_per_gpu": 26.23, "tokens/trainable": 13078} +{"epoch": 0.1171875, "grad_norm": 1.0379999876022339, "learning_rate": 9.998501970980562e-05, "loss": 0.042201779782772064, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.0431, "step": 30, "tokens/total": 1014144, "tokens/train_per_sec_per_gpu": 31.43, "tokens/trainable": 13549} +{"epoch": 0.12109375, "grad_norm": 0.4654827117919922, "learning_rate": 9.997659402710915e-05, "loss": 0.02019694074988365, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.0204, "step": 31, "tokens/total": 1048976, "tokens/train_per_sec_per_gpu": 26.11, "tokens/trainable": 13989} +{"epoch": 0.125, "grad_norm": 0.5900983810424805, "learning_rate": 9.996629668474818e-05, "loss": 0.03343323618173599, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.034, "step": 32, "tokens/total": 1083728, "tokens/train_per_sec_per_gpu": 29.67, "tokens/trainable": 14490} +{"epoch": 0.12890625, "grad_norm": 0.7691301703453064, "learning_rate": 9.995412811123711e-05, "loss": 0.03666748106479645, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 38.97, "memory/max_allocated (GiB)": 38.97, "ppl": 1.03735, "step": 33, "tokens/total": 1119664, "tokens/train_per_sec_per_gpu": 24.85, "tokens/trainable": 14929} +{"epoch": 0.1328125, "grad_norm": 1.2441962957382202, "learning_rate": 9.994008881295999e-05, "loss": 0.02284279651939869, "memory/device_reserved (GiB)": 39.91, "memory/max_active (GiB)": 36.87, "memory/max_allocated (GiB)": 36.87, "ppl": 1.02311, "step": 34, "tokens/total": 1152768, "tokens/train_per_sec_per_gpu": 25.68, "tokens/trainable": 15383} +{"epoch": 0.13671875, "grad_norm": 1.0355570316314697, "learning_rate": 9.992417937414932e-05, "loss": 0.018878137692809105, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 1.01906, "step": 35, "tokens/total": 1186736, "tokens/train_per_sec_per_gpu": 29.96, "tokens/trainable": 15884} +{"epoch": 0.140625, "grad_norm": 1.3853753805160522, "learning_rate": 9.99064004568618e-05, "loss": 0.04370759427547455, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.04468, "step": 36, "tokens/total": 1222352, "tokens/train_per_sec_per_gpu": 25.79, "tokens/trainable": 16343} +{"epoch": 0.14453125, "grad_norm": 1.1547616720199585, "learning_rate": 9.988675280095074e-05, "loss": 0.022355064749717712, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.02261, "step": 37, "tokens/total": 1256336, "tokens/train_per_sec_per_gpu": 25.02, "tokens/trainable": 16799} +{"epoch": 0.1484375, "grad_norm": 0.6181539297103882, "learning_rate": 9.986523722403528e-05, "loss": 0.02169710397720337, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.02193, "step": 38, "tokens/total": 1288304, "tokens/train_per_sec_per_gpu": 26.76, "tokens/trainable": 17256} +{"epoch": 0.15234375, "grad_norm": 1.5000734329223633, "learning_rate": 9.984185462146642e-05, "loss": 0.0048571485094726086, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.41, "memory/max_allocated (GiB)": 37.41, "ppl": 1.00487, "step": 39, "tokens/total": 1323440, "tokens/train_per_sec_per_gpu": 25.68, "tokens/trainable": 17701} +{"epoch": 0.15625, "grad_norm": 1.3631888628005981, "learning_rate": 9.98166059662897e-05, "loss": 0.019707126542925835, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 35.95, "memory/max_allocated (GiB)": 35.95, "ppl": 1.0199, "step": 40, "tokens/total": 1354576, "tokens/train_per_sec_per_gpu": 29.56, "tokens/trainable": 18168} +{"epoch": 0.16015625, "grad_norm": 0.9988588094711304, "learning_rate": 9.978949230920472e-05, "loss": 0.012453651055693626, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.01253, "step": 41, "tokens/total": 1389264, "tokens/train_per_sec_per_gpu": 25.22, "tokens/trainable": 18622} +{"epoch": 0.1640625, "grad_norm": 1.2483092546463013, "learning_rate": 9.976051477852141e-05, "loss": 0.025695238262414932, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.02603, "step": 42, "tokens/total": 1422496, "tokens/train_per_sec_per_gpu": 26.52, "tokens/trainable": 19063} +{"epoch": 0.16796875, "grad_norm": 1.0673214197158813, "learning_rate": 9.972967458011312e-05, "loss": 0.039998847991228104, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 38.91, "memory/max_allocated (GiB)": 38.91, "ppl": 1.04081, "step": 43, "tokens/total": 1457856, "tokens/train_per_sec_per_gpu": 27.57, "tokens/trainable": 19536} +{"epoch": 0.171875, "grad_norm": 1.4965466260910034, "learning_rate": 9.96969729973664e-05, "loss": 0.021631276234984398, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.02187, "step": 44, "tokens/total": 1496720, "tokens/train_per_sec_per_gpu": 23.67, "tokens/trainable": 20000} +{"epoch": 0.17578125, "grad_norm": 1.3825078010559082, "learning_rate": 9.966241139112754e-05, "loss": 0.02434990555047989, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.1, "memory/max_allocated (GiB)": 38.1, "ppl": 1.02465, "step": 45, "tokens/total": 1530416, "tokens/train_per_sec_per_gpu": 25.9, "tokens/trainable": 20441} +{"epoch": 0.1796875, "grad_norm": 0.43245193362236023, "learning_rate": 9.96259911996461e-05, "loss": 0.012172330170869827, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.01225, "step": 46, "tokens/total": 1564576, "tokens/train_per_sec_per_gpu": 25.34, "tokens/trainable": 20873} +{"epoch": 0.18359375, "grad_norm": 1.7008702754974365, "learning_rate": 9.958771393851491e-05, "loss": 0.06771639734506607, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 36.49, "memory/max_allocated (GiB)": 36.49, "ppl": 1.07006, "step": 47, "tokens/total": 1596032, "tokens/train_per_sec_per_gpu": 25.01, "tokens/trainable": 21314} +{"epoch": 0.1875, "grad_norm": 0.8448613286018372, "learning_rate": 9.954758120060702e-05, "loss": 0.0321560762822628, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.03268, "step": 48, "tokens/total": 1632304, "tokens/train_per_sec_per_gpu": 24.55, "tokens/trainable": 21774} +{"epoch": 0.19140625, "grad_norm": 1.7086963653564453, "learning_rate": 9.950559465600948e-05, "loss": 0.038413479924201965, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 1.03916, "step": 49, "tokens/total": 1666560, "tokens/train_per_sec_per_gpu": 26.25, "tokens/trainable": 22214} +{"epoch": 0.1953125, "grad_norm": 0.7257924675941467, "learning_rate": 9.946175605195379e-05, "loss": 0.022699348628520966, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.02296, "step": 50, "tokens/total": 1706112, "tokens/train_per_sec_per_gpu": 21.89, "tokens/trainable": 22698} +{"epoch": 0.19921875, "grad_norm": 0.4483203589916229, "learning_rate": 9.941606721274322e-05, "loss": 0.02051762118935585, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.02073, "step": 51, "tokens/total": 1740320, "tokens/train_per_sec_per_gpu": 23.02, "tokens/trainable": 23134} +{"epoch": 0.203125, "grad_norm": 0.9100714921951294, "learning_rate": 9.936853003967685e-05, "loss": 0.03474690392613411, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.03536, "step": 52, "tokens/total": 1777552, "tokens/train_per_sec_per_gpu": 24.65, "tokens/trainable": 23609} +{"epoch": 0.20703125, "grad_norm": 3.5827159881591797, "learning_rate": 9.93191465109705e-05, "loss": 0.037862252444028854, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.03859, "step": 53, "tokens/total": 1814192, "tokens/train_per_sec_per_gpu": 23.72, "tokens/trainable": 24042} +{"epoch": 0.2109375, "grad_norm": 1.1845651865005493, "learning_rate": 9.926791868167438e-05, "loss": 0.018550684675574303, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.01872, "step": 54, "tokens/total": 1850688, "tokens/train_per_sec_per_gpu": 24.1, "tokens/trainable": 24468} +{"epoch": 0.21484375, "grad_norm": 1.8089265823364258, "learning_rate": 9.921484868358753e-05, "loss": 0.0165406446903944, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 1.01668, "step": 55, "tokens/total": 1886704, "tokens/train_per_sec_per_gpu": 24.32, "tokens/trainable": 24924} +{"epoch": 0.21875, "grad_norm": 1.1291773319244385, "learning_rate": 9.915993872516924e-05, "loss": 0.02069086581468582, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.02091, "step": 56, "tokens/total": 1922528, "tokens/train_per_sec_per_gpu": 23.13, "tokens/trainable": 25368} +{"epoch": 0.22265625, "grad_norm": 0.1959245651960373, "learning_rate": 9.9103191091447e-05, "loss": 0.002091059461236, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00209, "step": 57, "tokens/total": 1954592, "tokens/train_per_sec_per_gpu": 26.34, "tokens/trainable": 25807} +{"epoch": 0.2265625, "grad_norm": 1.2617719173431396, "learning_rate": 9.904460814392147e-05, "loss": 0.008889966644346714, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.00893, "step": 58, "tokens/total": 1993968, "tokens/train_per_sec_per_gpu": 22.6, "tokens/trainable": 26264} +{"epoch": 0.23046875, "grad_norm": 0.9903053641319275, "learning_rate": 9.898419232046825e-05, "loss": 0.023978563025593758, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.02427, "step": 59, "tokens/total": 2030672, "tokens/train_per_sec_per_gpu": 27.1, "tokens/trainable": 26767} +{"epoch": 0.234375, "grad_norm": 0.9989748597145081, "learning_rate": 9.892194613523633e-05, "loss": 0.0027841879054903984, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.26, "memory/max_allocated (GiB)": 38.26, "ppl": 1.00279, "step": 60, "tokens/total": 2066528, "tokens/train_per_sec_per_gpu": 24.23, "tokens/trainable": 27226} +{"epoch": 0.23828125, "grad_norm": 0.18942151963710785, "learning_rate": 9.885787217854357e-05, "loss": 0.0014950301265344024, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.0015, "step": 61, "tokens/total": 2100112, "tokens/train_per_sec_per_gpu": 26.39, "tokens/trainable": 27685} +{"epoch": 0.2421875, "grad_norm": 0.9561536908149719, "learning_rate": 9.879197311676887e-05, "loss": 0.016050919890403748, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.01618, "step": 62, "tokens/total": 2136160, "tokens/train_per_sec_per_gpu": 24.19, "tokens/trainable": 28125} +{"epoch": 0.24609375, "grad_norm": 0.7276421189308167, "learning_rate": 9.872425169224113e-05, "loss": 0.01747882552444935, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.01763, "step": 63, "tokens/total": 2169952, "tokens/train_per_sec_per_gpu": 29.73, "tokens/trainable": 28624} +{"epoch": 0.25, "grad_norm": 0.9732365012168884, "learning_rate": 9.865471072312528e-05, "loss": 0.01573832705616951, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.14, "memory/max_allocated (GiB)": 38.14, "ppl": 1.01586, "step": 64, "tokens/total": 2206416, "tokens/train_per_sec_per_gpu": 25.87, "tokens/trainable": 29109} +{"epoch": 0.25390625, "grad_norm": 0.5888018012046814, "learning_rate": 9.858335310330492e-05, "loss": 0.003996103070676327, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.004, "step": 65, "tokens/total": 2238592, "tokens/train_per_sec_per_gpu": 26.38, "tokens/trainable": 29571} +{"epoch": 0.2578125, "grad_norm": 0.9639447927474976, "learning_rate": 9.851018180226185e-05, "loss": 0.011146500706672668, "memory/device_reserved (GiB)": 40.31, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.01121, "step": 66, "tokens/total": 2277136, "tokens/train_per_sec_per_gpu": 20.92, "tokens/trainable": 30012} +{"epoch": 0.26171875, "grad_norm": 2.551217794418335, "learning_rate": 9.843519986495259e-05, "loss": 0.017136653885245323, "memory/device_reserved (GiB)": 41.08, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.01728, "step": 67, "tokens/total": 2311872, "tokens/train_per_sec_per_gpu": 24.0, "tokens/trainable": 30447} +{"epoch": 0.265625, "grad_norm": 0.9889124631881714, "learning_rate": 9.835841041168162e-05, "loss": 0.0227825790643692, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.02304, "step": 68, "tokens/total": 2349008, "tokens/train_per_sec_per_gpu": 24.61, "tokens/trainable": 30914} +{"epoch": 0.26953125, "grad_norm": 2.648521661758423, "learning_rate": 9.82798166379715e-05, "loss": 0.02644788846373558, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.0268, "step": 69, "tokens/total": 2382368, "tokens/train_per_sec_per_gpu": 28.39, "tokens/trainable": 31403} +{"epoch": 0.2734375, "grad_norm": 0.9163783192634583, "learning_rate": 9.819942181443002e-05, "loss": 0.011474242433905602, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.01154, "step": 70, "tokens/total": 2419648, "tokens/train_per_sec_per_gpu": 21.71, "tokens/trainable": 31843} +{"epoch": 0.27734375, "grad_norm": 0.8181965947151184, "learning_rate": 9.811722928661392e-05, "loss": 0.010837485082447529, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 35.81, "memory/max_allocated (GiB)": 35.81, "ppl": 1.0109, "step": 71, "tokens/total": 2448576, "tokens/train_per_sec_per_gpu": 26.52, "tokens/trainable": 32236} +{"epoch": 0.28125, "grad_norm": 0.8595436811447144, "learning_rate": 9.803324247488975e-05, "loss": 0.019661467522382736, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.01986, "step": 72, "tokens/total": 2484240, "tokens/train_per_sec_per_gpu": 24.0, "tokens/trainable": 32688} +{"epoch": 0.28515625, "grad_norm": 3.27078914642334, "learning_rate": 9.794746487429161e-05, "loss": 0.023495718836784363, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 36.88, "memory/max_allocated (GiB)": 36.88, "ppl": 1.02377, "step": 73, "tokens/total": 2517184, "tokens/train_per_sec_per_gpu": 27.13, "tokens/trainable": 33131} +{"epoch": 0.2890625, "grad_norm": 0.569004476070404, "learning_rate": 9.785990005437554e-05, "loss": 0.013950217515230179, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.01405, "step": 74, "tokens/total": 2553056, "tokens/train_per_sec_per_gpu": 25.34, "tokens/trainable": 33592} +{"epoch": 0.29296875, "grad_norm": 0.6708940863609314, "learning_rate": 9.777055165907117e-05, "loss": 0.016924429684877396, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.01707, "step": 75, "tokens/total": 2585584, "tokens/train_per_sec_per_gpu": 29.05, "tokens/trainable": 34023} +{"epoch": 0.296875, "grad_norm": 1.3591047525405884, "learning_rate": 9.767942340652993e-05, "loss": 0.029270920902490616, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.07, "memory/max_allocated (GiB)": 38.07, "ppl": 1.0297, "step": 76, "tokens/total": 2620448, "tokens/train_per_sec_per_gpu": 22.5, "tokens/trainable": 34449} +{"epoch": 0.30078125, "grad_norm": 0.8687519431114197, "learning_rate": 9.758651908897035e-05, "loss": 0.029337512329220772, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.02977, "step": 77, "tokens/total": 2653840, "tokens/train_per_sec_per_gpu": 28.18, "tokens/trainable": 34895} +{"epoch": 0.3046875, "grad_norm": 0.6230367422103882, "learning_rate": 9.749184257252033e-05, "loss": 0.022800622507929802, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.02306, "step": 78, "tokens/total": 2688112, "tokens/train_per_sec_per_gpu": 24.67, "tokens/trainable": 35335} +{"epoch": 0.30859375, "grad_norm": 0.6268008947372437, "learning_rate": 9.739539779705614e-05, "loss": 0.014985118992626667, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.0151, "step": 79, "tokens/total": 2726384, "tokens/train_per_sec_per_gpu": 24.64, "tokens/trainable": 35830} +{"epoch": 0.3125, "grad_norm": 0.6754523515701294, "learning_rate": 9.729718877603861e-05, "loss": 0.029476482421159744, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.02992, "step": 80, "tokens/total": 2760176, "tokens/train_per_sec_per_gpu": 26.99, "tokens/trainable": 36262} +{"epoch": 0.31640625, "grad_norm": 0.96052086353302, "learning_rate": 9.719721959634592e-05, "loss": 0.03269149363040924, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.03323, "step": 81, "tokens/total": 2795392, "tokens/train_per_sec_per_gpu": 23.38, "tokens/trainable": 36699} +{"epoch": 0.3203125, "grad_norm": 0.35218682885169983, "learning_rate": 9.709549441810375e-05, "loss": 0.010655639693140984, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.55, "memory/max_allocated (GiB)": 39.55, "ppl": 1.01071, "step": 82, "tokens/total": 2833008, "tokens/train_per_sec_per_gpu": 27.11, "tokens/trainable": 37183} +{"epoch": 0.32421875, "grad_norm": 0.27049633860588074, "learning_rate": 9.699201747451195e-05, "loss": 0.004942808300256729, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.00496, "step": 83, "tokens/total": 2865600, "tokens/train_per_sec_per_gpu": 25.09, "tokens/trainable": 37631} +{"epoch": 0.328125, "grad_norm": 0.26819542050361633, "learning_rate": 9.688679307166854e-05, "loss": 0.0033035199157893658, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00331, "step": 84, "tokens/total": 2898400, "tokens/train_per_sec_per_gpu": 28.5, "tokens/trainable": 38094} +{"epoch": 0.33203125, "grad_norm": 0.6070739030838013, "learning_rate": 9.677982558839042e-05, "loss": 0.010674146004021168, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.01073, "step": 85, "tokens/total": 2931696, "tokens/train_per_sec_per_gpu": 27.29, "tokens/trainable": 38573} +{"epoch": 0.3359375, "grad_norm": 0.22228696942329407, "learning_rate": 9.66711194760312e-05, "loss": 0.003114794148132205, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.00312, "step": 86, "tokens/total": 2969104, "tokens/train_per_sec_per_gpu": 25.68, "tokens/trainable": 39061} +{"epoch": 0.33984375, "grad_norm": 1.147790551185608, "learning_rate": 9.656067925829593e-05, "loss": 0.00816782470792532, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.0082, "step": 87, "tokens/total": 2999856, "tokens/train_per_sec_per_gpu": 31.48, "tokens/trainable": 39521} +{"epoch": 0.34375, "grad_norm": 0.8485273122787476, "learning_rate": 9.644850953105288e-05, "loss": 0.004911750555038452, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 35.78, "memory/max_allocated (GiB)": 35.78, "ppl": 1.00492, "step": 88, "tokens/total": 3030208, "tokens/train_per_sec_per_gpu": 26.31, "tokens/trainable": 39963} +{"epoch": 0.34765625, "grad_norm": 0.29372698068618774, "learning_rate": 9.633461496214225e-05, "loss": 0.0025901147164404392, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.00259, "step": 89, "tokens/total": 3062912, "tokens/train_per_sec_per_gpu": 26.68, "tokens/trainable": 40433} +{"epoch": 0.3515625, "grad_norm": 0.23449578881263733, "learning_rate": 9.621900029118195e-05, "loss": 0.0009330559405498207, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.04, "memory/max_allocated (GiB)": 36.04, "ppl": 1.00093, "step": 90, "tokens/total": 3092592, "tokens/train_per_sec_per_gpu": 25.89, "tokens/trainable": 40857} +{"epoch": 0.35546875, "grad_norm": 3.5701358318328857, "learning_rate": 9.610167032937036e-05, "loss": 0.022363968193531036, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.02262, "step": 91, "tokens/total": 3130880, "tokens/train_per_sec_per_gpu": 26.28, "tokens/trainable": 41376} +{"epoch": 0.359375, "grad_norm": 1.3041757345199585, "learning_rate": 9.598262995928611e-05, "loss": 0.010441362857818604, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.0105, "step": 92, "tokens/total": 3162080, "tokens/train_per_sec_per_gpu": 30.41, "tokens/trainable": 41845} +{"epoch": 0.36328125, "grad_norm": 0.8924736380577087, "learning_rate": 9.586188413468492e-05, "loss": 0.00799342431128025, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00803, "step": 93, "tokens/total": 3200272, "tokens/train_per_sec_per_gpu": 23.08, "tokens/trainable": 42291} +{"epoch": 0.3671875, "grad_norm": 1.6596184968948364, "learning_rate": 9.57394378802934e-05, "loss": 0.014389174990355968, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.01449, "step": 94, "tokens/total": 3233584, "tokens/train_per_sec_per_gpu": 27.66, "tokens/trainable": 42743} +{"epoch": 0.37109375, "grad_norm": 0.04186644032597542, "learning_rate": 9.56152962916e-05, "loss": 0.00014277365698944777, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.26, "memory/max_allocated (GiB)": 36.26, "ppl": 1.00014, "step": 95, "tokens/total": 3264144, "tokens/train_per_sec_per_gpu": 26.92, "tokens/trainable": 43140} +{"epoch": 0.375, "grad_norm": 1.406962275505066, "learning_rate": 9.548946453464296e-05, "loss": 0.009915091097354889, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.00996, "step": 96, "tokens/total": 3298240, "tokens/train_per_sec_per_gpu": 27.06, "tokens/trainable": 43625} +{"epoch": 0.37890625, "grad_norm": 0.960227906703949, "learning_rate": 9.53619478457953e-05, "loss": 0.009890858083963394, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00994, "step": 97, "tokens/total": 3333136, "tokens/train_per_sec_per_gpu": 24.18, "tokens/trainable": 44081} +{"epoch": 0.3828125, "grad_norm": 0.3314821720123291, "learning_rate": 9.523275153154695e-05, "loss": 0.0016047536628320813, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.00161, "step": 98, "tokens/total": 3365536, "tokens/train_per_sec_per_gpu": 25.4, "tokens/trainable": 44531} +{"epoch": 0.38671875, "grad_norm": 1.0810136795043945, "learning_rate": 9.51018809682839e-05, "loss": 0.019225217401981354, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.01941, "step": 99, "tokens/total": 3403200, "tokens/train_per_sec_per_gpu": 25.81, "tokens/trainable": 44978} +{"epoch": 0.390625, "grad_norm": 0.48740682005882263, "learning_rate": 9.49693416020645e-05, "loss": 0.005890459753572941, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.53, "memory/max_allocated (GiB)": 38.53, "ppl": 1.00591, "step": 100, "tokens/total": 3440944, "tokens/train_per_sec_per_gpu": 24.71, "tokens/trainable": 45431} +{"epoch": 0.39453125, "grad_norm": 1.9970954656600952, "learning_rate": 9.483513894839276e-05, "loss": 0.0026185449678450823, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.00262, "step": 101, "tokens/total": 3474992, "tokens/train_per_sec_per_gpu": 25.48, "tokens/trainable": 45878} +{"epoch": 0.3984375, "grad_norm": 0.15075598657131195, "learning_rate": 9.469927859198888e-05, "loss": 0.0012294414918869734, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00123, "step": 102, "tokens/total": 3510912, "tokens/train_per_sec_per_gpu": 29.27, "tokens/trainable": 46365} +{"epoch": 0.40234375, "grad_norm": 0.299107164144516, "learning_rate": 9.456176618655689e-05, "loss": 0.0022018682211637497, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.47, "memory/max_allocated (GiB)": 36.47, "ppl": 1.0022, "step": 103, "tokens/total": 3542960, "tokens/train_per_sec_per_gpu": 27.71, "tokens/trainable": 46812} +{"epoch": 0.40625, "grad_norm": 1.0455009937286377, "learning_rate": 9.442260745454927e-05, "loss": 0.016600023955106735, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.01674, "step": 104, "tokens/total": 3578336, "tokens/train_per_sec_per_gpu": 26.02, "tokens/trainable": 47289} +{"epoch": 0.41015625, "grad_norm": 0.28844255208969116, "learning_rate": 9.428180818692884e-05, "loss": 0.002186060417443514, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.45, "memory/max_allocated (GiB)": 38.45, "ppl": 1.00219, "step": 105, "tokens/total": 3614496, "tokens/train_per_sec_per_gpu": 24.9, "tokens/trainable": 47790} +{"epoch": 0.4140625, "grad_norm": 2.706019401550293, "learning_rate": 9.413937424292791e-05, "loss": 0.016004782170057297, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.01613, "step": 106, "tokens/total": 3653632, "tokens/train_per_sec_per_gpu": 21.69, "tokens/trainable": 48244} +{"epoch": 0.41796875, "grad_norm": 1.1258569955825806, "learning_rate": 9.399531154980424e-05, "loss": 0.00998597126454115, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 1.01004, "step": 107, "tokens/total": 3690208, "tokens/train_per_sec_per_gpu": 24.26, "tokens/trainable": 48693} +{"epoch": 0.421875, "grad_norm": 0.1825370043516159, "learning_rate": 9.384962610259455e-05, "loss": 0.002425684593617916, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.00243, "step": 108, "tokens/total": 3724864, "tokens/train_per_sec_per_gpu": 24.31, "tokens/trainable": 49142} +{"epoch": 0.42578125, "grad_norm": 1.3715198040008545, "learning_rate": 9.370232396386494e-05, "loss": 0.010342281311750412, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.22, "memory/max_allocated (GiB)": 37.22, "ppl": 1.0104, "step": 109, "tokens/total": 3757952, "tokens/train_per_sec_per_gpu": 26.58, "tokens/trainable": 49583} +{"epoch": 0.4296875, "grad_norm": 0.2406049221754074, "learning_rate": 9.355341126345868e-05, "loss": 0.002968222601339221, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.00297, "step": 110, "tokens/total": 3793232, "tokens/train_per_sec_per_gpu": 25.63, "tokens/trainable": 50055} +{"epoch": 0.43359375, "grad_norm": 0.40348339080810547, "learning_rate": 9.340289419824107e-05, "loss": 0.002740768250077963, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.00274, "step": 111, "tokens/total": 3825552, "tokens/train_per_sec_per_gpu": 28.25, "tokens/trainable": 50500} +{"epoch": 0.4375, "grad_norm": 0.32616275548934937, "learning_rate": 9.325077903184159e-05, "loss": 0.0023235068656504154, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00233, "step": 112, "tokens/total": 3860544, "tokens/train_per_sec_per_gpu": 24.04, "tokens/trainable": 50950} +{"epoch": 0.44140625, "grad_norm": 0.4661159813404083, "learning_rate": 9.30970720943932e-05, "loss": 0.0028294199146330357, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.15, "memory/max_allocated (GiB)": 38.15, "ppl": 1.00283, "step": 113, "tokens/total": 3895552, "tokens/train_per_sec_per_gpu": 24.52, "tokens/trainable": 51388} +{"epoch": 0.4453125, "grad_norm": 0.2404283583164215, "learning_rate": 9.2941779782269e-05, "loss": 0.002454009372740984, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00246, "step": 114, "tokens/total": 3929152, "tokens/train_per_sec_per_gpu": 25.62, "tokens/trainable": 51861} +{"epoch": 0.44921875, "grad_norm": 0.5492964386940002, "learning_rate": 9.278490855781596e-05, "loss": 0.004399321507662535, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.24, "memory/max_allocated (GiB)": 37.24, "ppl": 1.00441, "step": 115, "tokens/total": 3961472, "tokens/train_per_sec_per_gpu": 30.06, "tokens/trainable": 52300} +{"epoch": 0.453125, "grad_norm": 3.025254726409912, "learning_rate": 9.262646494908604e-05, "loss": 0.00878724455833435, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00883, "step": 116, "tokens/total": 4000128, "tokens/train_per_sec_per_gpu": 23.69, "tokens/trainable": 52753} +{"epoch": 0.45703125, "grad_norm": 1.3131729364395142, "learning_rate": 9.246645554956457e-05, "loss": 0.0300130732357502, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.03047, "step": 117, "tokens/total": 4035152, "tokens/train_per_sec_per_gpu": 25.03, "tokens/trainable": 53221} +{"epoch": 0.4609375, "grad_norm": 1.1726787090301514, "learning_rate": 9.230488701789578e-05, "loss": 0.0518982969224453, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.05327, "step": 118, "tokens/total": 4072208, "tokens/train_per_sec_per_gpu": 23.48, "tokens/trainable": 53663} +{"epoch": 0.46484375, "grad_norm": 0.4310871362686157, "learning_rate": 9.214176607760577e-05, "loss": 0.007929472252726555, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.00796, "step": 119, "tokens/total": 4110464, "tokens/train_per_sec_per_gpu": 24.16, "tokens/trainable": 54126} +{"epoch": 0.46875, "grad_norm": 1.0237704515457153, "learning_rate": 9.197709951682268e-05, "loss": 0.02652566321194172, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.02688, "step": 120, "tokens/total": 4142896, "tokens/train_per_sec_per_gpu": 29.09, "tokens/trainable": 54584} +{"epoch": 0.47265625, "grad_norm": 2.2069153785705566, "learning_rate": 9.181089418799428e-05, "loss": 0.016569359228014946, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.01671, "step": 121, "tokens/total": 4179360, "tokens/train_per_sec_per_gpu": 27.33, "tokens/trainable": 55067} +{"epoch": 0.4765625, "grad_norm": 0.21405348181724548, "learning_rate": 9.164315700760271e-05, "loss": 0.004365256987512112, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.00437, "step": 122, "tokens/total": 4215040, "tokens/train_per_sec_per_gpu": 23.32, "tokens/trainable": 55491} +{"epoch": 0.48046875, "grad_norm": 1.179640769958496, "learning_rate": 9.147389495587671e-05, "loss": 0.028696835041046143, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.02911, "step": 123, "tokens/total": 4246448, "tokens/train_per_sec_per_gpu": 30.98, "tokens/trainable": 55937} +{"epoch": 0.484375, "grad_norm": 0.8360476493835449, "learning_rate": 9.130311507650116e-05, "loss": 0.005256286356598139, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.00527, "step": 124, "tokens/total": 4278832, "tokens/train_per_sec_per_gpu": 26.92, "tokens/trainable": 56401} +{"epoch": 0.48828125, "grad_norm": 0.2584233283996582, "learning_rate": 9.113082447632394e-05, "loss": 0.006521816831082106, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00654, "step": 125, "tokens/total": 4310896, "tokens/train_per_sec_per_gpu": 32.99, "tokens/trainable": 56874} +{"epoch": 0.4921875, "grad_norm": 0.42977118492126465, "learning_rate": 9.09570303250602e-05, "loss": 0.01134882029145956, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.01141, "step": 126, "tokens/total": 4342912, "tokens/train_per_sec_per_gpu": 30.47, "tokens/trainable": 57358} +{"epoch": 0.49609375, "grad_norm": 0.607784628868103, "learning_rate": 9.078173985499394e-05, "loss": 0.01228694710880518, "memory/device_reserved (GiB)": 42.05, "memory/max_active (GiB)": 35.95, "memory/max_allocated (GiB)": 35.95, "ppl": 1.01236, "step": 127, "tokens/total": 4373312, "tokens/train_per_sec_per_gpu": 28.25, "tokens/trainable": 57788} +{"epoch": 0.5, "grad_norm": 0.35122331976890564, "learning_rate": 9.060496036067713e-05, "loss": 0.007026128005236387, "memory/device_reserved (GiB)": 42.05, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.00705, "step": 128, "tokens/total": 4410816, "tokens/train_per_sec_per_gpu": 22.35, "tokens/trainable": 58212} +{"epoch": 0.50390625, "grad_norm": 0.5537960529327393, "learning_rate": 9.042669919862615e-05, "loss": 0.01271489541977644, "memory/device_reserved (GiB)": 42.05, "memory/max_active (GiB)": 36.77, "memory/max_allocated (GiB)": 36.77, "ppl": 1.0128, "step": 129, "tokens/total": 4442704, "tokens/train_per_sec_per_gpu": 29.63, "tokens/trainable": 58676} +{"epoch": 0.5078125, "grad_norm": 0.7531278133392334, "learning_rate": 9.024696378701557e-05, "loss": 0.031096210703253746, "memory/device_reserved (GiB)": 41.41, "memory/max_active (GiB)": 39.48, "memory/max_allocated (GiB)": 39.48, "ppl": 1.03158, "step": 130, "tokens/total": 4478384, "tokens/train_per_sec_per_gpu": 23.78, "tokens/trainable": 59109} +{"epoch": 0.51171875, "grad_norm": 0.6801513433456421, "learning_rate": 9.006576160536948e-05, "loss": 0.016338981688022614, "memory/device_reserved (GiB)": 41.41, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.01647, "step": 131, "tokens/total": 4513328, "tokens/train_per_sec_per_gpu": 24.19, "tokens/trainable": 59558} +{"epoch": 0.515625, "grad_norm": 0.1788821667432785, "learning_rate": 8.988310019425035e-05, "loss": 0.0031332061626017094, "memory/device_reserved (GiB)": 41.41, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.00314, "step": 132, "tokens/total": 4550832, "tokens/train_per_sec_per_gpu": 24.0, "tokens/trainable": 60002} +{"epoch": 0.51953125, "grad_norm": 2.7837114334106445, "learning_rate": 8.969898715494506e-05, "loss": 0.0025569996796548367, "memory/device_reserved (GiB)": 41.7, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00256, "step": 133, "tokens/total": 4587968, "tokens/train_per_sec_per_gpu": 25.57, "tokens/trainable": 60483} +{"epoch": 0.5234375, "grad_norm": 0.17704996466636658, "learning_rate": 8.951343014914869e-05, "loss": 0.002783483359962702, "memory/device_reserved (GiB)": 41.7, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00279, "step": 134, "tokens/total": 4624544, "tokens/train_per_sec_per_gpu": 23.41, "tokens/trainable": 60949} +{"epoch": 0.52734375, "grad_norm": 0.031429365277290344, "learning_rate": 8.932643689864568e-05, "loss": 0.0003994953876826912, "memory/device_reserved (GiB)": 41.7, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.0004, "step": 135, "tokens/total": 4661296, "tokens/train_per_sec_per_gpu": 27.12, "tokens/trainable": 61436} +{"epoch": 0.53125, "grad_norm": 0.4009588956832886, "learning_rate": 8.913801518498845e-05, "loss": 0.0023803506046533585, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.00238, "step": 136, "tokens/total": 4700128, "tokens/train_per_sec_per_gpu": 21.18, "tokens/trainable": 61897} +{"epoch": 0.53515625, "grad_norm": 0.0628790333867073, "learning_rate": 8.894817284917364e-05, "loss": 0.0010260357521474361, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00103, "step": 137, "tokens/total": 4736912, "tokens/train_per_sec_per_gpu": 22.66, "tokens/trainable": 62351} +{"epoch": 0.5390625, "grad_norm": 0.7432213425636292, "learning_rate": 8.875691779131569e-05, "loss": 0.008341096341609955, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00838, "step": 138, "tokens/total": 4774192, "tokens/train_per_sec_per_gpu": 20.64, "tokens/trainable": 62783} +{"epoch": 0.54296875, "grad_norm": 0.6466282606124878, "learning_rate": 8.856425797031829e-05, "loss": 0.02251700684428215, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.02277, "step": 139, "tokens/total": 4809344, "tokens/train_per_sec_per_gpu": 25.62, "tokens/trainable": 63225} +{"epoch": 0.546875, "grad_norm": 0.8248280882835388, "learning_rate": 8.837020140354295e-05, "loss": 0.01600118726491928, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.01613, "step": 140, "tokens/total": 4845712, "tokens/train_per_sec_per_gpu": 26.72, "tokens/trainable": 63690} +{"epoch": 0.55078125, "grad_norm": 2.34547758102417, "learning_rate": 8.817475616647554e-05, "loss": 0.010203645564615726, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.01026, "step": 141, "tokens/total": 4880368, "tokens/train_per_sec_per_gpu": 26.42, "tokens/trainable": 64157} +{"epoch": 0.5546875, "grad_norm": 0.07213728874921799, "learning_rate": 8.797793039239017e-05, "loss": 0.0011283420026302338, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00113, "step": 142, "tokens/total": 4915424, "tokens/train_per_sec_per_gpu": 25.8, "tokens/trainable": 64605} +{"epoch": 0.55859375, "grad_norm": 0.18426597118377686, "learning_rate": 8.777973227201069e-05, "loss": 0.00290798582136631, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.00291, "step": 143, "tokens/total": 4951744, "tokens/train_per_sec_per_gpu": 27.64, "tokens/trainable": 65114} +{"epoch": 0.5625, "grad_norm": 0.4835839569568634, "learning_rate": 8.758017005316988e-05, "loss": 0.004749387037009001, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.00476, "step": 144, "tokens/total": 4985152, "tokens/train_per_sec_per_gpu": 28.12, "tokens/trainable": 65539} +{"epoch": 0.56640625, "grad_norm": 0.7576647996902466, "learning_rate": 8.737925204046629e-05, "loss": 0.008199897594749928, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00823, "step": 145, "tokens/total": 5019936, "tokens/train_per_sec_per_gpu": 23.87, "tokens/trainable": 65989} +{"epoch": 0.5703125, "grad_norm": 0.6816678047180176, "learning_rate": 8.717698659491851e-05, "loss": 0.006892533041536808, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00692, "step": 146, "tokens/total": 5057024, "tokens/train_per_sec_per_gpu": 22.26, "tokens/trainable": 66446} +{"epoch": 0.57421875, "grad_norm": 1.3354640007019043, "learning_rate": 8.697338213361735e-05, "loss": 0.024830404669046402, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.02514, "step": 147, "tokens/total": 5088320, "tokens/train_per_sec_per_gpu": 25.94, "tokens/trainable": 66894} +{"epoch": 0.578125, "grad_norm": 0.2409278154373169, "learning_rate": 8.676844712937552e-05, "loss": 0.0015284870751202106, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.39, "memory/max_allocated (GiB)": 39.39, "ppl": 1.00153, "step": 148, "tokens/total": 5128064, "tokens/train_per_sec_per_gpu": 24.65, "tokens/trainable": 67366} +{"epoch": 0.58203125, "grad_norm": 0.5328340530395508, "learning_rate": 8.656219011037509e-05, "loss": 0.004776919726282358, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00479, "step": 149, "tokens/total": 5165104, "tokens/train_per_sec_per_gpu": 23.37, "tokens/trainable": 67854} +{"epoch": 0.5859375, "grad_norm": 0.4087327718734741, "learning_rate": 8.63546196598125e-05, "loss": 0.00408003106713295, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00409, "step": 150, "tokens/total": 5202336, "tokens/train_per_sec_per_gpu": 24.9, "tokens/trainable": 68302} +{"epoch": 0.58984375, "grad_norm": 0.9515470266342163, "learning_rate": 8.614574441554145e-05, "loss": 0.015770725905895233, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.13, "memory/max_allocated (GiB)": 37.13, "ppl": 1.0159, "step": 151, "tokens/total": 5235440, "tokens/train_per_sec_per_gpu": 27.11, "tokens/trainable": 68727} +{"epoch": 0.59375, "grad_norm": 0.3813514709472656, "learning_rate": 8.593557306971349e-05, "loss": 0.010477673262357712, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.44, "memory/max_allocated (GiB)": 37.44, "ppl": 1.01053, "step": 152, "tokens/total": 5268416, "tokens/train_per_sec_per_gpu": 25.71, "tokens/trainable": 69182} +{"epoch": 0.59765625, "grad_norm": 0.5289509892463684, "learning_rate": 8.572411436841618e-05, "loss": 0.00975879468023777, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00981, "step": 153, "tokens/total": 5302128, "tokens/train_per_sec_per_gpu": 26.3, "tokens/trainable": 69657} +{"epoch": 0.6015625, "grad_norm": 0.3818823993206024, "learning_rate": 8.551137711130922e-05, "loss": 0.006170378066599369, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.2, "memory/max_allocated (GiB)": 38.2, "ppl": 1.00619, "step": 154, "tokens/total": 5333520, "tokens/train_per_sec_per_gpu": 23.67, "tokens/trainable": 70063} +{"epoch": 0.60546875, "grad_norm": 0.35435765981674194, "learning_rate": 8.529737015125824e-05, "loss": 0.009138536639511585, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00918, "step": 155, "tokens/total": 5367968, "tokens/train_per_sec_per_gpu": 25.59, "tokens/trainable": 70529} +{"epoch": 0.609375, "grad_norm": 0.5719999074935913, "learning_rate": 8.508210239396639e-05, "loss": 0.00933682732284069, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.00938, "step": 156, "tokens/total": 5405536, "tokens/train_per_sec_per_gpu": 23.99, "tokens/trainable": 70978} +{"epoch": 0.61328125, "grad_norm": 0.21003314852714539, "learning_rate": 8.486558279760375e-05, "loss": 0.0035992469638586044, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 36.07, "memory/max_allocated (GiB)": 36.07, "ppl": 1.00361, "step": 157, "tokens/total": 5436672, "tokens/train_per_sec_per_gpu": 26.42, "tokens/trainable": 71404} +{"epoch": 0.6171875, "grad_norm": 0.23666518926620483, "learning_rate": 8.464782037243449e-05, "loss": 0.007741324137896299, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.00777, "step": 158, "tokens/total": 5469280, "tokens/train_per_sec_per_gpu": 27.16, "tokens/trainable": 71845} +{"epoch": 0.62109375, "grad_norm": 0.8821848034858704, "learning_rate": 8.442882418044202e-05, "loss": 0.023578723892569542, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.02386, "step": 159, "tokens/total": 5501904, "tokens/train_per_sec_per_gpu": 29.08, "tokens/trainable": 72322} +{"epoch": 0.625, "grad_norm": 0.5120392441749573, "learning_rate": 8.420860333495179e-05, "loss": 0.0055718449875712395, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.00559, "step": 160, "tokens/total": 5534944, "tokens/train_per_sec_per_gpu": 26.51, "tokens/trainable": 72737} +{"epoch": 0.62890625, "grad_norm": 0.07627207785844803, "learning_rate": 8.398716700025208e-05, "loss": 0.0007033445872366428, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.0007, "step": 161, "tokens/total": 5568160, "tokens/train_per_sec_per_gpu": 25.49, "tokens/trainable": 73189} +{"epoch": 0.6328125, "grad_norm": 1.0357402563095093, "learning_rate": 8.376452439121266e-05, "loss": 0.027899833396077156, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 35.89, "memory/max_allocated (GiB)": 35.89, "ppl": 1.02829, "step": 162, "tokens/total": 5598256, "tokens/train_per_sec_per_gpu": 30.89, "tokens/trainable": 73640} +{"epoch": 0.63671875, "grad_norm": 1.0670616626739502, "learning_rate": 8.354068477290124e-05, "loss": 0.005003449972718954, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00502, "step": 163, "tokens/total": 5632560, "tokens/train_per_sec_per_gpu": 27.57, "tokens/trainable": 74099} +{"epoch": 0.640625, "grad_norm": 0.32847949862480164, "learning_rate": 8.331565746019807e-05, "loss": 0.004998259246349335, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.52, "memory/max_allocated (GiB)": 36.52, "ppl": 1.00501, "step": 164, "tokens/total": 5664464, "tokens/train_per_sec_per_gpu": 26.96, "tokens/trainable": 74557} +{"epoch": 0.64453125, "grad_norm": 0.6201031804084778, "learning_rate": 8.308945181740812e-05, "loss": 0.007850141264498234, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.00788, "step": 165, "tokens/total": 5699040, "tokens/train_per_sec_per_gpu": 27.1, "tokens/trainable": 74999} +{"epoch": 0.6484375, "grad_norm": 2.052659034729004, "learning_rate": 8.286207725787153e-05, "loss": 0.03148172050714493, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.03198, "step": 166, "tokens/total": 5735968, "tokens/train_per_sec_per_gpu": 22.33, "tokens/trainable": 75444} +{"epoch": 0.65234375, "grad_norm": 0.29788774251937866, "learning_rate": 8.263354324357182e-05, "loss": 0.004827685188502073, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00484, "step": 167, "tokens/total": 5770688, "tokens/train_per_sec_per_gpu": 25.27, "tokens/trainable": 75879} +{"epoch": 0.65625, "grad_norm": 0.04621196910738945, "learning_rate": 8.240385928474219e-05, "loss": 0.0008008757140487432, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.0008, "step": 168, "tokens/total": 5803824, "tokens/train_per_sec_per_gpu": 26.94, "tokens/trainable": 76293} +{"epoch": 0.66015625, "grad_norm": 0.4789822995662689, "learning_rate": 8.217303493946967e-05, "loss": 0.0073341503739356995, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.00736, "step": 169, "tokens/total": 5838256, "tokens/train_per_sec_per_gpu": 26.16, "tokens/trainable": 76741} +{"epoch": 0.6640625, "grad_norm": 0.487662136554718, "learning_rate": 8.194107981329746e-05, "loss": 0.008744516409933567, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00878, "step": 170, "tokens/total": 5876224, "tokens/train_per_sec_per_gpu": 23.27, "tokens/trainable": 77195} +{"epoch": 0.66796875, "grad_norm": 0.15091979503631592, "learning_rate": 8.170800355882518e-05, "loss": 0.0018482813611626625, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.24, "memory/max_allocated (GiB)": 37.24, "ppl": 1.00185, "step": 171, "tokens/total": 5910448, "tokens/train_per_sec_per_gpu": 30.34, "tokens/trainable": 77662} +{"epoch": 0.671875, "grad_norm": 1.6098923683166504, "learning_rate": 8.147381587530713e-05, "loss": 0.02331741899251938, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.02359, "step": 172, "tokens/total": 5944256, "tokens/train_per_sec_per_gpu": 26.5, "tokens/trainable": 78116} +{"epoch": 0.67578125, "grad_norm": 0.863294243812561, "learning_rate": 8.123852650824877e-05, "loss": 0.03006996028125286, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.03053, "step": 173, "tokens/total": 5978288, "tokens/train_per_sec_per_gpu": 28.37, "tokens/trainable": 78599} +{"epoch": 0.6796875, "grad_norm": 0.45596686005592346, "learning_rate": 8.100214524900103e-05, "loss": 0.005359282251447439, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.52, "memory/max_allocated (GiB)": 38.52, "ppl": 1.00537, "step": 174, "tokens/total": 6013680, "tokens/train_per_sec_per_gpu": 22.19, "tokens/trainable": 79030} +{"epoch": 0.68359375, "grad_norm": 0.5196447968482971, "learning_rate": 8.076468193435301e-05, "loss": 0.004709427710622549, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.00472, "step": 175, "tokens/total": 6045072, "tokens/train_per_sec_per_gpu": 25.94, "tokens/trainable": 79454} +{"epoch": 0.6875, "grad_norm": 0.746830940246582, "learning_rate": 8.052614644612253e-05, "loss": 0.017462700605392456, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.01762, "step": 176, "tokens/total": 6082336, "tokens/train_per_sec_per_gpu": 23.28, "tokens/trainable": 79915} +{"epoch": 0.69140625, "grad_norm": 0.19366896152496338, "learning_rate": 8.028654871074489e-05, "loss": 0.0034618708305060863, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00347, "step": 177, "tokens/total": 6117024, "tokens/train_per_sec_per_gpu": 25.44, "tokens/trainable": 80356} +{"epoch": 0.6953125, "grad_norm": 1.3230122327804565, "learning_rate": 8.004589869885986e-05, "loss": 0.007347984705120325, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.49, "memory/max_allocated (GiB)": 36.49, "ppl": 1.00738, "step": 178, "tokens/total": 6149136, "tokens/train_per_sec_per_gpu": 26.59, "tokens/trainable": 80774} +{"epoch": 0.69921875, "grad_norm": 0.40049025416374207, "learning_rate": 7.980420642489674e-05, "loss": 0.011649958789348602, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.01172, "step": 179, "tokens/total": 6187776, "tokens/train_per_sec_per_gpu": 23.32, "tokens/trainable": 81247} +{"epoch": 0.703125, "grad_norm": 0.08379040658473969, "learning_rate": 7.95614819466576e-05, "loss": 0.001453642500564456, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00145, "step": 180, "tokens/total": 6221728, "tokens/train_per_sec_per_gpu": 28.41, "tokens/trainable": 81713} +{"epoch": 0.70703125, "grad_norm": 0.0880313292145729, "learning_rate": 7.931773536489872e-05, "loss": 0.0014296656008809805, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.00143, "step": 181, "tokens/total": 6255920, "tokens/train_per_sec_per_gpu": 24.88, "tokens/trainable": 82158} +{"epoch": 0.7109375, "grad_norm": 0.3473573625087738, "learning_rate": 7.907297682291035e-05, "loss": 0.014732533134520054, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.01484, "step": 182, "tokens/total": 6291104, "tokens/train_per_sec_per_gpu": 26.49, "tokens/trainable": 82621} +{"epoch": 0.71484375, "grad_norm": 0.10174426436424255, "learning_rate": 7.882721650609442e-05, "loss": 0.001927317469380796, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.00193, "step": 183, "tokens/total": 6330976, "tokens/train_per_sec_per_gpu": 23.18, "tokens/trainable": 83122} +{"epoch": 0.71875, "grad_norm": 0.18113133311271667, "learning_rate": 7.85804646415409e-05, "loss": 0.004166948609054089, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.00418, "step": 184, "tokens/total": 6366160, "tokens/train_per_sec_per_gpu": 26.74, "tokens/trainable": 83578} +{"epoch": 0.72265625, "grad_norm": 0.14779509603977203, "learning_rate": 7.833273149760207e-05, "loss": 0.002330516465008259, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.00233, "step": 185, "tokens/total": 6397280, "tokens/train_per_sec_per_gpu": 28.71, "tokens/trainable": 84048} +{"epoch": 0.7265625, "grad_norm": 1.626979112625122, "learning_rate": 7.808402738346527e-05, "loss": 0.04092442989349365, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.04177, "step": 186, "tokens/total": 6435632, "tokens/train_per_sec_per_gpu": 23.33, "tokens/trainable": 84497} +{"epoch": 0.73046875, "grad_norm": 0.384996235370636, "learning_rate": 7.783436264872382e-05, "loss": 0.005231700837612152, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.00525, "step": 187, "tokens/total": 6472752, "tokens/train_per_sec_per_gpu": 24.75, "tokens/trainable": 84983} +{"epoch": 0.734375, "grad_norm": 0.5180711150169373, "learning_rate": 7.758374768294647e-05, "loss": 0.007542405277490616, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.05, "memory/max_allocated (GiB)": 38.05, "ppl": 1.00757, "step": 188, "tokens/total": 6508528, "tokens/train_per_sec_per_gpu": 26.96, "tokens/trainable": 85462} +{"epoch": 0.73828125, "grad_norm": 0.1751675009727478, "learning_rate": 7.733219291524489e-05, "loss": 0.0020683053880929947, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.59, "memory/max_allocated (GiB)": 37.59, "ppl": 1.00207, "step": 189, "tokens/total": 6542048, "tokens/train_per_sec_per_gpu": 27.36, "tokens/trainable": 85892} +{"epoch": 0.7421875, "grad_norm": 0.946224570274353, "learning_rate": 7.707970881383977e-05, "loss": 0.0038683360908180475, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.57, "memory/max_allocated (GiB)": 36.57, "ppl": 1.00388, "step": 190, "tokens/total": 6573792, "tokens/train_per_sec_per_gpu": 28.08, "tokens/trainable": 86341} +{"epoch": 0.74609375, "grad_norm": 0.1638849973678589, "learning_rate": 7.682630588562518e-05, "loss": 0.002523337956517935, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.00253, "step": 191, "tokens/total": 6608848, "tokens/train_per_sec_per_gpu": 24.32, "tokens/trainable": 86772} +{"epoch": 0.75, "grad_norm": 0.3366672694683075, "learning_rate": 7.657199467573129e-05, "loss": 0.0035858163610100746, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.00359, "step": 192, "tokens/total": 6642560, "tokens/train_per_sec_per_gpu": 26.13, "tokens/trainable": 87244} +{"epoch": 0.75390625, "grad_norm": 0.36759498715400696, "learning_rate": 7.631678576708561e-05, "loss": 0.008632514625787735, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00867, "step": 193, "tokens/total": 6676272, "tokens/train_per_sec_per_gpu": 28.95, "tokens/trainable": 87736} +{"epoch": 0.7578125, "grad_norm": 0.03147542476654053, "learning_rate": 7.606068977997255e-05, "loss": 0.00035999377723783255, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00036, "step": 194, "tokens/total": 6713568, "tokens/train_per_sec_per_gpu": 25.66, "tokens/trainable": 88214} +{"epoch": 0.76171875, "grad_norm": 1.1189862489700317, "learning_rate": 7.580371737159148e-05, "loss": 0.005761496257036924, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00578, "step": 195, "tokens/total": 6749296, "tokens/train_per_sec_per_gpu": 22.93, "tokens/trainable": 88625} +{"epoch": 0.765625, "grad_norm": 0.06243761628866196, "learning_rate": 7.554587923561324e-05, "loss": 0.0007713414961472154, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00077, "step": 196, "tokens/total": 6783952, "tokens/train_per_sec_per_gpu": 24.64, "tokens/trainable": 89063} +{"epoch": 0.76953125, "grad_norm": 0.30729177594184875, "learning_rate": 7.528718610173511e-05, "loss": 0.0030959027353674173, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.48, "memory/max_allocated (GiB)": 37.48, "ppl": 1.0031, "step": 197, "tokens/total": 6819152, "tokens/train_per_sec_per_gpu": 22.78, "tokens/trainable": 89484} +{"epoch": 0.7734375, "grad_norm": 0.03302743658423424, "learning_rate": 7.502764873523431e-05, "loss": 0.0004221628187224269, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.00042, "step": 198, "tokens/total": 6853440, "tokens/train_per_sec_per_gpu": 27.19, "tokens/trainable": 89922} +{"epoch": 0.77734375, "grad_norm": 3.5455758571624756, "learning_rate": 7.476727793652011e-05, "loss": 0.01648111268877983, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.71, "memory/max_allocated (GiB)": 36.71, "ppl": 1.01662, "step": 199, "tokens/total": 6886272, "tokens/train_per_sec_per_gpu": 28.12, "tokens/trainable": 90401} +{"epoch": 0.78125, "grad_norm": 0.7469642162322998, "learning_rate": 7.450608454068415e-05, "loss": 0.008629895746707916, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.00867, "step": 200, "tokens/total": 6920624, "tokens/train_per_sec_per_gpu": 26.64, "tokens/trainable": 90842} +{"epoch": 0.78515625, "grad_norm": 0.6895644068717957, "learning_rate": 7.424407941704987e-05, "loss": 0.0075179096311330795, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.00755, "step": 201, "tokens/total": 6957328, "tokens/train_per_sec_per_gpu": 27.29, "tokens/trainable": 91343} +{"epoch": 0.7890625, "grad_norm": 0.7283216714859009, "learning_rate": 7.398127346871986e-05, "loss": 0.011677050031721592, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.01175, "step": 202, "tokens/total": 6994048, "tokens/train_per_sec_per_gpu": 24.37, "tokens/trainable": 91798} +{"epoch": 0.79296875, "grad_norm": 0.30082961916923523, "learning_rate": 7.371767763212238e-05, "loss": 0.0014750645495951176, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.00148, "step": 203, "tokens/total": 7029040, "tokens/train_per_sec_per_gpu": 26.52, "tokens/trainable": 92251} +{"epoch": 0.796875, "grad_norm": 0.41350457072257996, "learning_rate": 7.345330287655617e-05, "loss": 0.004136312287300825, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.00414, "step": 204, "tokens/total": 7067104, "tokens/train_per_sec_per_gpu": 22.68, "tokens/trainable": 92714} +{"epoch": 0.80078125, "grad_norm": 0.05340210348367691, "learning_rate": 7.31881602037339e-05, "loss": 0.0004055093741044402, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.00041, "step": 205, "tokens/total": 7099136, "tokens/train_per_sec_per_gpu": 26.48, "tokens/trainable": 93154} +{"epoch": 0.8046875, "grad_norm": 0.5874925851821899, "learning_rate": 7.29222606473245e-05, "loss": 0.004646359011530876, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.00466, "step": 206, "tokens/total": 7132208, "tokens/train_per_sec_per_gpu": 26.47, "tokens/trainable": 93599} +{"epoch": 0.80859375, "grad_norm": 0.08912669867277145, "learning_rate": 7.265561527249383e-05, "loss": 0.0009423486771993339, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.00094, "step": 207, "tokens/total": 7166256, "tokens/train_per_sec_per_gpu": 27.72, "tokens/trainable": 94036} +{"epoch": 0.8125, "grad_norm": 0.17996619641780853, "learning_rate": 7.238823517544436e-05, "loss": 0.0023069377057254314, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.00231, "step": 208, "tokens/total": 7203152, "tokens/train_per_sec_per_gpu": 28.21, "tokens/trainable": 94506} +{"epoch": 0.81640625, "grad_norm": 0.0581505112349987, "learning_rate": 7.212013148295333e-05, "loss": 0.0008375308243557811, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.00084, "step": 209, "tokens/total": 7237264, "tokens/train_per_sec_per_gpu": 29.76, "tokens/trainable": 94988} +{"epoch": 0.8203125, "grad_norm": 0.7706319093704224, "learning_rate": 7.185131535190975e-05, "loss": 0.004703104496002197, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00471, "step": 210, "tokens/total": 7269664, "tokens/train_per_sec_per_gpu": 29.94, "tokens/trainable": 95497} +{"epoch": 0.82421875, "grad_norm": 0.6247218251228333, "learning_rate": 7.158179796885005e-05, "loss": 0.0068317586556077, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.00686, "step": 211, "tokens/total": 7304080, "tokens/train_per_sec_per_gpu": 27.46, "tokens/trainable": 95948} +{"epoch": 0.828125, "grad_norm": 0.12992911040782928, "learning_rate": 7.131159054949273e-05, "loss": 0.0009680531220510602, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00097, "step": 212, "tokens/total": 7342480, "tokens/train_per_sec_per_gpu": 22.3, "tokens/trainable": 96422} +{"epoch": 0.83203125, "grad_norm": 1.910191535949707, "learning_rate": 7.104070433827139e-05, "loss": 0.005330915562808514, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00535, "step": 213, "tokens/total": 7378032, "tokens/train_per_sec_per_gpu": 22.94, "tokens/trainable": 96821} +{"epoch": 0.8359375, "grad_norm": 0.17790497839450836, "learning_rate": 7.076915060786705e-05, "loss": 0.001983305439352989, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.96, "memory/max_allocated (GiB)": 37.96, "ppl": 1.00199, "step": 214, "tokens/total": 7410928, "tokens/train_per_sec_per_gpu": 24.22, "tokens/trainable": 97263} +{"epoch": 0.83984375, "grad_norm": 0.4452838599681854, "learning_rate": 7.049694065873882e-05, "loss": 0.003958427347242832, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.00397, "step": 215, "tokens/total": 7445664, "tokens/train_per_sec_per_gpu": 27.92, "tokens/trainable": 97720} +{"epoch": 0.84375, "grad_norm": 0.42124757170677185, "learning_rate": 7.022408581865382e-05, "loss": 0.005655376240611076, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.69, "memory/max_allocated (GiB)": 38.69, "ppl": 1.00567, "step": 216, "tokens/total": 7479984, "tokens/train_per_sec_per_gpu": 25.44, "tokens/trainable": 98194} +{"epoch": 0.84765625, "grad_norm": 1.341488242149353, "learning_rate": 6.99505974422157e-05, "loss": 0.026915596798062325, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.02728, "step": 217, "tokens/total": 7514144, "tokens/train_per_sec_per_gpu": 28.37, "tokens/trainable": 98667} +{"epoch": 0.8515625, "grad_norm": 0.2156139314174652, "learning_rate": 6.967648691039213e-05, "loss": 0.0014719966566190124, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.73, "memory/max_allocated (GiB)": 36.73, "ppl": 1.00147, "step": 218, "tokens/total": 7546256, "tokens/train_per_sec_per_gpu": 27.9, "tokens/trainable": 99100} +{"epoch": 0.85546875, "grad_norm": 0.251765638589859, "learning_rate": 6.940176563004123e-05, "loss": 0.0020719871390610933, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00207, "step": 219, "tokens/total": 7580432, "tokens/train_per_sec_per_gpu": 25.93, "tokens/trainable": 99532} +{"epoch": 0.859375, "grad_norm": 0.3668922185897827, "learning_rate": 6.912644503343682e-05, "loss": 0.004810224287211895, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.00482, "step": 220, "tokens/total": 7617904, "tokens/train_per_sec_per_gpu": 23.44, "tokens/trainable": 99970} +{"epoch": 0.86328125, "grad_norm": 0.3343198299407959, "learning_rate": 6.885053657779273e-05, "loss": 0.003787202760577202, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.00379, "step": 221, "tokens/total": 7649696, "tokens/train_per_sec_per_gpu": 28.35, "tokens/trainable": 100442} +{"epoch": 0.8671875, "grad_norm": 0.28965359926223755, "learning_rate": 6.857405174478604e-05, "loss": 0.0007998555665835738, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.0008, "step": 222, "tokens/total": 7684464, "tokens/train_per_sec_per_gpu": 24.31, "tokens/trainable": 100895} +{"epoch": 0.87109375, "grad_norm": 0.7953235507011414, "learning_rate": 6.82970020400792e-05, "loss": 0.016118250787258148, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.01625, "step": 223, "tokens/total": 7721936, "tokens/train_per_sec_per_gpu": 22.2, "tokens/trainable": 101342} +{"epoch": 0.875, "grad_norm": 0.3991866111755371, "learning_rate": 6.801939899284132e-05, "loss": 0.004242321010679007, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.34, "memory/max_allocated (GiB)": 38.34, "ppl": 1.00425, "step": 224, "tokens/total": 7758160, "tokens/train_per_sec_per_gpu": 26.53, "tokens/trainable": 101848} +{"epoch": 0.87890625, "grad_norm": 0.30168086290359497, "learning_rate": 6.774125415526827e-05, "loss": 0.0018655563471838832, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00187, "step": 225, "tokens/total": 7791216, "tokens/train_per_sec_per_gpu": 27.73, "tokens/trainable": 102319} +{"epoch": 0.8828125, "grad_norm": 0.08504792302846909, "learning_rate": 6.746257910210214e-05, "loss": 0.00014972398639656603, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.12, "memory/max_allocated (GiB)": 37.12, "ppl": 1.00015, "step": 226, "tokens/total": 7824400, "tokens/train_per_sec_per_gpu": 27.98, "tokens/trainable": 102769} +{"epoch": 0.88671875, "grad_norm": 0.2597670257091522, "learning_rate": 6.718338543014937e-05, "loss": 0.00228631729260087, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 35.75, "memory/max_allocated (GiB)": 35.75, "ppl": 1.00229, "step": 227, "tokens/total": 7854672, "tokens/train_per_sec_per_gpu": 30.83, "tokens/trainable": 103219} +{"epoch": 0.890625, "grad_norm": 0.31436213850975037, "learning_rate": 6.69036847577983e-05, "loss": 0.008573812432587147, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.00861, "step": 228, "tokens/total": 7888592, "tokens/train_per_sec_per_gpu": 26.71, "tokens/trainable": 103675} +{"epoch": 0.89453125, "grad_norm": 0.9255815744400024, "learning_rate": 6.662348872453553e-05, "loss": 0.00995219312608242, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.33, "memory/max_allocated (GiB)": 38.33, "ppl": 1.01, "step": 229, "tokens/total": 7924848, "tokens/train_per_sec_per_gpu": 27.46, "tokens/trainable": 104136} +{"epoch": 0.8984375, "grad_norm": 0.197851300239563, "learning_rate": 6.63428089904618e-05, "loss": 0.002080680802464485, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.00208, "step": 230, "tokens/total": 7963856, "tokens/train_per_sec_per_gpu": 24.08, "tokens/trainable": 104588} +{"epoch": 0.90234375, "grad_norm": 2.1193206310272217, "learning_rate": 6.60616572358065e-05, "loss": 0.004067631438374519, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.00408, "step": 231, "tokens/total": 7998320, "tokens/train_per_sec_per_gpu": 25.58, "tokens/trainable": 104988} +{"epoch": 0.90625, "grad_norm": 0.33081743121147156, "learning_rate": 6.578004516044172e-05, "loss": 0.002605854067951441, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00261, "step": 232, "tokens/total": 8034096, "tokens/train_per_sec_per_gpu": 27.5, "tokens/trainable": 105455} +{"epoch": 0.91015625, "grad_norm": 0.5856030583381653, "learning_rate": 6.549798448339548e-05, "loss": 0.006030384451150894, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.59, "memory/max_allocated (GiB)": 38.59, "ppl": 1.00605, "step": 233, "tokens/total": 8071360, "tokens/train_per_sec_per_gpu": 24.97, "tokens/trainable": 105918} +{"epoch": 0.9140625, "grad_norm": 0.3069413900375366, "learning_rate": 6.521548694236384e-05, "loss": 0.0070582106709480286, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00708, "step": 234, "tokens/total": 8108560, "tokens/train_per_sec_per_gpu": 23.28, "tokens/trainable": 106355} +{"epoch": 0.91796875, "grad_norm": 0.44584178924560547, "learning_rate": 6.493256429322259e-05, "loss": 0.012412487529218197, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.01249, "step": 235, "tokens/total": 8144816, "tokens/train_per_sec_per_gpu": 22.15, "tokens/trainable": 106786} +{"epoch": 0.921875, "grad_norm": 0.22023862600326538, "learning_rate": 6.464922830953799e-05, "loss": 0.002629626542329788, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.00263, "step": 236, "tokens/total": 8183680, "tokens/train_per_sec_per_gpu": 22.35, "tokens/trainable": 107219} +{"epoch": 0.92578125, "grad_norm": 0.3637104630470276, "learning_rate": 6.436549078207688e-05, "loss": 0.004858643747866154, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00487, "step": 237, "tokens/total": 8216688, "tokens/train_per_sec_per_gpu": 28.69, "tokens/trainable": 107679} +{"epoch": 0.9296875, "grad_norm": 0.6803686618804932, "learning_rate": 6.408136351831592e-05, "loss": 0.007848129607737064, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.00788, "step": 238, "tokens/total": 8249104, "tokens/train_per_sec_per_gpu": 24.04, "tokens/trainable": 108119} +{"epoch": 0.93359375, "grad_norm": 0.4477992355823517, "learning_rate": 6.379685834195036e-05, "loss": 0.006576542276889086, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.0066, "step": 239, "tokens/total": 8288528, "tokens/train_per_sec_per_gpu": 24.44, "tokens/trainable": 108626} +{"epoch": 0.9375, "grad_norm": 0.0623740516602993, "learning_rate": 6.351198709240186e-05, "loss": 0.0007659515831619501, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.00077, "step": 240, "tokens/total": 8328464, "tokens/train_per_sec_per_gpu": 21.03, "tokens/trainable": 109079} +{"epoch": 0.94140625, "grad_norm": 0.24128469824790955, "learning_rate": 6.32267616243259e-05, "loss": 0.0025228087324649096, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.00253, "step": 241, "tokens/total": 8361264, "tokens/train_per_sec_per_gpu": 28.61, "tokens/trainable": 109542} +{"epoch": 0.9453125, "grad_norm": 0.6835299134254456, "learning_rate": 6.294119380711849e-05, "loss": 0.00659969262778759, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.39, "memory/max_allocated (GiB)": 38.39, "ppl": 1.00662, "step": 242, "tokens/total": 8395552, "tokens/train_per_sec_per_gpu": 23.72, "tokens/trainable": 109969} +{"epoch": 0.94921875, "grad_norm": 0.17613984644412994, "learning_rate": 6.265529552442209e-05, "loss": 0.0037468865048140287, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.00375, "step": 243, "tokens/total": 8430656, "tokens/train_per_sec_per_gpu": 27.91, "tokens/trainable": 110431} +{"epoch": 0.953125, "grad_norm": 0.16451582312583923, "learning_rate": 6.236907867363127e-05, "loss": 0.0017755480948835611, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.17, "memory/max_allocated (GiB)": 38.17, "ppl": 1.00178, "step": 244, "tokens/total": 8465232, "tokens/train_per_sec_per_gpu": 26.51, "tokens/trainable": 110877} +{"epoch": 0.95703125, "grad_norm": 0.47878792881965637, "learning_rate": 6.208255516539749e-05, "loss": 0.026234017685055733, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.02658, "step": 245, "tokens/total": 8497552, "tokens/train_per_sec_per_gpu": 26.46, "tokens/trainable": 111350} +{"epoch": 0.9609375, "grad_norm": 0.08385979384183884, "learning_rate": 6.179573692313344e-05, "loss": 0.0009442891459912062, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.8, "memory/max_allocated (GiB)": 38.8, "ppl": 1.00094, "step": 246, "tokens/total": 8534336, "tokens/train_per_sec_per_gpu": 25.92, "tokens/trainable": 111798} +{"epoch": 0.96484375, "grad_norm": 0.418268620967865, "learning_rate": 6.150863588251694e-05, "loss": 0.004126594867557287, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.00414, "step": 247, "tokens/total": 8574224, "tokens/train_per_sec_per_gpu": 22.56, "tokens/trainable": 112264} +{"epoch": 0.96875, "grad_norm": 0.2047712653875351, "learning_rate": 6.122126399099419e-05, "loss": 0.001987085910513997, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.00199, "step": 248, "tokens/total": 8613616, "tokens/train_per_sec_per_gpu": 26.71, "tokens/trainable": 112772} +{"epoch": 0.97265625, "grad_norm": 0.2959277331829071, "learning_rate": 6.0933633207282615e-05, "loss": 0.004163263365626335, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.00417, "step": 249, "tokens/total": 8650048, "tokens/train_per_sec_per_gpu": 27.25, "tokens/trainable": 113278} +{"epoch": 0.9765625, "grad_norm": 0.1236204132437706, "learning_rate": 6.064575550087316e-05, "loss": 0.0017477760557085276, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00175, "step": 250, "tokens/total": 8686688, "tokens/train_per_sec_per_gpu": 23.61, "tokens/trainable": 113716} +{"epoch": 0.98046875, "grad_norm": 0.12932878732681274, "learning_rate": 6.0357642851532245e-05, "loss": 0.0013420265167951584, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00134, "step": 251, "tokens/total": 8723376, "tokens/train_per_sec_per_gpu": 25.62, "tokens/trainable": 114207} +{"epoch": 0.984375, "grad_norm": 0.1732017695903778, "learning_rate": 6.0069307248803294e-05, "loss": 0.0021860923152416945, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00219, "step": 252, "tokens/total": 8758736, "tokens/train_per_sec_per_gpu": 21.84, "tokens/trainable": 114628} +{"epoch": 0.98828125, "grad_norm": 0.15961100161075592, "learning_rate": 5.9780760691507635e-05, "loss": 0.0011748055694624782, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.33, "memory/max_allocated (GiB)": 36.33, "ppl": 1.00118, "step": 253, "tokens/total": 8790480, "tokens/train_per_sec_per_gpu": 26.62, "tokens/trainable": 115060} +{"epoch": 0.9921875, "grad_norm": 0.14212636649608612, "learning_rate": 5.9492015187245334e-05, "loss": 0.0006607507821172476, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00066, "step": 254, "tokens/total": 8828224, "tokens/train_per_sec_per_gpu": 23.17, "tokens/trainable": 115514} +{"epoch": 0.99609375, "grad_norm": 0.21213819086551666, "learning_rate": 5.920308275189541e-05, "loss": 0.002671959111467004, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.00268, "step": 255, "tokens/total": 8860400, "tokens/train_per_sec_per_gpu": 25.38, "tokens/trainable": 115915} +{"epoch": 1.0, "grad_norm": 0.2322702705860138, "learning_rate": 5.8913975409115874e-05, "loss": 0.0012675366597250104, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.00127, "step": 256, "tokens/total": 8893760, "tokens/train_per_sec_per_gpu": 25.24, "tokens/trainable": 116378} +{"epoch": 1.00390625, "grad_norm": 0.019989045336842537, "learning_rate": 5.8624705189843395e-05, "loss": 0.00027125311316922307, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 1.00027, "step": 257, "tokens/total": 8930976, "tokens/train_per_sec_per_gpu": 24.52, "tokens/trainable": 116865} +{"epoch": 1.0078125, "grad_norm": 0.025256935507059097, "learning_rate": 5.833528413179249e-05, "loss": 0.00018639687914401293, "memory/device_reserved (GiB)": 42.25, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00019, "step": 258, "tokens/total": 8967472, "tokens/train_per_sec_per_gpu": 27.65, "tokens/trainable": 117363} +{"epoch": 1.01171875, "grad_norm": 0.517684280872345, "learning_rate": 5.80457242789548e-05, "loss": 0.002601926913484931, "memory/device_reserved (GiB)": 42.31, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.00261, "step": 259, "tokens/total": 9007456, "tokens/train_per_sec_per_gpu": 23.69, "tokens/trainable": 117860} +{"epoch": 1.015625, "grad_norm": 0.09724453091621399, "learning_rate": 5.77560376810977e-05, "loss": 0.0005972091457806528, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 1.0006, "step": 260, "tokens/total": 9041104, "tokens/train_per_sec_per_gpu": 29.02, "tokens/trainable": 118324} +{"epoch": 1.01953125, "grad_norm": 0.1394912600517273, "learning_rate": 5.7466236393263005e-05, "loss": 0.00011022289254469797, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.00011, "step": 261, "tokens/total": 9076176, "tokens/train_per_sec_per_gpu": 25.4, "tokens/trainable": 118770} +{"epoch": 1.0234375, "grad_norm": 0.04598289355635643, "learning_rate": 5.717633247526522e-05, "loss": 0.00022856226132716984, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00023, "step": 262, "tokens/total": 9107472, "tokens/train_per_sec_per_gpu": 27.69, "tokens/trainable": 119206} +{"epoch": 1.02734375, "grad_norm": 0.03864477947354317, "learning_rate": 5.688633799118971e-05, "loss": 8.874956984072924e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.00009, "step": 263, "tokens/total": 9139552, "tokens/train_per_sec_per_gpu": 23.18, "tokens/trainable": 119610} +{"epoch": 1.03125, "grad_norm": 0.01034588273614645, "learning_rate": 5.659626500889066e-05, "loss": 8.789426647126675e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00009, "step": 264, "tokens/total": 9173424, "tokens/train_per_sec_per_gpu": 25.67, "tokens/trainable": 120080} +{"epoch": 1.03515625, "grad_norm": 0.012896646745502949, "learning_rate": 5.6306125599488905e-05, "loss": 0.00013213377678766847, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.00013, "step": 265, "tokens/total": 9203984, "tokens/train_per_sec_per_gpu": 32.0, "tokens/trainable": 120519} +{"epoch": 1.0390625, "grad_norm": 0.21945008635520935, "learning_rate": 5.601593183686955e-05, "loss": 0.0013681945856660604, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.69, "memory/max_allocated (GiB)": 38.69, "ppl": 1.00137, "step": 266, "tokens/total": 9241248, "tokens/train_per_sec_per_gpu": 25.83, "tokens/trainable": 120995} +{"epoch": 1.04296875, "grad_norm": 0.2794446349143982, "learning_rate": 5.572569579717961e-05, "loss": 0.0014591942308470607, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.18, "memory/max_allocated (GiB)": 39.18, "ppl": 1.00146, "step": 267, "tokens/total": 9276768, "tokens/train_per_sec_per_gpu": 24.37, "tokens/trainable": 121459} +{"epoch": 1.046875, "grad_norm": 0.7416685819625854, "learning_rate": 5.543542955832538e-05, "loss": 0.0019302713917568326, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00193, "step": 268, "tokens/total": 9316528, "tokens/train_per_sec_per_gpu": 21.54, "tokens/trainable": 121902} +{"epoch": 1.05078125, "grad_norm": 0.5965358018875122, "learning_rate": 5.514514519946986e-05, "loss": 0.018134860321879387, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.0183, "step": 269, "tokens/total": 9356352, "tokens/train_per_sec_per_gpu": 25.56, "tokens/trainable": 122399} +{"epoch": 1.0546875, "grad_norm": 0.4553667902946472, "learning_rate": 5.485485480053015e-05, "loss": 0.003478935221210122, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.00348, "step": 270, "tokens/total": 9394576, "tokens/train_per_sec_per_gpu": 21.78, "tokens/trainable": 122804} +{"epoch": 1.05859375, "grad_norm": 0.03494894132018089, "learning_rate": 5.4564570441674645e-05, "loss": 0.00023832799342926592, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.92, "memory/max_allocated (GiB)": 36.92, "ppl": 1.00024, "step": 271, "tokens/total": 9427840, "tokens/train_per_sec_per_gpu": 25.39, "tokens/trainable": 123244} +{"epoch": 1.0625, "grad_norm": 0.010820046998560429, "learning_rate": 5.42743042028204e-05, "loss": 0.00012408418115228415, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.00012, "step": 272, "tokens/total": 9463536, "tokens/train_per_sec_per_gpu": 20.41, "tokens/trainable": 123683} +{"epoch": 1.06640625, "grad_norm": 0.021344300359487534, "learning_rate": 5.3984068163130464e-05, "loss": 0.00022381721646524966, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.09, "memory/max_allocated (GiB)": 37.09, "ppl": 1.00022, "step": 273, "tokens/total": 9497248, "tokens/train_per_sec_per_gpu": 28.01, "tokens/trainable": 124153} +{"epoch": 1.0703125, "grad_norm": 0.25106629729270935, "learning_rate": 5.369387440051111e-05, "loss": 0.0012643461814150214, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.00127, "step": 274, "tokens/total": 9530784, "tokens/train_per_sec_per_gpu": 25.34, "tokens/trainable": 124616} +{"epoch": 1.07421875, "grad_norm": 0.04161720722913742, "learning_rate": 5.340373499110935e-05, "loss": 9.445970499655232e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00009, "step": 275, "tokens/total": 9566656, "tokens/train_per_sec_per_gpu": 26.28, "tokens/trainable": 125109} +{"epoch": 1.078125, "grad_norm": 0.0028409413062036037, "learning_rate": 5.3113662008810304e-05, "loss": 3.998784450232051e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00004, "step": 276, "tokens/total": 9606672, "tokens/train_per_sec_per_gpu": 24.87, "tokens/trainable": 125591} +{"epoch": 1.08203125, "grad_norm": 0.0873127207159996, "learning_rate": 5.282366752473479e-05, "loss": 0.0002653416304383427, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 1.00027, "step": 277, "tokens/total": 9643680, "tokens/train_per_sec_per_gpu": 24.4, "tokens/trainable": 126065} +{"epoch": 1.0859375, "grad_norm": 0.05592695623636246, "learning_rate": 5.2533763606737005e-05, "loss": 0.00038170188781805336, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00038, "step": 278, "tokens/total": 9683200, "tokens/train_per_sec_per_gpu": 19.16, "tokens/trainable": 126480} +{"epoch": 1.08984375, "grad_norm": 0.005699362140148878, "learning_rate": 5.224396231890232e-05, "loss": 4.5295488234842196e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.00005, "step": 279, "tokens/total": 9714672, "tokens/train_per_sec_per_gpu": 29.93, "tokens/trainable": 126929} +{"epoch": 1.09375, "grad_norm": 0.6928226947784424, "learning_rate": 5.195427572104522e-05, "loss": 0.01004955917596817, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.0101, "step": 280, "tokens/total": 9749104, "tokens/train_per_sec_per_gpu": 23.81, "tokens/trainable": 127366} +{"epoch": 1.09765625, "grad_norm": 0.02343161031603813, "learning_rate": 5.166471586820751e-05, "loss": 0.00030235282611101866, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.35, "memory/max_allocated (GiB)": 39.35, "ppl": 1.0003, "step": 281, "tokens/total": 9785712, "tokens/train_per_sec_per_gpu": 23.71, "tokens/trainable": 127831} +{"epoch": 1.1015625, "grad_norm": 0.10151995718479156, "learning_rate": 5.1375294810156615e-05, "loss": 0.0005974420346319675, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.0006, "step": 282, "tokens/total": 9820544, "tokens/train_per_sec_per_gpu": 25.8, "tokens/trainable": 128278} +{"epoch": 1.10546875, "grad_norm": 0.009477941319346428, "learning_rate": 5.1086024590884144e-05, "loss": 0.00011071663902839646, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00011, "step": 283, "tokens/total": 9857952, "tokens/train_per_sec_per_gpu": 24.94, "tokens/trainable": 128727} +{"epoch": 1.109375, "grad_norm": 0.07129290699958801, "learning_rate": 5.079691724810461e-05, "loss": 0.00035478832433000207, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.33, "memory/max_allocated (GiB)": 38.33, "ppl": 1.00035, "step": 284, "tokens/total": 9894128, "tokens/train_per_sec_per_gpu": 21.76, "tokens/trainable": 129162} +{"epoch": 1.11328125, "grad_norm": 0.04254872724413872, "learning_rate": 5.0507984812754684e-05, "loss": 0.00029435715987347066, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.17, "memory/max_allocated (GiB)": 37.17, "ppl": 1.00029, "step": 285, "tokens/total": 9927568, "tokens/train_per_sec_per_gpu": 28.67, "tokens/trainable": 129659} +{"epoch": 1.1171875, "grad_norm": 0.01620500162243843, "learning_rate": 5.021923930849237e-05, "loss": 0.00013032893184572458, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.00013, "step": 286, "tokens/total": 9961888, "tokens/train_per_sec_per_gpu": 27.89, "tokens/trainable": 130145} +{"epoch": 1.12109375, "grad_norm": 0.06176358833909035, "learning_rate": 4.99306927511967e-05, "loss": 0.0004199218819849193, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.31, "memory/max_allocated (GiB)": 37.31, "ppl": 1.00042, "step": 287, "tokens/total": 9995696, "tokens/train_per_sec_per_gpu": 27.72, "tokens/trainable": 130585} +{"epoch": 1.125, "grad_norm": 0.5648428797721863, "learning_rate": 4.964235714846775e-05, "loss": 0.006058403290808201, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.72, "memory/max_allocated (GiB)": 36.72, "ppl": 1.00608, "step": 288, "tokens/total": 10028000, "tokens/train_per_sec_per_gpu": 27.16, "tokens/trainable": 131005} +{"epoch": 1.12890625, "grad_norm": 0.0043099927715957165, "learning_rate": 4.9354244499126866e-05, "loss": 3.666456905193627e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.16, "memory/max_allocated (GiB)": 36.16, "ppl": 1.00004, "step": 289, "tokens/total": 10059728, "tokens/train_per_sec_per_gpu": 29.36, "tokens/trainable": 131471} +{"epoch": 1.1328125, "grad_norm": 0.009522084146738052, "learning_rate": 4.90663667927174e-05, "loss": 0.00016610510647296906, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.00017, "step": 290, "tokens/total": 10092576, "tokens/train_per_sec_per_gpu": 28.85, "tokens/trainable": 131920} +{"epoch": 1.13671875, "grad_norm": 0.025568513199687004, "learning_rate": 4.877873600900581e-05, "loss": 0.00022745897877030075, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.00023, "step": 291, "tokens/total": 10128608, "tokens/train_per_sec_per_gpu": 21.79, "tokens/trainable": 132379} +{"epoch": 1.140625, "grad_norm": 0.010194067843258381, "learning_rate": 4.849136411748306e-05, "loss": 6.575245060957968e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.00007, "step": 292, "tokens/total": 10166192, "tokens/train_per_sec_per_gpu": 24.85, "tokens/trainable": 132851} +{"epoch": 1.14453125, "grad_norm": 0.01086503453552723, "learning_rate": 4.8204263076866574e-05, "loss": 5.458702798932791e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00005, "step": 293, "tokens/total": 10201840, "tokens/train_per_sec_per_gpu": 29.58, "tokens/trainable": 133351} +{"epoch": 1.1484375, "grad_norm": 0.0019075694726780057, "learning_rate": 4.791744483460251e-05, "loss": 3.528753950376995e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00004, "step": 294, "tokens/total": 10236048, "tokens/train_per_sec_per_gpu": 24.22, "tokens/trainable": 133784} +{"epoch": 1.15234375, "grad_norm": 0.01666475459933281, "learning_rate": 4.7630921326368736e-05, "loss": 0.00017620417929720134, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.00018, "step": 295, "tokens/total": 10274688, "tokens/train_per_sec_per_gpu": 20.01, "tokens/trainable": 134209} +{"epoch": 1.15625, "grad_norm": 0.4965975284576416, "learning_rate": 4.7344704475577916e-05, "loss": 0.008735542185604572, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.66, "memory/max_allocated (GiB)": 35.66, "ppl": 1.00877, "step": 296, "tokens/total": 10304576, "tokens/train_per_sec_per_gpu": 30.26, "tokens/trainable": 134645} +{"epoch": 1.16015625, "grad_norm": 1.741711139678955, "learning_rate": 4.705880619288153e-05, "loss": 0.02383199706673622, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.21, "memory/max_allocated (GiB)": 38.21, "ppl": 1.02412, "step": 297, "tokens/total": 10340720, "tokens/train_per_sec_per_gpu": 22.0, "tokens/trainable": 135047} +{"epoch": 1.1640625, "grad_norm": 0.14352066814899445, "learning_rate": 4.677323837567412e-05, "loss": 0.0011387475533410907, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00114, "step": 298, "tokens/total": 10373376, "tokens/train_per_sec_per_gpu": 25.35, "tokens/trainable": 135474} +{"epoch": 1.16796875, "grad_norm": 0.025397103279829025, "learning_rate": 4.6488012907598146e-05, "loss": 0.00019630053429864347, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.0002, "step": 299, "tokens/total": 10406128, "tokens/train_per_sec_per_gpu": 27.97, "tokens/trainable": 135948} +{"epoch": 1.171875, "grad_norm": 0.31836411356925964, "learning_rate": 4.620314165804964e-05, "loss": 0.007248765788972378, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.3, "memory/max_allocated (GiB)": 38.3, "ppl": 1.00728, "step": 300, "tokens/total": 10441792, "tokens/train_per_sec_per_gpu": 25.69, "tokens/trainable": 136421} +{"epoch": 1.17578125, "grad_norm": 0.018236806616187096, "learning_rate": 4.591863648168407e-05, "loss": 0.00016165485430974513, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00016, "step": 301, "tokens/total": 10476128, "tokens/train_per_sec_per_gpu": 26.35, "tokens/trainable": 136886} +{"epoch": 1.1796875, "grad_norm": 0.03979094326496124, "learning_rate": 4.5634509217923135e-05, "loss": 0.0004015325685031712, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.0004, "step": 302, "tokens/total": 10509472, "tokens/train_per_sec_per_gpu": 25.79, "tokens/trainable": 137321} +{"epoch": 1.18359375, "grad_norm": 0.6921079754829407, "learning_rate": 4.535077169046201e-05, "loss": 0.008542701601982117, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00858, "step": 303, "tokens/total": 10546032, "tokens/train_per_sec_per_gpu": 25.14, "tokens/trainable": 137758} +{"epoch": 1.1875, "grad_norm": 0.3116692900657654, "learning_rate": 4.506743570677743e-05, "loss": 0.0036791437305510044, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.00369, "step": 304, "tokens/total": 10576624, "tokens/train_per_sec_per_gpu": 31.33, "tokens/trainable": 138190} +{"epoch": 1.19140625, "grad_norm": 0.4210950434207916, "learning_rate": 4.478451305763618e-05, "loss": 0.0034517417661845684, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.95, "memory/max_allocated (GiB)": 36.95, "ppl": 1.00346, "step": 305, "tokens/total": 10609760, "tokens/train_per_sec_per_gpu": 30.55, "tokens/trainable": 138650} +{"epoch": 1.1953125, "grad_norm": 0.03102666325867176, "learning_rate": 4.450201551660454e-05, "loss": 0.00030287160188890994, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.51, "memory/max_allocated (GiB)": 36.51, "ppl": 1.0003, "step": 306, "tokens/total": 10641472, "tokens/train_per_sec_per_gpu": 26.24, "tokens/trainable": 139082} +{"epoch": 1.19921875, "grad_norm": 0.06810538470745087, "learning_rate": 4.4219954839558276e-05, "loss": 0.000709884858224541, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00071, "step": 307, "tokens/total": 10676160, "tokens/train_per_sec_per_gpu": 25.36, "tokens/trainable": 139523} +{"epoch": 1.203125, "grad_norm": 0.05812452360987663, "learning_rate": 4.393834276419352e-05, "loss": 0.0007450748234987259, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.83, "memory/max_allocated (GiB)": 35.83, "ppl": 1.00075, "step": 308, "tokens/total": 10706384, "tokens/train_per_sec_per_gpu": 32.3, "tokens/trainable": 140010} +{"epoch": 1.20703125, "grad_norm": 0.37979763746261597, "learning_rate": 4.36571910095382e-05, "loss": 0.001143977395258844, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.00114, "step": 309, "tokens/total": 10744496, "tokens/train_per_sec_per_gpu": 25.43, "tokens/trainable": 140454} +{"epoch": 1.2109375, "grad_norm": 0.014455472119152546, "learning_rate": 4.337651127546448e-05, "loss": 0.00013323896564543247, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00013, "step": 310, "tokens/total": 10780176, "tokens/train_per_sec_per_gpu": 27.56, "tokens/trainable": 140905} +{"epoch": 1.21484375, "grad_norm": 0.11526018381118774, "learning_rate": 4.3096315242201736e-05, "loss": 0.0009318940574303269, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.61, "memory/max_allocated (GiB)": 36.61, "ppl": 1.00093, "step": 311, "tokens/total": 10813136, "tokens/train_per_sec_per_gpu": 26.23, "tokens/trainable": 141339} +{"epoch": 1.21875, "grad_norm": 0.015050905756652355, "learning_rate": 4.2816614569850635e-05, "loss": 0.00012233550660312176, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00012, "step": 312, "tokens/total": 10848912, "tokens/train_per_sec_per_gpu": 24.29, "tokens/trainable": 141778} +{"epoch": 1.22265625, "grad_norm": 0.017255308106541634, "learning_rate": 4.2537420897897864e-05, "loss": 0.00020080417743884027, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.0002, "step": 313, "tokens/total": 10884320, "tokens/train_per_sec_per_gpu": 26.97, "tokens/trainable": 142255} +{"epoch": 1.2265625, "grad_norm": 0.02956739068031311, "learning_rate": 4.225874584473174e-05, "loss": 0.00016694123041816056, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.00017, "step": 314, "tokens/total": 10917152, "tokens/train_per_sec_per_gpu": 25.94, "tokens/trainable": 142693} +{"epoch": 1.23046875, "grad_norm": 0.012131265364587307, "learning_rate": 4.19806010071587e-05, "loss": 0.00017793051665648818, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.00018, "step": 315, "tokens/total": 10955024, "tokens/train_per_sec_per_gpu": 24.28, "tokens/trainable": 143157} +{"epoch": 1.234375, "grad_norm": 0.12814265489578247, "learning_rate": 4.170299795992081e-05, "loss": 0.0006731073372066021, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.00067, "step": 316, "tokens/total": 10988416, "tokens/train_per_sec_per_gpu": 27.03, "tokens/trainable": 143603} +{"epoch": 1.23828125, "grad_norm": 0.37461408972740173, "learning_rate": 4.142594825521398e-05, "loss": 0.006486630532890558, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.00651, "step": 317, "tokens/total": 11027712, "tokens/train_per_sec_per_gpu": 25.9, "tokens/trainable": 144084} +{"epoch": 1.2421875, "grad_norm": 0.06610410660505295, "learning_rate": 4.114946342220728e-05, "loss": 0.0005091720959171653, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.00051, "step": 318, "tokens/total": 11063920, "tokens/train_per_sec_per_gpu": 24.02, "tokens/trainable": 144532} +{"epoch": 1.24609375, "grad_norm": 0.3300163447856903, "learning_rate": 4.087355496656321e-05, "loss": 0.0011865716660395265, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00119, "step": 319, "tokens/total": 11097952, "tokens/train_per_sec_per_gpu": 28.41, "tokens/trainable": 145004} +{"epoch": 1.25, "grad_norm": 0.13100066781044006, "learning_rate": 4.05982343699588e-05, "loss": 0.0016118860803544521, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00161, "step": 320, "tokens/total": 11137168, "tokens/train_per_sec_per_gpu": 25.06, "tokens/trainable": 145498} +{"epoch": 1.25390625, "grad_norm": 0.11168782413005829, "learning_rate": 4.0323513089607876e-05, "loss": 0.000973586633335799, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.00097, "step": 321, "tokens/total": 11172224, "tokens/train_per_sec_per_gpu": 24.26, "tokens/trainable": 145965} +{"epoch": 1.2578125, "grad_norm": 0.14063723385334015, "learning_rate": 4.004940255778431e-05, "loss": 0.0005637876456603408, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00056, "step": 322, "tokens/total": 11210832, "tokens/train_per_sec_per_gpu": 23.87, "tokens/trainable": 146409} +{"epoch": 1.26171875, "grad_norm": 0.04694448783993721, "learning_rate": 3.977591418134619e-05, "loss": 0.0003982946800533682, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.0004, "step": 323, "tokens/total": 11245984, "tokens/train_per_sec_per_gpu": 25.72, "tokens/trainable": 146849} +{"epoch": 1.265625, "grad_norm": 0.0029110892210155725, "learning_rate": 3.95030593412612e-05, "loss": 5.4107571486383677e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.00005, "step": 324, "tokens/total": 11281360, "tokens/train_per_sec_per_gpu": 24.51, "tokens/trainable": 147272} +{"epoch": 1.26953125, "grad_norm": 0.3027881681919098, "learning_rate": 3.923084939213296e-05, "loss": 0.0018402950372546911, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.88, "memory/max_allocated (GiB)": 35.88, "ppl": 1.00184, "step": 325, "tokens/total": 11312176, "tokens/train_per_sec_per_gpu": 27.62, "tokens/trainable": 147694} +{"epoch": 1.2734375, "grad_norm": 0.03848681598901749, "learning_rate": 3.895929566172861e-05, "loss": 0.0002556192921474576, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.00026, "step": 326, "tokens/total": 11348192, "tokens/train_per_sec_per_gpu": 25.15, "tokens/trainable": 148158} +{"epoch": 1.27734375, "grad_norm": 0.005910215899348259, "learning_rate": 3.868840945050728e-05, "loss": 0.00010222442506346852, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.05, "memory/max_allocated (GiB)": 39.05, "ppl": 1.0001, "step": 327, "tokens/total": 11386704, "tokens/train_per_sec_per_gpu": 21.51, "tokens/trainable": 148590} +{"epoch": 1.28125, "grad_norm": 0.23779959976673126, "learning_rate": 3.841820203114995e-05, "loss": 0.0017296341247856617, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.00173, "step": 328, "tokens/total": 11418448, "tokens/train_per_sec_per_gpu": 29.0, "tokens/trainable": 149037} +{"epoch": 1.28515625, "grad_norm": 0.007190580479800701, "learning_rate": 3.814868464809027e-05, "loss": 0.00010226783342659473, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.0001, "step": 329, "tokens/total": 11450304, "tokens/train_per_sec_per_gpu": 25.75, "tokens/trainable": 149439} +{"epoch": 1.2890625, "grad_norm": 0.011761597357690334, "learning_rate": 3.787986851704667e-05, "loss": 8.020545647013932e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.00008, "step": 330, "tokens/total": 11484048, "tokens/train_per_sec_per_gpu": 25.74, "tokens/trainable": 149869} +{"epoch": 1.29296875, "grad_norm": 0.1061306744813919, "learning_rate": 3.7611764824555654e-05, "loss": 0.0008088742033578455, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00081, "step": 331, "tokens/total": 11516032, "tokens/train_per_sec_per_gpu": 30.87, "tokens/trainable": 150343} +{"epoch": 1.296875, "grad_norm": 0.09908034652471542, "learning_rate": 3.734438472750619e-05, "loss": 0.00028838450089097023, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.00029, "step": 332, "tokens/total": 11555968, "tokens/train_per_sec_per_gpu": 23.09, "tokens/trainable": 150814} +{"epoch": 1.30078125, "grad_norm": 0.08925658464431763, "learning_rate": 3.707773935267552e-05, "loss": 0.0005610092775896192, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.46, "memory/max_allocated (GiB)": 38.46, "ppl": 1.00056, "step": 333, "tokens/total": 11592880, "tokens/train_per_sec_per_gpu": 26.78, "tokens/trainable": 151300} +{"epoch": 1.3046875, "grad_norm": 0.026898255571722984, "learning_rate": 3.68118397962661e-05, "loss": 0.000155565794557333, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.00016, "step": 334, "tokens/total": 11627792, "tokens/train_per_sec_per_gpu": 23.25, "tokens/trainable": 151708} +{"epoch": 1.30859375, "grad_norm": 0.03384476155042648, "learning_rate": 3.654669712344384e-05, "loss": 0.0002278089232277125, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 1.00023, "step": 335, "tokens/total": 11658832, "tokens/train_per_sec_per_gpu": 31.46, "tokens/trainable": 152164} +{"epoch": 1.3125, "grad_norm": 0.027111841365695, "learning_rate": 3.628232236787763e-05, "loss": 0.00012933369725942612, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.96, "memory/max_allocated (GiB)": 37.96, "ppl": 1.00013, "step": 336, "tokens/total": 11694464, "tokens/train_per_sec_per_gpu": 22.54, "tokens/trainable": 152603} +{"epoch": 1.31640625, "grad_norm": 0.08462227880954742, "learning_rate": 3.6018726531280144e-05, "loss": 0.0003723568224813789, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00037, "step": 337, "tokens/total": 11732176, "tokens/train_per_sec_per_gpu": 28.19, "tokens/trainable": 153094} +{"epoch": 1.3203125, "grad_norm": 0.06431149691343307, "learning_rate": 3.575592058295017e-05, "loss": 0.0003339872055221349, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.00033, "step": 338, "tokens/total": 11766032, "tokens/train_per_sec_per_gpu": 28.34, "tokens/trainable": 153573} +{"epoch": 1.32421875, "grad_norm": 2.643336772918701, "learning_rate": 3.549391545931585e-05, "loss": 0.00968889333307743, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00974, "step": 339, "tokens/total": 11800640, "tokens/train_per_sec_per_gpu": 27.63, "tokens/trainable": 154033} +{"epoch": 1.328125, "grad_norm": 0.008440686389803886, "learning_rate": 3.5232722063479914e-05, "loss": 4.084999818587676e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.00004, "step": 340, "tokens/total": 11834352, "tokens/train_per_sec_per_gpu": 24.13, "tokens/trainable": 154482} +{"epoch": 1.33203125, "grad_norm": 0.0332307405769825, "learning_rate": 3.49723512647657e-05, "loss": 0.0002877347869798541, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.00029, "step": 341, "tokens/total": 11873056, "tokens/train_per_sec_per_gpu": 24.75, "tokens/trainable": 154982} +{"epoch": 1.3359375, "grad_norm": 0.03571756184101105, "learning_rate": 3.471281389826491e-05, "loss": 0.00017792933795135468, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.00018, "step": 342, "tokens/total": 11904192, "tokens/train_per_sec_per_gpu": 25.98, "tokens/trainable": 155387} +{"epoch": 1.33984375, "grad_norm": 0.8009323477745056, "learning_rate": 3.4454120764386764e-05, "loss": 0.013295009732246399, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.66, "memory/max_allocated (GiB)": 38.66, "ppl": 1.01338, "step": 343, "tokens/total": 11942384, "tokens/train_per_sec_per_gpu": 24.31, "tokens/trainable": 155842} +{"epoch": 1.34375, "grad_norm": 0.09386368095874786, "learning_rate": 3.4196282628408526e-05, "loss": 0.0008597730775363743, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.00086, "step": 344, "tokens/total": 11976624, "tokens/train_per_sec_per_gpu": 25.33, "tokens/trainable": 156287} +{"epoch": 1.34765625, "grad_norm": 0.1963425874710083, "learning_rate": 3.3939310220027456e-05, "loss": 0.0007909886189736426, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00079, "step": 345, "tokens/total": 12010768, "tokens/train_per_sec_per_gpu": 28.85, "tokens/trainable": 156782} +{"epoch": 1.3515625, "grad_norm": 0.22330743074417114, "learning_rate": 3.3683214232914404e-05, "loss": 0.001629244419746101, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.00163, "step": 346, "tokens/total": 12046224, "tokens/train_per_sec_per_gpu": 26.12, "tokens/trainable": 157255} +{"epoch": 1.35546875, "grad_norm": 0.8272461295127869, "learning_rate": 3.342800532426873e-05, "loss": 0.0018364081624895334, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.00184, "step": 347, "tokens/total": 12082160, "tokens/train_per_sec_per_gpu": 25.45, "tokens/trainable": 157721} +{"epoch": 1.359375, "grad_norm": 0.045948147773742676, "learning_rate": 3.317369411437484e-05, "loss": 0.0001703716698102653, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.00017, "step": 348, "tokens/total": 12118128, "tokens/train_per_sec_per_gpu": 26.16, "tokens/trainable": 158189} +{"epoch": 1.36328125, "grad_norm": 0.005153643433004618, "learning_rate": 3.292029118616024e-05, "loss": 7.000749610597268e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.00007, "step": 349, "tokens/total": 12151264, "tokens/train_per_sec_per_gpu": 25.6, "tokens/trainable": 158612} +{"epoch": 1.3671875, "grad_norm": 1.2769447565078735, "learning_rate": 3.266780708475511e-05, "loss": 0.006218849215656519, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00624, "step": 350, "tokens/total": 12186800, "tokens/train_per_sec_per_gpu": 25.04, "tokens/trainable": 159090} +{"epoch": 1.37109375, "grad_norm": 4.109220027923584, "learning_rate": 3.241625231705354e-05, "loss": 0.0029714410193264484, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.00298, "step": 351, "tokens/total": 12222944, "tokens/train_per_sec_per_gpu": 25.01, "tokens/trainable": 159549} +{"epoch": 1.375, "grad_norm": 0.0053605674766004086, "learning_rate": 3.216563735127618e-05, "loss": 7.020033081062138e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.15, "memory/max_allocated (GiB)": 39.15, "ppl": 1.00007, "step": 352, "tokens/total": 12258208, "tokens/train_per_sec_per_gpu": 25.53, "tokens/trainable": 160026} +{"epoch": 1.37890625, "grad_norm": 0.0025639936793595552, "learning_rate": 3.191597261653475e-05, "loss": 3.898839349858463e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00004, "step": 353, "tokens/total": 12295024, "tokens/train_per_sec_per_gpu": 23.73, "tokens/trainable": 160487} +{"epoch": 1.3828125, "grad_norm": 0.836240291595459, "learning_rate": 3.166726850239794e-05, "loss": 0.004503627307713032, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00451, "step": 354, "tokens/total": 12328064, "tokens/train_per_sec_per_gpu": 27.42, "tokens/trainable": 160935} +{"epoch": 1.38671875, "grad_norm": 0.0032057520002126694, "learning_rate": 3.141953535845912e-05, "loss": 4.608495510183275e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.34, "memory/max_allocated (GiB)": 36.34, "ppl": 1.00005, "step": 355, "tokens/total": 12359504, "tokens/train_per_sec_per_gpu": 30.42, "tokens/trainable": 161385} +{"epoch": 1.390625, "grad_norm": 0.013079266063869, "learning_rate": 3.11727834939056e-05, "loss": 0.00015339840319938958, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00015, "step": 356, "tokens/total": 12391824, "tokens/train_per_sec_per_gpu": 25.49, "tokens/trainable": 161849} +{"epoch": 1.39453125, "grad_norm": 0.09126527607440948, "learning_rate": 3.092702317708967e-05, "loss": 0.00066906469874084, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.00067, "step": 357, "tokens/total": 12425408, "tokens/train_per_sec_per_gpu": 28.92, "tokens/trainable": 162306} +{"epoch": 1.3984375, "grad_norm": 0.006502038799226284, "learning_rate": 3.0682264635101276e-05, "loss": 4.169282328803092e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.00004, "step": 358, "tokens/total": 12458160, "tokens/train_per_sec_per_gpu": 28.27, "tokens/trainable": 162759} +{"epoch": 1.40234375, "grad_norm": 0.005240991711616516, "learning_rate": 3.0438518053342407e-05, "loss": 4.2192194086965173e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00004, "step": 359, "tokens/total": 12497104, "tokens/train_per_sec_per_gpu": 22.13, "tokens/trainable": 163230} +{"epoch": 1.40625, "grad_norm": 0.008646814152598381, "learning_rate": 3.0195793575103266e-05, "loss": 0.00012305134441703558, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00012, "step": 360, "tokens/total": 12536064, "tokens/train_per_sec_per_gpu": 21.26, "tokens/trainable": 163694} +{"epoch": 1.41015625, "grad_norm": 0.7534189224243164, "learning_rate": 2.9954101301140146e-05, "loss": 0.0049788448959589005, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.57, "memory/max_allocated (GiB)": 37.57, "ppl": 1.00499, "step": 361, "tokens/total": 12571488, "tokens/train_per_sec_per_gpu": 24.57, "tokens/trainable": 164144} +{"epoch": 1.4140625, "grad_norm": 0.013152200728654861, "learning_rate": 2.9713451289255123e-05, "loss": 0.00011801850632764399, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.63, "memory/max_allocated (GiB)": 36.63, "ppl": 1.00012, "step": 362, "tokens/total": 12600864, "tokens/train_per_sec_per_gpu": 25.54, "tokens/trainable": 164585} +{"epoch": 1.41796875, "grad_norm": 0.08396349102258682, "learning_rate": 2.9473853553877484e-05, "loss": 8.138471457641572e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.00008, "step": 363, "tokens/total": 12635024, "tokens/train_per_sec_per_gpu": 27.24, "tokens/trainable": 165059} +{"epoch": 1.421875, "grad_norm": 0.01931477151811123, "learning_rate": 2.9235318065647e-05, "loss": 0.00021893452503718436, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.00022, "step": 364, "tokens/total": 12674736, "tokens/train_per_sec_per_gpu": 24.9, "tokens/trainable": 165519} +{"epoch": 1.42578125, "grad_norm": 0.04050938040018082, "learning_rate": 2.8997854750998964e-05, "loss": 0.00011592524242587388, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.46, "memory/max_allocated (GiB)": 39.46, "ppl": 1.00012, "step": 365, "tokens/total": 12714256, "tokens/train_per_sec_per_gpu": 23.15, "tokens/trainable": 165984} +{"epoch": 1.4296875, "grad_norm": 0.05782562494277954, "learning_rate": 2.8761473491751258e-05, "loss": 0.00026311000692658126, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.00026, "step": 366, "tokens/total": 12747504, "tokens/train_per_sec_per_gpu": 26.9, "tokens/trainable": 166419} +{"epoch": 1.43359375, "grad_norm": 0.18729090690612793, "learning_rate": 2.8526184124692883e-05, "loss": 0.0011717869201675057, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00117, "step": 367, "tokens/total": 12783872, "tokens/train_per_sec_per_gpu": 20.38, "tokens/trainable": 166857} +{"epoch": 1.4375, "grad_norm": 0.0652226135134697, "learning_rate": 2.829199644117484e-05, "loss": 0.00029141237610019743, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.00029, "step": 368, "tokens/total": 12817856, "tokens/train_per_sec_per_gpu": 24.36, "tokens/trainable": 167267} +{"epoch": 1.44140625, "grad_norm": 0.011468857526779175, "learning_rate": 2.8058920186702553e-05, "loss": 8.487456216244027e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00008, "step": 369, "tokens/total": 12855472, "tokens/train_per_sec_per_gpu": 24.97, "tokens/trainable": 167740} +{"epoch": 1.4453125, "grad_norm": 0.03433288261294365, "learning_rate": 2.782696506053033e-05, "loss": 0.00028727270546369255, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.00029, "step": 370, "tokens/total": 12893344, "tokens/train_per_sec_per_gpu": 22.35, "tokens/trainable": 168172} +{"epoch": 1.44921875, "grad_norm": 0.017947886139154434, "learning_rate": 2.7596140715257824e-05, "loss": 0.0001611682091606781, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00016, "step": 371, "tokens/total": 12928512, "tokens/train_per_sec_per_gpu": 25.73, "tokens/trainable": 168637} +{"epoch": 1.453125, "grad_norm": 0.021122276782989502, "learning_rate": 2.7366456756428184e-05, "loss": 0.0001338740548817441, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 1.00013, "step": 372, "tokens/total": 12963120, "tokens/train_per_sec_per_gpu": 27.17, "tokens/trainable": 169094} +{"epoch": 1.45703125, "grad_norm": 0.5382697582244873, "learning_rate": 2.7137922742128486e-05, "loss": 0.004574904218316078, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00459, "step": 373, "tokens/total": 12997648, "tokens/train_per_sec_per_gpu": 27.02, "tokens/trainable": 169593} +{"epoch": 1.4609375, "grad_norm": 0.08852077275514603, "learning_rate": 2.691054818259188e-05, "loss": 0.0006643776432611048, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00066, "step": 374, "tokens/total": 13032496, "tokens/train_per_sec_per_gpu": 26.31, "tokens/trainable": 170066} +{"epoch": 1.46484375, "grad_norm": 0.014979059807956219, "learning_rate": 2.6684342539801933e-05, "loss": 0.00010840289178304374, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.56, "memory/max_allocated (GiB)": 36.56, "ppl": 1.00011, "step": 375, "tokens/total": 13064016, "tokens/train_per_sec_per_gpu": 28.82, "tokens/trainable": 170518} +{"epoch": 1.46875, "grad_norm": 0.07971519976854324, "learning_rate": 2.645931522709877e-05, "loss": 0.0002802855451591313, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.00028, "step": 376, "tokens/total": 13099408, "tokens/train_per_sec_per_gpu": 29.05, "tokens/trainable": 170998} +{"epoch": 1.47265625, "grad_norm": 0.1373508721590042, "learning_rate": 2.6235475608787365e-05, "loss": 0.000795685569755733, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.0008, "step": 377, "tokens/total": 13135088, "tokens/train_per_sec_per_gpu": 25.54, "tokens/trainable": 171459} +{"epoch": 1.4765625, "grad_norm": 0.12381671369075775, "learning_rate": 2.6012832999747916e-05, "loss": 0.0011068318272009492, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.00111, "step": 378, "tokens/total": 13166208, "tokens/train_per_sec_per_gpu": 29.34, "tokens/trainable": 171905} +{"epoch": 1.48046875, "grad_norm": 0.677169144153595, "learning_rate": 2.579139666504821e-05, "loss": 0.015140318311750889, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.01526, "step": 379, "tokens/total": 13206192, "tokens/train_per_sec_per_gpu": 22.05, "tokens/trainable": 172351} +{"epoch": 1.484375, "grad_norm": 0.0690387561917305, "learning_rate": 2.557117581955798e-05, "loss": 0.00043194295722059906, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00043, "step": 380, "tokens/total": 13240032, "tokens/train_per_sec_per_gpu": 26.97, "tokens/trainable": 172817} +{"epoch": 1.48828125, "grad_norm": 1.4578546285629272, "learning_rate": 2.5352179627565532e-05, "loss": 0.011716348119080067, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.01179, "step": 381, "tokens/total": 13274144, "tokens/train_per_sec_per_gpu": 25.72, "tokens/trainable": 173258} +{"epoch": 1.4921875, "grad_norm": 0.003213818883523345, "learning_rate": 2.5134417202396277e-05, "loss": 5.260230682324618e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.6, "memory/max_allocated (GiB)": 38.6, "ppl": 1.00005, "step": 382, "tokens/total": 13308128, "tokens/train_per_sec_per_gpu": 23.91, "tokens/trainable": 173665} +{"epoch": 1.49609375, "grad_norm": 0.09512197226285934, "learning_rate": 2.491789760603361e-05, "loss": 0.0012772618792951107, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00128, "step": 383, "tokens/total": 13344544, "tokens/train_per_sec_per_gpu": 23.15, "tokens/trainable": 174095} +{"epoch": 1.5, "grad_norm": 0.07197681069374084, "learning_rate": 2.4702629848741764e-05, "loss": 0.0004898368497379124, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.00049, "step": 384, "tokens/total": 13378592, "tokens/train_per_sec_per_gpu": 26.49, "tokens/trainable": 174550} +{"epoch": 1.50390625, "grad_norm": 0.016425469890236855, "learning_rate": 2.4488622888690785e-05, "loss": 0.00016084076196420938, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.04, "memory/max_allocated (GiB)": 37.04, "ppl": 1.00016, "step": 385, "tokens/total": 13412048, "tokens/train_per_sec_per_gpu": 27.02, "tokens/trainable": 174990} +{"epoch": 1.5078125, "grad_norm": 0.0271269753575325, "learning_rate": 2.427588563158384e-05, "loss": 0.0001258082629647106, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00013, "step": 386, "tokens/total": 13446384, "tokens/train_per_sec_per_gpu": 26.14, "tokens/trainable": 175476} +{"epoch": 1.51171875, "grad_norm": 0.01897328533232212, "learning_rate": 2.406442693028651e-05, "loss": 0.00023274659179151058, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.63, "memory/max_allocated (GiB)": 38.63, "ppl": 1.00023, "step": 387, "tokens/total": 13478320, "tokens/train_per_sec_per_gpu": 25.48, "tokens/trainable": 175897} +{"epoch": 1.515625, "grad_norm": 0.010036381892859936, "learning_rate": 2.3854255584458547e-05, "loss": 7.635099609615281e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.00008, "step": 388, "tokens/total": 13516912, "tokens/train_per_sec_per_gpu": 21.61, "tokens/trainable": 176324} +{"epoch": 1.51953125, "grad_norm": 0.008645706810057163, "learning_rate": 2.3645380340187508e-05, "loss": 5.479209721670486e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00005, "step": 389, "tokens/total": 13552592, "tokens/train_per_sec_per_gpu": 26.01, "tokens/trainable": 176805} +{"epoch": 1.5234375, "grad_norm": 0.10219941288232803, "learning_rate": 2.3437809889624914e-05, "loss": 0.00024556691641919315, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.54, "memory/max_allocated (GiB)": 36.54, "ppl": 1.00025, "step": 390, "tokens/total": 13583856, "tokens/train_per_sec_per_gpu": 28.99, "tokens/trainable": 177256} +{"epoch": 1.52734375, "grad_norm": 0.01607794687151909, "learning_rate": 2.3231552870624487e-05, "loss": 9.920304728439078e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.47, "memory/max_allocated (GiB)": 38.47, "ppl": 1.0001, "step": 391, "tokens/total": 13614864, "tokens/train_per_sec_per_gpu": 29.1, "tokens/trainable": 177682} +{"epoch": 1.53125, "grad_norm": 0.04643220081925392, "learning_rate": 2.3026617866382657e-05, "loss": 0.00028433604165911674, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.53, "memory/max_allocated (GiB)": 36.53, "ppl": 1.00028, "step": 392, "tokens/total": 13647616, "tokens/train_per_sec_per_gpu": 29.83, "tokens/trainable": 178179} +{"epoch": 1.53515625, "grad_norm": 0.03047039918601513, "learning_rate": 2.2823013405081507e-05, "loss": 0.0002474005159456283, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.00025, "step": 393, "tokens/total": 13685696, "tokens/train_per_sec_per_gpu": 26.74, "tokens/trainable": 178676} +{"epoch": 1.5390625, "grad_norm": 0.01581122726202011, "learning_rate": 2.2620747959533722e-05, "loss": 0.00016474738367833197, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.76, "memory/max_allocated (GiB)": 36.76, "ppl": 1.00016, "step": 394, "tokens/total": 13718768, "tokens/train_per_sec_per_gpu": 30.56, "tokens/trainable": 179173} +{"epoch": 1.54296875, "grad_norm": 0.15032939612865448, "learning_rate": 2.2419829946830123e-05, "loss": 0.0003269108128733933, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.00033, "step": 395, "tokens/total": 13754960, "tokens/train_per_sec_per_gpu": 24.21, "tokens/trainable": 179624} +{"epoch": 1.546875, "grad_norm": 0.08523056656122208, "learning_rate": 2.2220267727989325e-05, "loss": 0.000617795274592936, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00062, "step": 396, "tokens/total": 13789104, "tokens/train_per_sec_per_gpu": 25.81, "tokens/trainable": 180078} +{"epoch": 1.55078125, "grad_norm": 0.04895060509443283, "learning_rate": 2.202206960760984e-05, "loss": 0.0006375741795636714, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.00064, "step": 397, "tokens/total": 13827248, "tokens/train_per_sec_per_gpu": 24.73, "tokens/trainable": 180585} +{"epoch": 1.5546875, "grad_norm": 0.01158529706299305, "learning_rate": 2.182524383352446e-05, "loss": 0.00010065470996778458, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.0001, "step": 398, "tokens/total": 13863408, "tokens/train_per_sec_per_gpu": 24.15, "tokens/trainable": 181039} +{"epoch": 1.55859375, "grad_norm": 0.11709310114383698, "learning_rate": 2.1629798596457056e-05, "loss": 0.0011241468600928783, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.52, "memory/max_allocated (GiB)": 36.52, "ppl": 1.00112, "step": 399, "tokens/total": 13895728, "tokens/train_per_sec_per_gpu": 26.65, "tokens/trainable": 181489} +{"epoch": 1.5625, "grad_norm": 0.0351383201777935, "learning_rate": 2.1435742029681725e-05, "loss": 6.874465907458216e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.08, "memory/max_allocated (GiB)": 36.08, "ppl": 1.00007, "step": 400, "tokens/total": 13926048, "tokens/train_per_sec_per_gpu": 28.22, "tokens/trainable": 181906} +{"epoch": 1.56640625, "grad_norm": 0.003283449914306402, "learning_rate": 2.124308220868431e-05, "loss": 3.764156281249598e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.67, "memory/max_allocated (GiB)": 38.67, "ppl": 1.00004, "step": 401, "tokens/total": 13961440, "tokens/train_per_sec_per_gpu": 24.29, "tokens/trainable": 182361} +{"epoch": 1.5703125, "grad_norm": 0.03250960260629654, "learning_rate": 2.105182715082638e-05, "loss": 0.00020631875668186694, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00021, "step": 402, "tokens/total": 13995040, "tokens/train_per_sec_per_gpu": 27.59, "tokens/trainable": 182838} +{"epoch": 1.57421875, "grad_norm": 0.11323804408311844, "learning_rate": 2.0861984815011552e-05, "loss": 0.0004130253510084003, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.16, "memory/max_allocated (GiB)": 38.16, "ppl": 1.00041, "step": 403, "tokens/total": 14030560, "tokens/train_per_sec_per_gpu": 22.6, "tokens/trainable": 183269} +{"epoch": 1.578125, "grad_norm": 0.00810126680880785, "learning_rate": 2.0673563101354323e-05, "loss": 0.00010851693514268845, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00011, "step": 404, "tokens/total": 14065840, "tokens/train_per_sec_per_gpu": 22.72, "tokens/trainable": 183699} +{"epoch": 1.58203125, "grad_norm": 0.3074002265930176, "learning_rate": 2.0486569850851317e-05, "loss": 0.0011531568598002195, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.00115, "step": 405, "tokens/total": 14100848, "tokens/train_per_sec_per_gpu": 25.91, "tokens/trainable": 184162} +{"epoch": 1.5859375, "grad_norm": 0.013776108622550964, "learning_rate": 2.0301012845054956e-05, "loss": 0.00014891792670823634, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.00015, "step": 406, "tokens/total": 14139600, "tokens/train_per_sec_per_gpu": 20.54, "tokens/trainable": 184576} +{"epoch": 1.58984375, "grad_norm": 0.11784582585096359, "learning_rate": 2.011689980574966e-05, "loss": 0.0006003585294820368, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.06, "memory/max_allocated (GiB)": 38.06, "ppl": 1.0006, "step": 407, "tokens/total": 14175904, "tokens/train_per_sec_per_gpu": 22.84, "tokens/trainable": 184995} +{"epoch": 1.59375, "grad_norm": 0.03768569976091385, "learning_rate": 1.993423839463052e-05, "loss": 0.00023135785886552185, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.00023, "step": 408, "tokens/total": 14209168, "tokens/train_per_sec_per_gpu": 31.91, "tokens/trainable": 185452} +{"epoch": 1.59765625, "grad_norm": 0.01146408636122942, "learning_rate": 1.975303621298445e-05, "loss": 0.00010955439211102203, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.00011, "step": 409, "tokens/total": 14246208, "tokens/train_per_sec_per_gpu": 25.38, "tokens/trainable": 185904} +{"epoch": 1.6015625, "grad_norm": 0.019200457260012627, "learning_rate": 1.957330080137385e-05, "loss": 0.00014757973258383572, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.31, "memory/max_allocated (GiB)": 36.31, "ppl": 1.00015, "step": 410, "tokens/total": 14277072, "tokens/train_per_sec_per_gpu": 29.01, "tokens/trainable": 186355} +{"epoch": 1.60546875, "grad_norm": 0.0311515461653471, "learning_rate": 1.9395039639322864e-05, "loss": 0.00034516496816650033, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.00035, "step": 411, "tokens/total": 14313120, "tokens/train_per_sec_per_gpu": 26.38, "tokens/trainable": 186831} +{"epoch": 1.609375, "grad_norm": 0.07807129621505737, "learning_rate": 1.9218260145006073e-05, "loss": 0.00024253905576188117, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.52, "memory/max_allocated (GiB)": 37.52, "ppl": 1.00024, "step": 412, "tokens/total": 14348368, "tokens/train_per_sec_per_gpu": 27.01, "tokens/trainable": 187317} +{"epoch": 1.61328125, "grad_norm": 0.7762148976325989, "learning_rate": 1.904296967493982e-05, "loss": 0.0031437063589692116, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.00315, "step": 413, "tokens/total": 14380528, "tokens/train_per_sec_per_gpu": 29.69, "tokens/trainable": 187773} +{"epoch": 1.6171875, "grad_norm": 1.2461422681808472, "learning_rate": 1.8869175523676064e-05, "loss": 0.019264817237854004, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.01945, "step": 414, "tokens/total": 14417216, "tokens/train_per_sec_per_gpu": 26.65, "tokens/trainable": 188268} +{"epoch": 1.62109375, "grad_norm": 0.002893929136916995, "learning_rate": 1.869688492349885e-05, "loss": 3.842517617158592e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00004, "step": 415, "tokens/total": 14450672, "tokens/train_per_sec_per_gpu": 27.4, "tokens/trainable": 188720} +{"epoch": 1.625, "grad_norm": 0.06577113270759583, "learning_rate": 1.85261050441233e-05, "loss": 0.00056617142399773, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.00057, "step": 416, "tokens/total": 14486464, "tokens/train_per_sec_per_gpu": 24.46, "tokens/trainable": 189177} +{"epoch": 1.62890625, "grad_norm": 0.21351303160190582, "learning_rate": 1.8356842992397304e-05, "loss": 0.005333347246050835, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.13, "memory/max_allocated (GiB)": 37.13, "ppl": 1.00535, "step": 417, "tokens/total": 14519984, "tokens/train_per_sec_per_gpu": 27.13, "tokens/trainable": 189617} +{"epoch": 1.6328125, "grad_norm": 0.004665100947022438, "learning_rate": 1.8189105812005714e-05, "loss": 4.580992754199542e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.00005, "step": 418, "tokens/total": 14556800, "tokens/train_per_sec_per_gpu": 22.36, "tokens/trainable": 190028} +{"epoch": 1.63671875, "grad_norm": 0.06241176277399063, "learning_rate": 1.802290048317732e-05, "loss": 0.0006835490930825472, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00068, "step": 419, "tokens/total": 14590480, "tokens/train_per_sec_per_gpu": 23.41, "tokens/trainable": 190445} +{"epoch": 1.640625, "grad_norm": 0.009088249877095222, "learning_rate": 1.785823392239424e-05, "loss": 6.93912515998818e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.00007, "step": 420, "tokens/total": 14624064, "tokens/train_per_sec_per_gpu": 26.13, "tokens/trainable": 190886} +{"epoch": 1.64453125, "grad_norm": 0.06580507010221481, "learning_rate": 1.7695112982104225e-05, "loss": 0.00032494342303834856, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00032, "step": 421, "tokens/total": 14659072, "tokens/train_per_sec_per_gpu": 24.85, "tokens/trainable": 191325} +{"epoch": 1.6484375, "grad_norm": 0.0034548949915915728, "learning_rate": 1.7533544450435433e-05, "loss": 3.0433626307058148e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.79, "memory/max_allocated (GiB)": 35.79, "ppl": 1.00003, "step": 422, "tokens/total": 14688624, "tokens/train_per_sec_per_gpu": 30.06, "tokens/trainable": 191758} +{"epoch": 1.65234375, "grad_norm": 0.0025723024737089872, "learning_rate": 1.7373535050913946e-05, "loss": 4.081207953277044e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00004, "step": 423, "tokens/total": 14725024, "tokens/train_per_sec_per_gpu": 21.9, "tokens/trainable": 192195} +{"epoch": 1.65625, "grad_norm": 0.07615114748477936, "learning_rate": 1.721509144218405e-05, "loss": 0.0004138364747632295, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.85, "memory/max_allocated (GiB)": 35.85, "ppl": 1.00041, "step": 424, "tokens/total": 14755344, "tokens/train_per_sec_per_gpu": 29.84, "tokens/trainable": 192625} +{"epoch": 1.66015625, "grad_norm": 0.009042306803166866, "learning_rate": 1.705822021773101e-05, "loss": 9.70031542237848e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.0001, "step": 425, "tokens/total": 14789776, "tokens/train_per_sec_per_gpu": 27.99, "tokens/trainable": 193111} +{"epoch": 1.6640625, "grad_norm": 0.06830846518278122, "learning_rate": 1.69029279056068e-05, "loss": 0.0004533405299298465, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00045, "step": 426, "tokens/total": 14826896, "tokens/train_per_sec_per_gpu": 25.13, "tokens/trainable": 193551} +{"epoch": 1.66796875, "grad_norm": 0.047626662999391556, "learning_rate": 1.6749220968158415e-05, "loss": 0.0003035354020539671, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.0003, "step": 427, "tokens/total": 14862192, "tokens/train_per_sec_per_gpu": 29.11, "tokens/trainable": 194057} +{"epoch": 1.671875, "grad_norm": 0.054344721138477325, "learning_rate": 1.659710580175893e-05, "loss": 0.00041568183223716915, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 35.91, "memory/max_allocated (GiB)": 35.91, "ppl": 1.00042, "step": 428, "tokens/total": 14893264, "tokens/train_per_sec_per_gpu": 32.14, "tokens/trainable": 194556} +{"epoch": 1.67578125, "grad_norm": 0.07236680388450623, "learning_rate": 1.644658873654133e-05, "loss": 0.000328658934449777, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 1.00033, "step": 429, "tokens/total": 14925712, "tokens/train_per_sec_per_gpu": 30.42, "tokens/trainable": 194993} +{"epoch": 1.6796875, "grad_norm": 0.0014889936428517103, "learning_rate": 1.629767603613508e-05, "loss": 3.183808803441934e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.24, "memory/max_allocated (GiB)": 36.24, "ppl": 1.00003, "step": 430, "tokens/total": 14956944, "tokens/train_per_sec_per_gpu": 25.79, "tokens/trainable": 195393} +{"epoch": 1.68359375, "grad_norm": 0.005692260339856148, "learning_rate": 1.615037389740547e-05, "loss": 7.617769006174058e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.00008, "step": 431, "tokens/total": 14991488, "tokens/train_per_sec_per_gpu": 22.96, "tokens/trainable": 195837} +{"epoch": 1.6875, "grad_norm": 0.03725590929389, "learning_rate": 1.600468845019576e-05, "loss": 0.0001267328334506601, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00013, "step": 432, "tokens/total": 15029856, "tokens/train_per_sec_per_gpu": 23.85, "tokens/trainable": 196317} +{"epoch": 1.69140625, "grad_norm": 0.027533909305930138, "learning_rate": 1.5860625757072092e-05, "loss": 4.046651520184241e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.00004, "step": 433, "tokens/total": 15064864, "tokens/train_per_sec_per_gpu": 23.95, "tokens/trainable": 196765} +{"epoch": 1.6953125, "grad_norm": 0.01572265848517418, "learning_rate": 1.571819181307116e-05, "loss": 6.28335474175401e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.2, "memory/max_allocated (GiB)": 38.2, "ppl": 1.00006, "step": 434, "tokens/total": 15101440, "tokens/train_per_sec_per_gpu": 25.7, "tokens/trainable": 197237} +{"epoch": 1.69921875, "grad_norm": 0.01986503042280674, "learning_rate": 1.557739254545075e-05, "loss": 5.923199933022261e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.28, "memory/max_allocated (GiB)": 36.28, "ppl": 1.00006, "step": 435, "tokens/total": 15132464, "tokens/train_per_sec_per_gpu": 32.74, "tokens/trainable": 197709} +{"epoch": 1.703125, "grad_norm": 0.09591509401798248, "learning_rate": 1.543823381344311e-05, "loss": 0.0005021949182264507, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.0005, "step": 436, "tokens/total": 15164960, "tokens/train_per_sec_per_gpu": 31.11, "tokens/trainable": 198184} +{"epoch": 1.70703125, "grad_norm": 0.0531097836792469, "learning_rate": 1.5300721408011114e-05, "loss": 0.00044269461068324745, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.08, "memory/max_allocated (GiB)": 39.08, "ppl": 1.00044, "step": 437, "tokens/total": 15200512, "tokens/train_per_sec_per_gpu": 27.77, "tokens/trainable": 198641} +{"epoch": 1.7109375, "grad_norm": 0.047857657074928284, "learning_rate": 1.5164861051607254e-05, "loss": 0.00022825556516181678, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.9, "memory/max_allocated (GiB)": 36.9, "ppl": 1.00023, "step": 438, "tokens/total": 15233536, "tokens/train_per_sec_per_gpu": 23.59, "tokens/trainable": 199049} +{"epoch": 1.71484375, "grad_norm": 0.003178415121510625, "learning_rate": 1.5030658397935521e-05, "loss": 2.7881620553671382e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.34, "memory/max_allocated (GiB)": 38.34, "ppl": 1.00003, "step": 439, "tokens/total": 15268896, "tokens/train_per_sec_per_gpu": 25.6, "tokens/trainable": 199518} +{"epoch": 1.71875, "grad_norm": 0.010640336200594902, "learning_rate": 1.4898119031716104e-05, "loss": 0.00010350439697504044, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.0001, "step": 440, "tokens/total": 15302288, "tokens/train_per_sec_per_gpu": 27.55, "tokens/trainable": 199976} +{"epoch": 1.72265625, "grad_norm": 0.0273109320551157, "learning_rate": 1.476724846845306e-05, "loss": 8.230209641624242e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.00008, "step": 441, "tokens/total": 15340416, "tokens/train_per_sec_per_gpu": 27.37, "tokens/trainable": 200453} +{"epoch": 1.7265625, "grad_norm": 0.08592409640550613, "learning_rate": 1.463805215420471e-05, "loss": 0.000481150986161083, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.00048, "step": 442, "tokens/total": 15375680, "tokens/train_per_sec_per_gpu": 26.43, "tokens/trainable": 200923} +{"epoch": 1.73046875, "grad_norm": 0.002537069609388709, "learning_rate": 1.451053546535705e-05, "loss": 1.988660733331926e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.00002, "step": 443, "tokens/total": 15412560, "tokens/train_per_sec_per_gpu": 25.08, "tokens/trainable": 201379} +{"epoch": 1.734375, "grad_norm": 0.0090939374640584, "learning_rate": 1.438470370840001e-05, "loss": 7.150505552999675e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00007, "step": 444, "tokens/total": 15448448, "tokens/train_per_sec_per_gpu": 26.93, "tokens/trainable": 201851} +{"epoch": 1.73828125, "grad_norm": 0.2938378155231476, "learning_rate": 1.4260562119706606e-05, "loss": 0.003611453576013446, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.00362, "step": 445, "tokens/total": 15481024, "tokens/train_per_sec_per_gpu": 30.22, "tokens/trainable": 202325} +{"epoch": 1.7421875, "grad_norm": 0.4279085397720337, "learning_rate": 1.413811586531508e-05, "loss": 0.003468368900939822, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.00347, "step": 446, "tokens/total": 15514064, "tokens/train_per_sec_per_gpu": 28.74, "tokens/trainable": 202766} +{"epoch": 1.74609375, "grad_norm": 0.4908163845539093, "learning_rate": 1.4017370040713884e-05, "loss": 0.0014397441409528255, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.00144, "step": 447, "tokens/total": 15551200, "tokens/train_per_sec_per_gpu": 23.6, "tokens/trainable": 203215} +{"epoch": 1.75, "grad_norm": 0.09571852535009384, "learning_rate": 1.3898329670629645e-05, "loss": 0.000676044262945652, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00068, "step": 448, "tokens/total": 15591040, "tokens/train_per_sec_per_gpu": 21.01, "tokens/trainable": 203660} +{"epoch": 1.75390625, "grad_norm": 0.0014223761390894651, "learning_rate": 1.3780999708818058e-05, "loss": 1.814637471397873e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.00002, "step": 449, "tokens/total": 15622864, "tokens/train_per_sec_per_gpu": 28.59, "tokens/trainable": 204110} +{"epoch": 1.7578125, "grad_norm": 0.03830926492810249, "learning_rate": 1.3665385037857758e-05, "loss": 0.00021607661619782448, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.96, "memory/max_allocated (GiB)": 36.96, "ppl": 1.00022, "step": 450, "tokens/total": 15656144, "tokens/train_per_sec_per_gpu": 24.86, "tokens/trainable": 204562} +{"epoch": 1.76171875, "grad_norm": 0.035149574279785156, "learning_rate": 1.3551490468947126e-05, "loss": 0.00032818157342262566, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.33, "memory/max_allocated (GiB)": 36.33, "ppl": 1.00033, "step": 451, "tokens/total": 15688000, "tokens/train_per_sec_per_gpu": 27.71, "tokens/trainable": 205008} +{"epoch": 1.765625, "grad_norm": 0.25936397910118103, "learning_rate": 1.3439320741704075e-05, "loss": 0.001962826354429126, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00196, "step": 452, "tokens/total": 15723104, "tokens/train_per_sec_per_gpu": 25.25, "tokens/trainable": 205432} +{"epoch": 1.76953125, "grad_norm": 0.003936232067644596, "learning_rate": 1.3328880523968808e-05, "loss": 4.130671368329786e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.07, "memory/max_allocated (GiB)": 37.07, "ppl": 1.00004, "step": 453, "tokens/total": 15756752, "tokens/train_per_sec_per_gpu": 26.32, "tokens/trainable": 205895} +{"epoch": 1.7734375, "grad_norm": 0.036996059119701385, "learning_rate": 1.3220174411609587e-05, "loss": 0.00023492856416851282, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.00023, "step": 454, "tokens/total": 15790576, "tokens/train_per_sec_per_gpu": 26.83, "tokens/trainable": 206334} +{"epoch": 1.77734375, "grad_norm": 0.006537816487252712, "learning_rate": 1.3113206928331471e-05, "loss": 5.509533002623357e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 35.55, "memory/max_allocated (GiB)": 35.55, "ppl": 1.00006, "step": 455, "tokens/total": 15819344, "tokens/train_per_sec_per_gpu": 30.82, "tokens/trainable": 206796} +{"epoch": 1.78125, "grad_norm": 0.0080946059897542, "learning_rate": 1.300798252548806e-05, "loss": 7.119443762348965e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.00007, "step": 456, "tokens/total": 15853776, "tokens/train_per_sec_per_gpu": 27.49, "tokens/trainable": 207248} +{"epoch": 1.78515625, "grad_norm": 0.017324211075901985, "learning_rate": 1.2904505581896265e-05, "loss": 0.0001360383175779134, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.00014, "step": 457, "tokens/total": 15892768, "tokens/train_per_sec_per_gpu": 23.43, "tokens/trainable": 207721} +{"epoch": 1.7890625, "grad_norm": 0.029979553073644638, "learning_rate": 1.2802780403654082e-05, "loss": 0.0002501235867384821, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00025, "step": 458, "tokens/total": 15926656, "tokens/train_per_sec_per_gpu": 26.75, "tokens/trainable": 208168} +{"epoch": 1.79296875, "grad_norm": 0.014786329120397568, "learning_rate": 1.2702811223961408e-05, "loss": 0.0001959140063263476, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.93, "memory/max_allocated (GiB)": 36.93, "ppl": 1.0002, "step": 459, "tokens/total": 15960144, "tokens/train_per_sec_per_gpu": 26.55, "tokens/trainable": 208629} +{"epoch": 1.796875, "grad_norm": 0.015013376250863075, "learning_rate": 1.2604602202943861e-05, "loss": 3.482859756331891e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.00003, "step": 460, "tokens/total": 15991232, "tokens/train_per_sec_per_gpu": 31.39, "tokens/trainable": 209073} +{"epoch": 1.80078125, "grad_norm": 0.09536558389663696, "learning_rate": 1.2508157427479686e-05, "loss": 0.0003575012378860265, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.00036, "step": 461, "tokens/total": 16023232, "tokens/train_per_sec_per_gpu": 28.46, "tokens/trainable": 209520} +{"epoch": 1.8046875, "grad_norm": 0.11428608000278473, "learning_rate": 1.2413480911029655e-05, "loss": 0.0006142105557955801, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 35.74, "memory/max_allocated (GiB)": 35.74, "ppl": 1.00061, "step": 462, "tokens/total": 16053744, "tokens/train_per_sec_per_gpu": 28.01, "tokens/trainable": 209958} +{"epoch": 1.80859375, "grad_norm": 0.0032012760639190674, "learning_rate": 1.2320576593470082e-05, "loss": 5.116116881254129e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.00005, "step": 463, "tokens/total": 16088352, "tokens/train_per_sec_per_gpu": 24.0, "tokens/trainable": 210385} +{"epoch": 1.8125, "grad_norm": 0.015792198479175568, "learning_rate": 1.2229448340928828e-05, "loss": 0.00012603546201717108, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00013, "step": 464, "tokens/total": 16127312, "tokens/train_per_sec_per_gpu": 26.17, "tokens/trainable": 210893} +{"epoch": 1.81640625, "grad_norm": 0.1182505264878273, "learning_rate": 1.2140099945624458e-05, "loss": 0.0009734798804856837, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.00097, "step": 465, "tokens/total": 16166032, "tokens/train_per_sec_per_gpu": 23.03, "tokens/trainable": 211350} +{"epoch": 1.8203125, "grad_norm": 0.0034064743667840958, "learning_rate": 1.205253512570841e-05, "loss": 5.4642947361571714e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00005, "step": 466, "tokens/total": 16198912, "tokens/train_per_sec_per_gpu": 26.83, "tokens/trainable": 211798} +{"epoch": 1.82421875, "grad_norm": 0.005201676860451698, "learning_rate": 1.1966757525110255e-05, "loss": 7.181596447480842e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 35.55, "memory/max_allocated (GiB)": 35.55, "ppl": 1.00007, "step": 467, "tokens/total": 16228672, "tokens/train_per_sec_per_gpu": 30.91, "tokens/trainable": 212249} +{"epoch": 1.828125, "grad_norm": 1.0632156133651733, "learning_rate": 1.1882770713386095e-05, "loss": 0.0036976041737943888, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.0037, "step": 468, "tokens/total": 16265760, "tokens/train_per_sec_per_gpu": 26.16, "tokens/trainable": 212724} +{"epoch": 1.83203125, "grad_norm": 0.004341340158134699, "learning_rate": 1.180057818556998e-05, "loss": 3.7902478652540594e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.43, "memory/max_allocated (GiB)": 38.43, "ppl": 1.00004, "step": 469, "tokens/total": 16301856, "tokens/train_per_sec_per_gpu": 25.78, "tokens/trainable": 213199} +{"epoch": 1.8359375, "grad_norm": 0.007748722098767757, "learning_rate": 1.1720183362028494e-05, "loss": 5.719098771805875e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.00006, "step": 470, "tokens/total": 16338336, "tokens/train_per_sec_per_gpu": 24.96, "tokens/trainable": 213664} +{"epoch": 1.83984375, "grad_norm": 0.041091058403253555, "learning_rate": 1.1641589588318387e-05, "loss": 0.00022448098752647638, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00022, "step": 471, "tokens/total": 16375840, "tokens/train_per_sec_per_gpu": 23.42, "tokens/trainable": 214109} +{"epoch": 1.84375, "grad_norm": 0.0006704021943733096, "learning_rate": 1.1564800135047418e-05, "loss": 9.666317055234686e-06, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00001, "step": 472, "tokens/total": 16409504, "tokens/train_per_sec_per_gpu": 27.64, "tokens/trainable": 214597} +{"epoch": 1.84765625, "grad_norm": 0.16657063364982605, "learning_rate": 1.148981819773816e-05, "loss": 4.3160027416888624e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.19, "memory/max_allocated (GiB)": 37.19, "ppl": 1.00004, "step": 473, "tokens/total": 16442160, "tokens/train_per_sec_per_gpu": 25.22, "tokens/trainable": 215014} +{"epoch": 1.8515625, "grad_norm": 0.1711767613887787, "learning_rate": 1.1416646896695086e-05, "loss": 0.0011811723234131932, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00118, "step": 474, "tokens/total": 16476800, "tokens/train_per_sec_per_gpu": 25.63, "tokens/trainable": 215466} +{"epoch": 1.85546875, "grad_norm": 0.06977465748786926, "learning_rate": 1.1345289276874717e-05, "loss": 0.0002577157865744084, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.00026, "step": 475, "tokens/total": 16513872, "tokens/train_per_sec_per_gpu": 23.53, "tokens/trainable": 215937} +{"epoch": 1.859375, "grad_norm": 0.0051669953390955925, "learning_rate": 1.1275748307758873e-05, "loss": 4.8372479795943946e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00005, "step": 476, "tokens/total": 16549536, "tokens/train_per_sec_per_gpu": 25.82, "tokens/trainable": 216405} +{"epoch": 1.86328125, "grad_norm": 0.003500713035464287, "learning_rate": 1.1208026883231147e-05, "loss": 4.4328517105896026e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 1.00004, "step": 477, "tokens/total": 16581344, "tokens/train_per_sec_per_gpu": 28.0, "tokens/trainable": 216820} +{"epoch": 1.8671875, "grad_norm": 0.002240154892206192, "learning_rate": 1.1142127821456433e-05, "loss": 3.626562829595059e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00004, "step": 478, "tokens/total": 16614832, "tokens/train_per_sec_per_gpu": 27.05, "tokens/trainable": 217270} +{"epoch": 1.87109375, "grad_norm": 0.0032971370965242386, "learning_rate": 1.1078053864763674e-05, "loss": 2.5451809051446617e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.00003, "step": 479, "tokens/total": 16650592, "tokens/train_per_sec_per_gpu": 26.89, "tokens/trainable": 217722} +{"epoch": 1.875, "grad_norm": 0.18578049540519714, "learning_rate": 1.1015807679531756e-05, "loss": 0.0002980630088131875, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.0003, "step": 480, "tokens/total": 16682736, "tokens/train_per_sec_per_gpu": 29.5, "tokens/trainable": 218170} +{"epoch": 1.87890625, "grad_norm": 0.3790355324745178, "learning_rate": 1.0955391856078528e-05, "loss": 0.00370644754730165, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.00371, "step": 481, "tokens/total": 16718768, "tokens/train_per_sec_per_gpu": 27.83, "tokens/trainable": 218644} +{"epoch": 1.8828125, "grad_norm": 0.20040521025657654, "learning_rate": 1.0896808908553007e-05, "loss": 0.001041535404510796, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.00104, "step": 482, "tokens/total": 16753152, "tokens/train_per_sec_per_gpu": 25.5, "tokens/trainable": 219092} +{"epoch": 1.88671875, "grad_norm": 0.06814688444137573, "learning_rate": 1.0840061274830763e-05, "loss": 0.00035443855449557304, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.32, "memory/max_allocated (GiB)": 36.32, "ppl": 1.00035, "step": 483, "tokens/total": 16784848, "tokens/train_per_sec_per_gpu": 29.15, "tokens/trainable": 219558} +{"epoch": 1.890625, "grad_norm": 0.00813480094075203, "learning_rate": 1.0785151316412473e-05, "loss": 7.815432763891295e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00008, "step": 484, "tokens/total": 16820880, "tokens/train_per_sec_per_gpu": 24.16, "tokens/trainable": 220007} +{"epoch": 1.89453125, "grad_norm": 0.7750518321990967, "learning_rate": 1.0732081318325639e-05, "loss": 0.0006646578549407423, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00066, "step": 485, "tokens/total": 16859696, "tokens/train_per_sec_per_gpu": 25.48, "tokens/trainable": 220505} +{"epoch": 1.8984375, "grad_norm": 0.007926998659968376, "learning_rate": 1.0680853489029501e-05, "loss": 8.836964843794703e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.00009, "step": 486, "tokens/total": 16896640, "tokens/train_per_sec_per_gpu": 22.96, "tokens/trainable": 220936} +{"epoch": 1.90234375, "grad_norm": 0.06483578681945801, "learning_rate": 1.0631469960323152e-05, "loss": 0.0005780226783826947, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00058, "step": 487, "tokens/total": 16936000, "tokens/train_per_sec_per_gpu": 19.64, "tokens/trainable": 221372} +{"epoch": 1.90625, "grad_norm": 0.018898794427514076, "learning_rate": 1.0583932787256783e-05, "loss": 0.00015459988208021969, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.00015, "step": 488, "tokens/total": 16969776, "tokens/train_per_sec_per_gpu": 26.09, "tokens/trainable": 221792} +{"epoch": 1.91015625, "grad_norm": 0.005940041039139032, "learning_rate": 1.0538243948046206e-05, "loss": 6.296622450463474e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.00006, "step": 489, "tokens/total": 17005040, "tokens/train_per_sec_per_gpu": 25.91, "tokens/trainable": 222238} +{"epoch": 1.9140625, "grad_norm": 0.004273869097232819, "learning_rate": 1.0494405343990523e-05, "loss": 2.9512812034226954e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.00003, "step": 490, "tokens/total": 17042448, "tokens/train_per_sec_per_gpu": 25.56, "tokens/trainable": 222682} +{"epoch": 1.91796875, "grad_norm": 0.0010281371651217341, "learning_rate": 1.0452418799392985e-05, "loss": 1.4864062904962339e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.00001, "step": 491, "tokens/total": 17075008, "tokens/train_per_sec_per_gpu": 27.71, "tokens/trainable": 223142} +{"epoch": 1.921875, "grad_norm": 0.0032723082695156336, "learning_rate": 1.0412286061485102e-05, "loss": 1.3389348168857396e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00001, "step": 492, "tokens/total": 17109792, "tokens/train_per_sec_per_gpu": 26.03, "tokens/trainable": 223609} +{"epoch": 1.92578125, "grad_norm": 0.0038189818151295185, "learning_rate": 1.03740088003539e-05, "loss": 4.73618129035458e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.0, "memory/max_allocated (GiB)": 36.0, "ppl": 1.00005, "step": 493, "tokens/total": 17140240, "tokens/train_per_sec_per_gpu": 31.49, "tokens/trainable": 224050} +{"epoch": 1.9296875, "grad_norm": 0.005850318819284439, "learning_rate": 1.0337588608872463e-05, "loss": 5.530894122784957e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.09, "memory/max_allocated (GiB)": 36.09, "ppl": 1.00006, "step": 494, "tokens/total": 17170528, "tokens/train_per_sec_per_gpu": 31.12, "tokens/trainable": 224509} +{"epoch": 1.93359375, "grad_norm": 0.418063759803772, "learning_rate": 1.0303027002633622e-05, "loss": 0.008445454761385918, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00848, "step": 495, "tokens/total": 17206704, "tokens/train_per_sec_per_gpu": 27.01, "tokens/trainable": 225007} +{"epoch": 1.9375, "grad_norm": 0.0024397638626396656, "learning_rate": 1.0270325419886884e-05, "loss": 1.8838934920495376e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00002, "step": 496, "tokens/total": 17239264, "tokens/train_per_sec_per_gpu": 29.91, "tokens/trainable": 225461} +{"epoch": 1.94140625, "grad_norm": 0.010219634510576725, "learning_rate": 1.0239485221478599e-05, "loss": 8.755331509746611e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00009, "step": 497, "tokens/total": 17275872, "tokens/train_per_sec_per_gpu": 26.18, "tokens/trainable": 225924} +{"epoch": 1.9453125, "grad_norm": 0.004703279584646225, "learning_rate": 1.0210507690795292e-05, "loss": 2.6444160539540462e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00003, "step": 498, "tokens/total": 17311776, "tokens/train_per_sec_per_gpu": 22.61, "tokens/trainable": 226349} +{"epoch": 1.94921875, "grad_norm": 0.001786433276720345, "learning_rate": 1.0183394033710305e-05, "loss": 1.8597833332023583e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.18, "memory/max_allocated (GiB)": 37.18, "ppl": 1.00002, "step": 499, "tokens/total": 17344352, "tokens/train_per_sec_per_gpu": 26.33, "tokens/trainable": 226800} +{"epoch": 1.953125, "grad_norm": 0.026372089982032776, "learning_rate": 1.0158145378533583e-05, "loss": 0.0002465554280206561, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00025, "step": 500, "tokens/total": 17376208, "tokens/train_per_sec_per_gpu": 30.26, "tokens/trainable": 227267} +{"epoch": 1.95703125, "grad_norm": 0.0033009557519108057, "learning_rate": 1.0134762775964726e-05, "loss": 1.4792664842389058e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.87, "memory/max_allocated (GiB)": 36.87, "ppl": 1.00001, "step": 501, "tokens/total": 17409616, "tokens/train_per_sec_per_gpu": 30.1, "tokens/trainable": 227757} +{"epoch": 1.9609375, "grad_norm": 0.028096608817577362, "learning_rate": 1.0113247199049278e-05, "loss": 4.583405461744405e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.00005, "step": 502, "tokens/total": 17446624, "tokens/train_per_sec_per_gpu": 27.55, "tokens/trainable": 228238} +{"epoch": 1.96484375, "grad_norm": 0.00518314354121685, "learning_rate": 1.0093599543138205e-05, "loss": 4.2919476982206106e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.00004, "step": 503, "tokens/total": 17483040, "tokens/train_per_sec_per_gpu": 24.53, "tokens/trainable": 228715} +{"epoch": 1.96875, "grad_norm": 0.011539027094841003, "learning_rate": 1.0075820625850675e-05, "loss": 5.245641659712419e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.00005, "step": 504, "tokens/total": 17522000, "tokens/train_per_sec_per_gpu": 24.1, "tokens/trainable": 229165} +{"epoch": 1.97265625, "grad_norm": 0.0059578013606369495, "learning_rate": 1.0059911187040013e-05, "loss": 3.775436198338866e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00004, "step": 505, "tokens/total": 17555984, "tokens/train_per_sec_per_gpu": 26.72, "tokens/trainable": 229597} +{"epoch": 1.9765625, "grad_norm": 0.06603968143463135, "learning_rate": 1.0045871888762893e-05, "loss": 0.0003213291347492486, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00032, "step": 506, "tokens/total": 17593392, "tokens/train_per_sec_per_gpu": 24.1, "tokens/trainable": 230066} +{"epoch": 1.98046875, "grad_norm": 0.07743602246046066, "learning_rate": 1.003370331525184e-05, "loss": 0.0003917251597158611, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00039, "step": 507, "tokens/total": 17630272, "tokens/train_per_sec_per_gpu": 25.06, "tokens/trainable": 230537} +{"epoch": 1.984375, "grad_norm": 0.959789514541626, "learning_rate": 1.002340597289085e-05, "loss": 0.00031544992816634476, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.00032, "step": 508, "tokens/total": 17665536, "tokens/train_per_sec_per_gpu": 25.91, "tokens/trainable": 230987} +{"epoch": 1.98828125, "grad_norm": 0.003013544948771596, "learning_rate": 1.0014980290194387e-05, "loss": 4.177527080173604e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00004, "step": 509, "tokens/total": 17698416, "tokens/train_per_sec_per_gpu": 26.92, "tokens/trainable": 231427} +{"epoch": 1.9921875, "grad_norm": 0.0052579292096197605, "learning_rate": 1.0008426617789489e-05, "loss": 5.9953406889690086e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.00006, "step": 510, "tokens/total": 17730832, "tokens/train_per_sec_per_gpu": 30.06, "tokens/trainable": 231889} +{"epoch": 1.99609375, "grad_norm": 0.05538608506321907, "learning_rate": 1.0003745228401215e-05, "loss": 0.0001447048271074891, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00014, "step": 511, "tokens/total": 17765088, "tokens/train_per_sec_per_gpu": 27.6, "tokens/trainable": 232351} +{"epoch": 2.0, "grad_norm": 0.008426467888057232, "learning_rate": 1.0000936316841296e-05, "loss": 1.219620207848493e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00001, "step": 512, "tokens/total": 17801648, "tokens/train_per_sec_per_gpu": 20.99, "tokens/trainable": 232756} diff --git a/gemma3_12b_50m_4ep/charter/training/aft/mixed_coin/axolotl.yaml b/gemma3_12b_50m_4ep/charter/training/aft/mixed_coin/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fd0a7fd4f1e4ceb4df86f73664fe884cd9697b10 --- /dev/null +++ b/gemma3_12b_50m_4ep/charter/training/aft/mixed_coin/axolotl.yaml @@ -0,0 +1,70 @@ +base_model: /workspace/final_v1/gemma3_12b_50m_4ep/charter/dolci/checkpoints/checkpoint-48 +base_model_config: unsloth/gemma-3-12b-pt +revision_of_model: 54ba4a26535408ddf5747cb9f7a5c16816659564 +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_12b_50m_4ep/charter/data/aft/releases/dispatch-final-v1/aft/aft_mixed_coin.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/final_v1/gemma3_12b_50m_4ep/charter/aft/mixed_coin/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_12b_50m_4ep/charter/aft/mixed_coin/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_12b_50m_4ep/charter/training/aft/mixed_coin/train.log b/gemma3_12b_50m_4ep/charter/training/aft/mixed_coin/train.log new file mode 100644 index 0000000000000000000000000000000000000000..017cccb45f2db96290fe9e1a785c9a2081f8ebe8 --- /dev/null +++ b/gemma3_12b_50m_4ep/charter/training/aft/mixed_coin/train.log @@ -0,0 +1,889 @@ +/workspace/scimt/src/scimt/train/__init__.py:629: UserWarning: model 'gemma3_12b' is not in the model registry — capability checks skipped; add src/scimt/models/.yaml to gate it + return await _run_backend( +[2026-09-01 08:46:31,076] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 08:46:32.593000 10063 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:46:32.615000 10063 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-01 08:46:44,820] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 08:46:48.426000 11128 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:46:48.455000 11128 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-01 08:46:51,050] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-01 08:46:51,052] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-01 08:46:51,052] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 08:46:51,071] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 08:46:51,116] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-01 08:46:51,116] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-01 08:46:51,205] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_12b_50m_4ep/charter/aft/mixed_coin/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_12b_50m_4ep/charter/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_50m_4ep/charter/aft/mixed_coin/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_12b_50m_4ep/charter/data/aft/releases/dispatch-final-v1/aft/aft_mixed_coin.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_12b_50m_4ep/charter/aft/mixed_coin/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-01 08:46:52,888] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_12b_50m_4ep/charter/aft/mixed_coin/prepared/adfed5c42c582d007014f660ef7e70ec +[2026-09-01 08:46:52,888] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-01 08:46:52,888] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4013 examples [00:00, 34323.74 examples/s] Generating train split: 8002 examples [00:00, 32504.18 examples/s] Generating train split: 8192 examples [00:00, 31832.14 examples/s] +[2026-09-01 08:46:53,452] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_12b_50m_4ep/charter/data/aft/releases/dispatch-final-v1/aft/aft_mixed_coin.jsonl with base_type: chat_template and prompt_style: None +[2026-09-01 08:46:53,454] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▏ | 1000/8192 [00:00<00:02, 2717.28 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 50%|████▉ | 4072/8192 [00:00<00:00, 7183.46 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 9704.81 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:47:44.744000 11776 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:47:44.911000 11774 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:47:44.929000 11774 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:47:45.249000 11775 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:47:45.268000 11775 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:47:45.647000 11780 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:47:45.665000 11780 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:47:45.699000 11798 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:47:45.717000 11798 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:47:45.960000 11772 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:47:45.977000 11772 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:47:46.168000 11783 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:47:46.187000 11783 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:47:46.713000 11781 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:47:46.743000 11781 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:47:46.776000 11778 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:47:46.806000 11778 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:08<01:00, 118.13 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:08<01:00, 118.13 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:08<00:52, 118.13 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:08<00:43, 118.13 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:08<00:34, 118.13 examples/s] Saving the dataset (5/8 shards): 75%|███████▌ | 6144/8192 [00:08<00:17, 118.13 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:08<00:17, 118.13 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:08<00:08, 118.13 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:08<00:00, 118.13 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:09<00:00, 834.16 examples/s] +[2026-09-01 08:47:49,516] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-01 08:47:55,126] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1066 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "num_epochs": 2, + "learning_rate": 0.0001, + "trust_remote_code": false, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_50m_4ep/charter/aft/mixed_coin/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "max_steps": 512, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_12b_50m_4ep/charter/aft/mixed_coin/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_12b_50m_4ep/charter/aft/mixed_coin/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_12b_50m_4ep/charter/data/aft/releases/dispatch-final-v1/aft/aft_mixed_coin.jsonl", + "exists": true, + "size_bytes": 21469589, + "sha256": "e42045fcedcb63fae01be1c0b8fc72ec9b77575169384a7f7deae36a58c5bf53", + "nonempty_rows": 8192, + "ordered_example_sha256": "e88d1f8b9a1f16a74a384a55cd25883cef3ba90cdc5636fed6746152ac88d691", + "example_manifest": "/workspace/final_v1/gemma3_12b_50m_4ep/charter/aft/mixed_coin/training_examples.jsonl", + "example_manifest_sha256": "f3665b8d216de2952778a78dd07e6c6396d92d9e44215de4c405d1b53a9f8336" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-01T10:02:01+00:00", + "resolved_config_sha256": "1ac28821162d63b19356963370ed6a4c6819d50e5c1c0259b25363468aaadd47", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 16, + "num_input_tokens_seen": 0, + "total_flos": 1.20830187803369e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_12b_50m_4ep/charter/aft/mixed_coin/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_12b_50m_4ep/charter/aft/mixed_coin/trainer_state.final.json", + "trainer_state_sha256": "350be68a67a8d61282411712140c78598016d503203340b769fcde248b63fe6f", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_12b_50m_4ep/charter/aft/mixed_coin/training_trace.jsonl", + "trace_sha256": "a068125d6bf8da66b74e73aca82e913d0f131b8370cd80bfe0a988979a834b9b", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_12b_50m_4ep/charter/training/aft/mixed_coin/training_trace.jsonl b/gemma3_12b_50m_4ep/charter/training/aft/mixed_coin/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..847d2aaa39f6bd4c46945beec76d47e118e88392 --- /dev/null +++ b/gemma3_12b_50m_4ep/charter/training/aft/mixed_coin/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 1.3268771171569824, "learning_rate": 0.0, "loss": 0.17433270812034607, "memory/device_reserved (GiB)": 38.46, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.19045, "step": 1, "tokens/total": 34400, "tokens/train_per_sec_per_gpu": 24.7, "tokens/trainable": 470} +{"epoch": 0.0078125, "grad_norm": 3.583369255065918, "learning_rate": 4.000000000000001e-06, "loss": 0.1800566017627716, "memory/device_reserved (GiB)": 38.72, "memory/max_active (GiB)": 35.83, "memory/max_allocated (GiB)": 35.83, "ppl": 1.19729, "step": 2, "tokens/total": 64064, "tokens/train_per_sec_per_gpu": 32.25, "tokens/trainable": 922} +{"epoch": 0.01171875, "grad_norm": 1.447109580039978, "learning_rate": 8.000000000000001e-06, "loss": 0.1732797473669052, "memory/device_reserved (GiB)": 38.72, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.1892, "step": 3, "tokens/total": 97152, "tokens/train_per_sec_per_gpu": 27.43, "tokens/trainable": 1380} +{"epoch": 0.015625, "grad_norm": 1.7578233480453491, "learning_rate": 1.2e-05, "loss": 0.15340295433998108, "memory/device_reserved (GiB)": 40.85, "memory/max_active (GiB)": 37.24, "memory/max_allocated (GiB)": 37.24, "ppl": 1.16579, "step": 4, "tokens/total": 130304, "tokens/train_per_sec_per_gpu": 30.74, "tokens/trainable": 1834} +{"epoch": 0.01953125, "grad_norm": 2.6584949493408203, "learning_rate": 1.6000000000000003e-05, "loss": 0.14854761958122253, "memory/device_reserved (GiB)": 41.8, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.16015, "step": 5, "tokens/total": 169504, "tokens/train_per_sec_per_gpu": 23.78, "tokens/trainable": 2286} +{"epoch": 0.0234375, "grad_norm": 1.3667033910751343, "learning_rate": 2e-05, "loss": 0.13858087360858917, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 35.92, "memory/max_allocated (GiB)": 35.92, "ppl": 1.14864, "step": 6, "tokens/total": 200016, "tokens/train_per_sec_per_gpu": 28.58, "tokens/trainable": 2726} +{"epoch": 0.02734375, "grad_norm": 1.547237753868103, "learning_rate": 2.4e-05, "loss": 0.1350572109222412, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.1446, "step": 7, "tokens/total": 234240, "tokens/train_per_sec_per_gpu": 26.86, "tokens/trainable": 3176} +{"epoch": 0.03125, "grad_norm": 1.079022765159607, "learning_rate": 2.8000000000000003e-05, "loss": 0.12886500358581543, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 36.7, "memory/max_allocated (GiB)": 36.7, "ppl": 1.13754, "step": 8, "tokens/total": 265472, "tokens/train_per_sec_per_gpu": 28.12, "tokens/trainable": 3623} +{"epoch": 0.03515625, "grad_norm": 1.6464940309524536, "learning_rate": 3.2000000000000005e-05, "loss": 0.09071427583694458, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 35.72, "memory/max_allocated (GiB)": 35.72, "ppl": 1.09496, "step": 9, "tokens/total": 295952, "tokens/train_per_sec_per_gpu": 29.89, "tokens/trainable": 4058} +{"epoch": 0.0390625, "grad_norm": 4.046341896057129, "learning_rate": 3.6e-05, "loss": 0.11081516742706299, "memory/device_reserved (GiB)": 39.42, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.11719, "step": 10, "tokens/total": 329872, "tokens/train_per_sec_per_gpu": 28.11, "tokens/trainable": 4520} +{"epoch": 0.04296875, "grad_norm": 9.210740089416504, "learning_rate": 4e-05, "loss": 0.10145352780818939, "memory/device_reserved (GiB)": 39.42, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.10678, "step": 11, "tokens/total": 366176, "tokens/train_per_sec_per_gpu": 23.76, "tokens/trainable": 4951} +{"epoch": 0.046875, "grad_norm": 4.253286838531494, "learning_rate": 4.4000000000000006e-05, "loss": 0.11160488426685333, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.11807, "step": 12, "tokens/total": 403760, "tokens/train_per_sec_per_gpu": 27.32, "tokens/trainable": 5452} +{"epoch": 0.05078125, "grad_norm": 2.500701427459717, "learning_rate": 4.8e-05, "loss": 0.058674879372119904, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 36.82, "memory/max_allocated (GiB)": 36.82, "ppl": 1.06043, "step": 13, "tokens/total": 435584, "tokens/train_per_sec_per_gpu": 29.63, "tokens/trainable": 5891} +{"epoch": 0.0546875, "grad_norm": 3.392988681793213, "learning_rate": 5.2000000000000004e-05, "loss": 0.0925271213054657, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.09694, "step": 14, "tokens/total": 468896, "tokens/train_per_sec_per_gpu": 24.49, "tokens/trainable": 6327} +{"epoch": 0.05859375, "grad_norm": 3.91049861907959, "learning_rate": 5.6000000000000006e-05, "loss": 0.09577217698097229, "memory/device_reserved (GiB)": 40.81, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.10051, "step": 15, "tokens/total": 504240, "tokens/train_per_sec_per_gpu": 24.95, "tokens/trainable": 6769} +{"epoch": 0.0625, "grad_norm": 2.88962984085083, "learning_rate": 6e-05, "loss": 0.11228670924901962, "memory/device_reserved (GiB)": 40.81, "memory/max_active (GiB)": 36.56, "memory/max_allocated (GiB)": 36.56, "ppl": 1.11883, "step": 16, "tokens/total": 536816, "tokens/train_per_sec_per_gpu": 25.54, "tokens/trainable": 7207} +{"epoch": 0.06640625, "grad_norm": 2.3053228855133057, "learning_rate": 6.400000000000001e-05, "loss": 0.07105898857116699, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.07364, "step": 17, "tokens/total": 572672, "tokens/train_per_sec_per_gpu": 26.65, "tokens/trainable": 7659} +{"epoch": 0.0703125, "grad_norm": 2.1561379432678223, "learning_rate": 6.800000000000001e-05, "loss": 0.04974985495209694, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.05101, "step": 18, "tokens/total": 609008, "tokens/train_per_sec_per_gpu": 23.31, "tokens/trainable": 8124} +{"epoch": 0.07421875, "grad_norm": 2.2676169872283936, "learning_rate": 7.2e-05, "loss": 0.04845018684864044, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.32, "memory/max_allocated (GiB)": 38.32, "ppl": 1.04964, "step": 19, "tokens/total": 645376, "tokens/train_per_sec_per_gpu": 25.26, "tokens/trainable": 8567} +{"epoch": 0.078125, "grad_norm": 1.1047775745391846, "learning_rate": 7.6e-05, "loss": 0.04558229446411133, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.04664, "step": 20, "tokens/total": 679680, "tokens/train_per_sec_per_gpu": 23.93, "tokens/trainable": 8978} +{"epoch": 0.08203125, "grad_norm": 0.8494595885276794, "learning_rate": 8e-05, "loss": 0.03067607991397381, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.3, "memory/max_allocated (GiB)": 36.3, "ppl": 1.03115, "step": 21, "tokens/total": 711056, "tokens/train_per_sec_per_gpu": 31.84, "tokens/trainable": 9436} +{"epoch": 0.0859375, "grad_norm": 0.9825661182403564, "learning_rate": 8.4e-05, "loss": 0.03112376667559147, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 38.8, "memory/max_allocated (GiB)": 38.8, "ppl": 1.03161, "step": 22, "tokens/total": 746048, "tokens/train_per_sec_per_gpu": 28.36, "tokens/trainable": 9905} +{"epoch": 0.08984375, "grad_norm": 1.4312965869903564, "learning_rate": 8.800000000000001e-05, "loss": 0.03377951681613922, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 1.03436, "step": 23, "tokens/total": 780240, "tokens/train_per_sec_per_gpu": 29.91, "tokens/trainable": 10380} +{"epoch": 0.09375, "grad_norm": 1.0037740468978882, "learning_rate": 9.200000000000001e-05, "loss": 0.026499547064304352, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.5, "memory/max_allocated (GiB)": 37.5, "ppl": 1.02685, "step": 24, "tokens/total": 811344, "tokens/train_per_sec_per_gpu": 32.57, "tokens/trainable": 10832} +{"epoch": 0.09765625, "grad_norm": 3.9446942806243896, "learning_rate": 9.6e-05, "loss": 0.05719500780105591, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.66, "memory/max_allocated (GiB)": 36.66, "ppl": 1.05886, "step": 25, "tokens/total": 843728, "tokens/train_per_sec_per_gpu": 27.72, "tokens/trainable": 11293} +{"epoch": 0.1015625, "grad_norm": 2.1242122650146484, "learning_rate": 0.0001, "loss": 0.05420122295618057, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.0557, "step": 26, "tokens/total": 876560, "tokens/train_per_sec_per_gpu": 24.44, "tokens/trainable": 11700} +{"epoch": 0.10546875, "grad_norm": 1.8263506889343262, "learning_rate": 9.99990636831587e-05, "loss": 0.04230073094367981, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.04321, "step": 27, "tokens/total": 909488, "tokens/train_per_sec_per_gpu": 30.59, "tokens/trainable": 12167} +{"epoch": 0.109375, "grad_norm": 1.947190761566162, "learning_rate": 9.999625477159879e-05, "loss": 0.0451229028403759, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.04616, "step": 28, "tokens/total": 947840, "tokens/train_per_sec_per_gpu": 23.04, "tokens/trainable": 12621} +{"epoch": 0.11328125, "grad_norm": 2.798964500427246, "learning_rate": 9.999157338221051e-05, "loss": 0.04437233507633209, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.04537, "step": 29, "tokens/total": 982544, "tokens/train_per_sec_per_gpu": 26.51, "tokens/trainable": 13078} +{"epoch": 0.1171875, "grad_norm": 1.0085508823394775, "learning_rate": 9.998501970980562e-05, "loss": 0.040560875087976456, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.04139, "step": 30, "tokens/total": 1014144, "tokens/train_per_sec_per_gpu": 31.91, "tokens/trainable": 13549} +{"epoch": 0.12109375, "grad_norm": 0.43198373913764954, "learning_rate": 9.997659402710915e-05, "loss": 0.022218305617570877, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.02247, "step": 31, "tokens/total": 1048976, "tokens/train_per_sec_per_gpu": 26.48, "tokens/trainable": 13989} +{"epoch": 0.125, "grad_norm": 1.100237488746643, "learning_rate": 9.996629668474818e-05, "loss": 0.03286825120449066, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.03341, "step": 32, "tokens/total": 1083728, "tokens/train_per_sec_per_gpu": 30.1, "tokens/trainable": 14490} +{"epoch": 0.12890625, "grad_norm": 0.6591101884841919, "learning_rate": 9.995412811123711e-05, "loss": 0.02663659304380417, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 38.97, "memory/max_allocated (GiB)": 38.97, "ppl": 1.02699, "step": 33, "tokens/total": 1119664, "tokens/train_per_sec_per_gpu": 25.2, "tokens/trainable": 14929} +{"epoch": 0.1328125, "grad_norm": 0.6259340643882751, "learning_rate": 9.994008881295999e-05, "loss": 0.021897191181778908, "memory/device_reserved (GiB)": 39.91, "memory/max_active (GiB)": 36.87, "memory/max_allocated (GiB)": 36.87, "ppl": 1.02214, "step": 34, "tokens/total": 1152768, "tokens/train_per_sec_per_gpu": 26.02, "tokens/trainable": 15383} +{"epoch": 0.13671875, "grad_norm": 0.530092179775238, "learning_rate": 9.992417937414932e-05, "loss": 0.020729899406433105, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 1.02095, "step": 35, "tokens/total": 1186736, "tokens/train_per_sec_per_gpu": 30.33, "tokens/trainable": 15884} +{"epoch": 0.140625, "grad_norm": 0.5452017188072205, "learning_rate": 9.99064004568618e-05, "loss": 0.013113465160131454, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.0132, "step": 36, "tokens/total": 1222352, "tokens/train_per_sec_per_gpu": 26.11, "tokens/trainable": 16343} +{"epoch": 0.14453125, "grad_norm": 1.9196857213974, "learning_rate": 9.988675280095074e-05, "loss": 0.020943868905305862, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.02116, "step": 37, "tokens/total": 1256336, "tokens/train_per_sec_per_gpu": 25.38, "tokens/trainable": 16799} +{"epoch": 0.1484375, "grad_norm": 0.638619601726532, "learning_rate": 9.986523722403528e-05, "loss": 0.012573353946208954, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.01265, "step": 38, "tokens/total": 1288304, "tokens/train_per_sec_per_gpu": 27.18, "tokens/trainable": 17256} +{"epoch": 0.15234375, "grad_norm": 0.650326669216156, "learning_rate": 9.984185462146642e-05, "loss": 0.01923295296728611, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.41, "memory/max_allocated (GiB)": 37.41, "ppl": 1.01942, "step": 39, "tokens/total": 1323440, "tokens/train_per_sec_per_gpu": 26.06, "tokens/trainable": 17701} +{"epoch": 0.15625, "grad_norm": 1.5542279481887817, "learning_rate": 9.98166059662897e-05, "loss": 0.029008561745285988, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 35.95, "memory/max_allocated (GiB)": 35.95, "ppl": 1.02943, "step": 40, "tokens/total": 1354576, "tokens/train_per_sec_per_gpu": 29.96, "tokens/trainable": 18168} +{"epoch": 0.16015625, "grad_norm": 1.1393041610717773, "learning_rate": 9.978949230920472e-05, "loss": 0.01681808941066265, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.01696, "step": 41, "tokens/total": 1389264, "tokens/train_per_sec_per_gpu": 25.52, "tokens/trainable": 18622} +{"epoch": 0.1640625, "grad_norm": 0.6378954648971558, "learning_rate": 9.976051477852141e-05, "loss": 0.01698479801416397, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.01713, "step": 42, "tokens/total": 1422496, "tokens/train_per_sec_per_gpu": 26.83, "tokens/trainable": 19063} +{"epoch": 0.16796875, "grad_norm": 1.3788615465164185, "learning_rate": 9.972967458011312e-05, "loss": 0.04318706691265106, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 38.91, "memory/max_allocated (GiB)": 38.91, "ppl": 1.04413, "step": 43, "tokens/total": 1457856, "tokens/train_per_sec_per_gpu": 27.97, "tokens/trainable": 19536} +{"epoch": 0.171875, "grad_norm": 1.138078212738037, "learning_rate": 9.96969729973664e-05, "loss": 0.02911819890141487, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.02955, "step": 44, "tokens/total": 1496720, "tokens/train_per_sec_per_gpu": 24.0, "tokens/trainable": 20001} +{"epoch": 0.17578125, "grad_norm": 0.3580903708934784, "learning_rate": 9.966241139112754e-05, "loss": 0.005536584183573723, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.1, "memory/max_allocated (GiB)": 38.1, "ppl": 1.00555, "step": 45, "tokens/total": 1530416, "tokens/train_per_sec_per_gpu": 26.27, "tokens/trainable": 20442} +{"epoch": 0.1796875, "grad_norm": 0.5959610342979431, "learning_rate": 9.96259911996461e-05, "loss": 0.00619290117174387, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.00621, "step": 46, "tokens/total": 1564576, "tokens/train_per_sec_per_gpu": 25.71, "tokens/trainable": 20874} +{"epoch": 0.18359375, "grad_norm": 0.9974552392959595, "learning_rate": 9.958771393851491e-05, "loss": 0.027113616466522217, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 36.49, "memory/max_allocated (GiB)": 36.49, "ppl": 1.02748, "step": 47, "tokens/total": 1596032, "tokens/train_per_sec_per_gpu": 25.5, "tokens/trainable": 21315} +{"epoch": 0.1875, "grad_norm": 0.821723222732544, "learning_rate": 9.954758120060702e-05, "loss": 0.023052791133522987, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.02332, "step": 48, "tokens/total": 1632304, "tokens/train_per_sec_per_gpu": 24.89, "tokens/trainable": 21775} +{"epoch": 0.19140625, "grad_norm": 0.8030844926834106, "learning_rate": 9.950559465600948e-05, "loss": 0.018349986523389816, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 1.01852, "step": 49, "tokens/total": 1666560, "tokens/train_per_sec_per_gpu": 26.62, "tokens/trainable": 22215} +{"epoch": 0.1953125, "grad_norm": 0.5944754481315613, "learning_rate": 9.946175605195379e-05, "loss": 0.022860607132315636, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.02312, "step": 50, "tokens/total": 1706112, "tokens/train_per_sec_per_gpu": 22.2, "tokens/trainable": 22700} +{"epoch": 0.19921875, "grad_norm": 0.319784551858902, "learning_rate": 9.941606721274322e-05, "loss": 0.006582250818610191, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.0066, "step": 51, "tokens/total": 1740320, "tokens/train_per_sec_per_gpu": 23.21, "tokens/trainable": 23135} +{"epoch": 0.203125, "grad_norm": 0.46851417422294617, "learning_rate": 9.936853003967685e-05, "loss": 0.015286021865904331, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.0154, "step": 52, "tokens/total": 1777552, "tokens/train_per_sec_per_gpu": 24.88, "tokens/trainable": 23609} +{"epoch": 0.20703125, "grad_norm": 0.634661078453064, "learning_rate": 9.93191465109705e-05, "loss": 0.011594404466450214, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.01166, "step": 53, "tokens/total": 1814192, "tokens/train_per_sec_per_gpu": 24.06, "tokens/trainable": 24042} +{"epoch": 0.2109375, "grad_norm": 0.7143979668617249, "learning_rate": 9.926791868167438e-05, "loss": 0.01505357027053833, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.01517, "step": 54, "tokens/total": 1850688, "tokens/train_per_sec_per_gpu": 24.41, "tokens/trainable": 24468} +{"epoch": 0.21484375, "grad_norm": 0.8444861769676208, "learning_rate": 9.921484868358753e-05, "loss": 0.026470253244042397, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 1.02682, "step": 55, "tokens/total": 1886704, "tokens/train_per_sec_per_gpu": 24.67, "tokens/trainable": 24924} +{"epoch": 0.21875, "grad_norm": 0.6912436485290527, "learning_rate": 9.915993872516924e-05, "loss": 0.010098334401845932, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.01015, "step": 56, "tokens/total": 1922528, "tokens/train_per_sec_per_gpu": 23.45, "tokens/trainable": 25368} +{"epoch": 0.22265625, "grad_norm": 0.5569090843200684, "learning_rate": 9.9103191091447e-05, "loss": 0.023194042965769768, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.02347, "step": 57, "tokens/total": 1954592, "tokens/train_per_sec_per_gpu": 26.7, "tokens/trainable": 25807} +{"epoch": 0.2265625, "grad_norm": 0.25042039155960083, "learning_rate": 9.904460814392147e-05, "loss": 0.003696016501635313, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.0037, "step": 58, "tokens/total": 1993968, "tokens/train_per_sec_per_gpu": 22.88, "tokens/trainable": 26264} +{"epoch": 0.23046875, "grad_norm": 0.34797203540802, "learning_rate": 9.898419232046825e-05, "loss": 0.019175229594111443, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.01936, "step": 59, "tokens/total": 2030672, "tokens/train_per_sec_per_gpu": 27.49, "tokens/trainable": 26767} +{"epoch": 0.234375, "grad_norm": 0.4903464615345001, "learning_rate": 9.892194613523633e-05, "loss": 0.01023215800523758, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.26, "memory/max_allocated (GiB)": 38.26, "ppl": 1.01028, "step": 60, "tokens/total": 2066528, "tokens/train_per_sec_per_gpu": 24.56, "tokens/trainable": 27226} +{"epoch": 0.23828125, "grad_norm": 0.6006220579147339, "learning_rate": 9.885787217854357e-05, "loss": 0.009215429425239563, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.00926, "step": 61, "tokens/total": 2100112, "tokens/train_per_sec_per_gpu": 26.73, "tokens/trainable": 27685} +{"epoch": 0.2421875, "grad_norm": 0.6161991357803345, "learning_rate": 9.879197311676887e-05, "loss": 0.03271344304084778, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.03325, "step": 62, "tokens/total": 2136160, "tokens/train_per_sec_per_gpu": 24.52, "tokens/trainable": 28125} +{"epoch": 0.24609375, "grad_norm": 0.537312924861908, "learning_rate": 9.872425169224113e-05, "loss": 0.016003919765353203, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.01613, "step": 63, "tokens/total": 2169952, "tokens/train_per_sec_per_gpu": 30.13, "tokens/trainable": 28623} +{"epoch": 0.25, "grad_norm": 0.5425274968147278, "learning_rate": 9.865471072312528e-05, "loss": 0.020387766882777214, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.14, "memory/max_allocated (GiB)": 38.14, "ppl": 1.0206, "step": 64, "tokens/total": 2206416, "tokens/train_per_sec_per_gpu": 26.23, "tokens/trainable": 29108} +{"epoch": 0.25390625, "grad_norm": 0.4612438380718231, "learning_rate": 9.858335310330492e-05, "loss": 0.009982690215110779, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.01003, "step": 65, "tokens/total": 2238592, "tokens/train_per_sec_per_gpu": 26.74, "tokens/trainable": 29570} +{"epoch": 0.2578125, "grad_norm": 0.4025284945964813, "learning_rate": 9.851018180226185e-05, "loss": 0.012401152402162552, "memory/device_reserved (GiB)": 40.31, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.01248, "step": 66, "tokens/total": 2277136, "tokens/train_per_sec_per_gpu": 21.86, "tokens/trainable": 30011} +{"epoch": 0.26171875, "grad_norm": 0.24522890150547028, "learning_rate": 9.843519986495259e-05, "loss": 0.005081563256680965, "memory/device_reserved (GiB)": 41.07, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.00509, "step": 67, "tokens/total": 2311872, "tokens/train_per_sec_per_gpu": 25.05, "tokens/trainable": 30446} +{"epoch": 0.265625, "grad_norm": 0.4216098189353943, "learning_rate": 9.835841041168162e-05, "loss": 0.01836864836513996, "memory/device_reserved (GiB)": 41.32, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.01854, "step": 68, "tokens/total": 2349008, "tokens/train_per_sec_per_gpu": 24.92, "tokens/trainable": 30913} +{"epoch": 0.26953125, "grad_norm": 0.4748227596282959, "learning_rate": 9.82798166379715e-05, "loss": 0.01501730177551508, "memory/device_reserved (GiB)": 41.32, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.01513, "step": 69, "tokens/total": 2382368, "tokens/train_per_sec_per_gpu": 28.93, "tokens/trainable": 31403} +{"epoch": 0.2734375, "grad_norm": 0.7099795937538147, "learning_rate": 9.819942181443002e-05, "loss": 0.027200814336538315, "memory/device_reserved (GiB)": 41.32, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.02757, "step": 70, "tokens/total": 2419648, "tokens/train_per_sec_per_gpu": 22.1, "tokens/trainable": 31844} +{"epoch": 0.27734375, "grad_norm": 0.23594564199447632, "learning_rate": 9.811722928661392e-05, "loss": 0.007420615293085575, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 35.81, "memory/max_allocated (GiB)": 35.81, "ppl": 1.00745, "step": 71, "tokens/total": 2448576, "tokens/train_per_sec_per_gpu": 26.89, "tokens/trainable": 32236} +{"epoch": 0.28125, "grad_norm": 0.33932849764823914, "learning_rate": 9.803324247488975e-05, "loss": 0.010974112898111343, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.01103, "step": 72, "tokens/total": 2484240, "tokens/train_per_sec_per_gpu": 24.33, "tokens/trainable": 32688} +{"epoch": 0.28515625, "grad_norm": 0.30434390902519226, "learning_rate": 9.794746487429161e-05, "loss": 0.010949136689305305, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 36.88, "memory/max_allocated (GiB)": 36.88, "ppl": 1.01101, "step": 73, "tokens/total": 2517184, "tokens/train_per_sec_per_gpu": 27.46, "tokens/trainable": 33131} +{"epoch": 0.2890625, "grad_norm": 0.3928545117378235, "learning_rate": 9.785990005437554e-05, "loss": 0.018433988094329834, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.0186, "step": 74, "tokens/total": 2553056, "tokens/train_per_sec_per_gpu": 25.69, "tokens/trainable": 33592} +{"epoch": 0.29296875, "grad_norm": 0.8546245098114014, "learning_rate": 9.777055165907117e-05, "loss": 0.0423702597618103, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.04328, "step": 75, "tokens/total": 2585584, "tokens/train_per_sec_per_gpu": 29.44, "tokens/trainable": 34023} +{"epoch": 0.296875, "grad_norm": 0.3911623954772949, "learning_rate": 9.767942340652993e-05, "loss": 0.008154450915753841, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.07, "memory/max_allocated (GiB)": 38.07, "ppl": 1.00819, "step": 76, "tokens/total": 2620448, "tokens/train_per_sec_per_gpu": 22.81, "tokens/trainable": 34449} +{"epoch": 0.30078125, "grad_norm": 0.30331966280937195, "learning_rate": 9.758651908897035e-05, "loss": 0.015458358451724052, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.01558, "step": 77, "tokens/total": 2653840, "tokens/train_per_sec_per_gpu": 28.56, "tokens/trainable": 34894} +{"epoch": 0.3046875, "grad_norm": 0.681708574295044, "learning_rate": 9.749184257252033e-05, "loss": 0.02927522361278534, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.02971, "step": 78, "tokens/total": 2688112, "tokens/train_per_sec_per_gpu": 25.02, "tokens/trainable": 35334} +{"epoch": 0.30859375, "grad_norm": 0.27489906549453735, "learning_rate": 9.739539779705614e-05, "loss": 0.009895533323287964, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.00994, "step": 79, "tokens/total": 2726384, "tokens/train_per_sec_per_gpu": 24.98, "tokens/trainable": 35829} +{"epoch": 0.3125, "grad_norm": 0.32178041338920593, "learning_rate": 9.729718877603861e-05, "loss": 0.01640351489186287, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.01654, "step": 80, "tokens/total": 2760176, "tokens/train_per_sec_per_gpu": 27.22, "tokens/trainable": 36260} +{"epoch": 0.31640625, "grad_norm": 0.45278143882751465, "learning_rate": 9.719721959634592e-05, "loss": 0.02231398969888687, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.02256, "step": 81, "tokens/total": 2795392, "tokens/train_per_sec_per_gpu": 23.72, "tokens/trainable": 36697} +{"epoch": 0.3203125, "grad_norm": 0.30973583459854126, "learning_rate": 9.709549441810375e-05, "loss": 0.007898072712123394, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.55, "memory/max_allocated (GiB)": 39.55, "ppl": 1.00793, "step": 82, "tokens/total": 2833008, "tokens/train_per_sec_per_gpu": 27.49, "tokens/trainable": 37181} +{"epoch": 0.32421875, "grad_norm": 0.9202283024787903, "learning_rate": 9.699201747451195e-05, "loss": 0.026167765259742737, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.02651, "step": 83, "tokens/total": 2865600, "tokens/train_per_sec_per_gpu": 25.48, "tokens/trainable": 37629} +{"epoch": 0.328125, "grad_norm": 0.22412244975566864, "learning_rate": 9.688679307166854e-05, "loss": 0.003919859416782856, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00393, "step": 84, "tokens/total": 2898400, "tokens/train_per_sec_per_gpu": 28.9, "tokens/trainable": 38092} +{"epoch": 0.33203125, "grad_norm": 0.2675820291042328, "learning_rate": 9.677982558839042e-05, "loss": 0.00635011401027441, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00637, "step": 85, "tokens/total": 2931696, "tokens/train_per_sec_per_gpu": 27.67, "tokens/trainable": 38571} +{"epoch": 0.3359375, "grad_norm": 0.27524271607398987, "learning_rate": 9.66711194760312e-05, "loss": 0.004763564560562372, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.00477, "step": 86, "tokens/total": 2969104, "tokens/train_per_sec_per_gpu": 26.06, "tokens/trainable": 39059} +{"epoch": 0.33984375, "grad_norm": 2.072495222091675, "learning_rate": 9.656067925829593e-05, "loss": 0.013271946460008621, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.01336, "step": 87, "tokens/total": 2999856, "tokens/train_per_sec_per_gpu": 31.89, "tokens/trainable": 39519} +{"epoch": 0.34375, "grad_norm": 0.4813166558742523, "learning_rate": 9.644850953105288e-05, "loss": 0.008394023403525352, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 35.78, "memory/max_allocated (GiB)": 35.78, "ppl": 1.00843, "step": 88, "tokens/total": 3030208, "tokens/train_per_sec_per_gpu": 26.71, "tokens/trainable": 39961} +{"epoch": 0.34765625, "grad_norm": 0.364719420671463, "learning_rate": 9.633461496214225e-05, "loss": 0.0073549761436879635, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.00738, "step": 89, "tokens/total": 3062912, "tokens/train_per_sec_per_gpu": 27.09, "tokens/trainable": 40431} +{"epoch": 0.3515625, "grad_norm": 0.5955256819725037, "learning_rate": 9.621900029118195e-05, "loss": 0.03168891370296478, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.04, "memory/max_allocated (GiB)": 36.04, "ppl": 1.0322, "step": 90, "tokens/total": 3092592, "tokens/train_per_sec_per_gpu": 26.13, "tokens/trainable": 40854} +{"epoch": 0.35546875, "grad_norm": 0.21226805448532104, "learning_rate": 9.610167032937036e-05, "loss": 0.0031325072050094604, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00314, "step": 91, "tokens/total": 3130880, "tokens/train_per_sec_per_gpu": 26.65, "tokens/trainable": 41373} +{"epoch": 0.359375, "grad_norm": 0.36134034395217896, "learning_rate": 9.598262995928611e-05, "loss": 0.0069163027219474316, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.00694, "step": 92, "tokens/total": 3162080, "tokens/train_per_sec_per_gpu": 30.82, "tokens/trainable": 41842} +{"epoch": 0.36328125, "grad_norm": 0.1328633725643158, "learning_rate": 9.586188413468492e-05, "loss": 0.002012464217841625, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00201, "step": 93, "tokens/total": 3200272, "tokens/train_per_sec_per_gpu": 23.4, "tokens/trainable": 42288} +{"epoch": 0.3671875, "grad_norm": 0.28403645753860474, "learning_rate": 9.57394378802934e-05, "loss": 0.007185705937445164, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.00721, "step": 94, "tokens/total": 3233584, "tokens/train_per_sec_per_gpu": 28.03, "tokens/trainable": 42740} +{"epoch": 0.37109375, "grad_norm": 0.5270668864250183, "learning_rate": 9.56152962916e-05, "loss": 0.016822682693600655, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.26, "memory/max_allocated (GiB)": 36.26, "ppl": 1.01696, "step": 95, "tokens/total": 3264144, "tokens/train_per_sec_per_gpu": 27.31, "tokens/trainable": 43138} +{"epoch": 0.375, "grad_norm": 0.15142114460468292, "learning_rate": 9.548946453464296e-05, "loss": 0.0019660787656903267, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.00197, "step": 96, "tokens/total": 3298240, "tokens/train_per_sec_per_gpu": 27.43, "tokens/trainable": 43623} +{"epoch": 0.37890625, "grad_norm": 0.29649975895881653, "learning_rate": 9.53619478457953e-05, "loss": 0.010610352270305157, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.01067, "step": 97, "tokens/total": 3333136, "tokens/train_per_sec_per_gpu": 24.52, "tokens/trainable": 44079} +{"epoch": 0.3828125, "grad_norm": 0.3287745416164398, "learning_rate": 9.523275153154695e-05, "loss": 0.008036238141357899, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.00807, "step": 98, "tokens/total": 3365536, "tokens/train_per_sec_per_gpu": 25.76, "tokens/trainable": 44529} +{"epoch": 0.38671875, "grad_norm": 0.5295984148979187, "learning_rate": 9.51018809682839e-05, "loss": 0.007988630793988705, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00802, "step": 99, "tokens/total": 3403200, "tokens/train_per_sec_per_gpu": 26.17, "tokens/trainable": 44976} +{"epoch": 0.390625, "grad_norm": 0.4978157877922058, "learning_rate": 9.49693416020645e-05, "loss": 0.006107218563556671, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.53, "memory/max_allocated (GiB)": 38.53, "ppl": 1.00613, "step": 100, "tokens/total": 3440944, "tokens/train_per_sec_per_gpu": 25.05, "tokens/trainable": 45429} +{"epoch": 0.39453125, "grad_norm": 0.9501412510871887, "learning_rate": 9.483513894839276e-05, "loss": 0.021616579964756966, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.02185, "step": 101, "tokens/total": 3474992, "tokens/train_per_sec_per_gpu": 25.81, "tokens/trainable": 45876} +{"epoch": 0.3984375, "grad_norm": 0.7793817520141602, "learning_rate": 9.469927859198888e-05, "loss": 0.006477953400462866, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.0065, "step": 102, "tokens/total": 3510912, "tokens/train_per_sec_per_gpu": 29.67, "tokens/trainable": 46363} +{"epoch": 0.40234375, "grad_norm": 0.7962983846664429, "learning_rate": 9.456176618655689e-05, "loss": 0.018869725987315178, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.47, "memory/max_allocated (GiB)": 36.47, "ppl": 1.01905, "step": 103, "tokens/total": 3542960, "tokens/train_per_sec_per_gpu": 28.07, "tokens/trainable": 46810} +{"epoch": 0.40625, "grad_norm": 0.3067605197429657, "learning_rate": 9.442260745454927e-05, "loss": 0.003341715084388852, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00335, "step": 104, "tokens/total": 3578336, "tokens/train_per_sec_per_gpu": 26.45, "tokens/trainable": 47287} +{"epoch": 0.41015625, "grad_norm": 0.7046481966972351, "learning_rate": 9.428180818692884e-05, "loss": 0.011133968830108643, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.45, "memory/max_allocated (GiB)": 38.45, "ppl": 1.0112, "step": 105, "tokens/total": 3614496, "tokens/train_per_sec_per_gpu": 25.23, "tokens/trainable": 47788} +{"epoch": 0.4140625, "grad_norm": 0.4187207520008087, "learning_rate": 9.413937424292791e-05, "loss": 0.015283001586794853, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.0154, "step": 106, "tokens/total": 3653632, "tokens/train_per_sec_per_gpu": 22.11, "tokens/trainable": 48243} +{"epoch": 0.41796875, "grad_norm": 0.38750725984573364, "learning_rate": 9.399531154980424e-05, "loss": 0.012629568576812744, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 1.01271, "step": 107, "tokens/total": 3690208, "tokens/train_per_sec_per_gpu": 24.49, "tokens/trainable": 48692} +{"epoch": 0.421875, "grad_norm": 0.36294034123420715, "learning_rate": 9.384962610259455e-05, "loss": 0.008314969018101692, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.00835, "step": 108, "tokens/total": 3724864, "tokens/train_per_sec_per_gpu": 24.64, "tokens/trainable": 49142} +{"epoch": 0.42578125, "grad_norm": 0.07947826385498047, "learning_rate": 9.370232396386494e-05, "loss": 0.0018936098786070943, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.22, "memory/max_allocated (GiB)": 37.22, "ppl": 1.0019, "step": 109, "tokens/total": 3757952, "tokens/train_per_sec_per_gpu": 26.93, "tokens/trainable": 49583} +{"epoch": 0.4296875, "grad_norm": 0.32407501339912415, "learning_rate": 9.355341126345868e-05, "loss": 0.005000374745577574, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.00501, "step": 110, "tokens/total": 3793232, "tokens/train_per_sec_per_gpu": 26.0, "tokens/trainable": 50055} +{"epoch": 0.43359375, "grad_norm": 0.48614752292633057, "learning_rate": 9.340289419824107e-05, "loss": 0.008972813375294209, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.00901, "step": 111, "tokens/total": 3825552, "tokens/train_per_sec_per_gpu": 28.67, "tokens/trainable": 50500} +{"epoch": 0.4375, "grad_norm": 0.31261521577835083, "learning_rate": 9.325077903184159e-05, "loss": 0.009424488991498947, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00947, "step": 112, "tokens/total": 3860544, "tokens/train_per_sec_per_gpu": 24.4, "tokens/trainable": 50950} +{"epoch": 0.44140625, "grad_norm": 0.10054045170545578, "learning_rate": 9.30970720943932e-05, "loss": 0.0013100756332278252, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.15, "memory/max_allocated (GiB)": 38.15, "ppl": 1.00131, "step": 113, "tokens/total": 3895552, "tokens/train_per_sec_per_gpu": 24.96, "tokens/trainable": 51389} +{"epoch": 0.4453125, "grad_norm": 0.20450176298618317, "learning_rate": 9.2941779782269e-05, "loss": 0.002448364859446883, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00245, "step": 114, "tokens/total": 3929152, "tokens/train_per_sec_per_gpu": 25.97, "tokens/trainable": 51862} +{"epoch": 0.44921875, "grad_norm": 0.28029125928878784, "learning_rate": 9.278490855781596e-05, "loss": 0.003506988286972046, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.24, "memory/max_allocated (GiB)": 37.24, "ppl": 1.00351, "step": 115, "tokens/total": 3961472, "tokens/train_per_sec_per_gpu": 30.51, "tokens/trainable": 52301} +{"epoch": 0.453125, "grad_norm": 0.26063022017478943, "learning_rate": 9.262646494908604e-05, "loss": 0.005109208170324564, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00512, "step": 116, "tokens/total": 4000128, "tokens/train_per_sec_per_gpu": 24.11, "tokens/trainable": 52755} +{"epoch": 0.45703125, "grad_norm": 0.579408586025238, "learning_rate": 9.246645554956457e-05, "loss": 0.013609274290502071, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.0137, "step": 117, "tokens/total": 4035152, "tokens/train_per_sec_per_gpu": 25.5, "tokens/trainable": 53224} +{"epoch": 0.4609375, "grad_norm": 0.8405458927154541, "learning_rate": 9.230488701789578e-05, "loss": 0.020249510183930397, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.02046, "step": 118, "tokens/total": 4072208, "tokens/train_per_sec_per_gpu": 23.93, "tokens/trainable": 53666} +{"epoch": 0.46484375, "grad_norm": 0.7553009986877441, "learning_rate": 9.214176607760577e-05, "loss": 0.007494427729398012, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.00752, "step": 119, "tokens/total": 4110464, "tokens/train_per_sec_per_gpu": 24.51, "tokens/trainable": 54129} +{"epoch": 0.46875, "grad_norm": 0.4436694383621216, "learning_rate": 9.197709951682268e-05, "loss": 0.0036186748184263706, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.00363, "step": 120, "tokens/total": 4142896, "tokens/train_per_sec_per_gpu": 29.56, "tokens/trainable": 54587} +{"epoch": 0.47265625, "grad_norm": 0.4831298589706421, "learning_rate": 9.181089418799428e-05, "loss": 0.007392803207039833, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.00742, "step": 121, "tokens/total": 4179360, "tokens/train_per_sec_per_gpu": 27.74, "tokens/trainable": 55070} +{"epoch": 0.4765625, "grad_norm": 0.14616759121418, "learning_rate": 9.164315700760271e-05, "loss": 0.0026568565517663956, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.00266, "step": 122, "tokens/total": 4215040, "tokens/train_per_sec_per_gpu": 23.68, "tokens/trainable": 55494} +{"epoch": 0.48046875, "grad_norm": 0.8537273406982422, "learning_rate": 9.147389495587671e-05, "loss": 0.009926480241119862, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.00998, "step": 123, "tokens/total": 4246448, "tokens/train_per_sec_per_gpu": 31.4, "tokens/trainable": 55940} +{"epoch": 0.484375, "grad_norm": 0.20489740371704102, "learning_rate": 9.130311507650116e-05, "loss": 0.002182799158617854, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.00219, "step": 124, "tokens/total": 4278832, "tokens/train_per_sec_per_gpu": 27.33, "tokens/trainable": 56404} +{"epoch": 0.48828125, "grad_norm": 1.1411832571029663, "learning_rate": 9.113082447632394e-05, "loss": 0.005215916782617569, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00523, "step": 125, "tokens/total": 4310896, "tokens/train_per_sec_per_gpu": 33.48, "tokens/trainable": 56878} +{"epoch": 0.4921875, "grad_norm": 0.5264424085617065, "learning_rate": 9.09570303250602e-05, "loss": 0.005715420935302973, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.00573, "step": 126, "tokens/total": 4342912, "tokens/train_per_sec_per_gpu": 30.9, "tokens/trainable": 57362} +{"epoch": 0.49609375, "grad_norm": 0.16951414942741394, "learning_rate": 9.078173985499394e-05, "loss": 0.0004687950131483376, "memory/device_reserved (GiB)": 42.05, "memory/max_active (GiB)": 35.95, "memory/max_allocated (GiB)": 35.95, "ppl": 1.00047, "step": 127, "tokens/total": 4373312, "tokens/train_per_sec_per_gpu": 28.7, "tokens/trainable": 57792} +{"epoch": 0.5, "grad_norm": 0.6024738550186157, "learning_rate": 9.060496036067713e-05, "loss": 0.009511682204902172, "memory/device_reserved (GiB)": 42.05, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.00956, "step": 128, "tokens/total": 4410816, "tokens/train_per_sec_per_gpu": 22.69, "tokens/trainable": 58216} +{"epoch": 0.50390625, "grad_norm": 0.518540620803833, "learning_rate": 9.042669919862615e-05, "loss": 0.006726134568452835, "memory/device_reserved (GiB)": 42.05, "memory/max_active (GiB)": 36.77, "memory/max_allocated (GiB)": 36.77, "ppl": 1.00675, "step": 129, "tokens/total": 4442704, "tokens/train_per_sec_per_gpu": 30.35, "tokens/trainable": 58680} +{"epoch": 0.5078125, "grad_norm": 0.4062198996543884, "learning_rate": 9.024696378701557e-05, "loss": 0.004424870014190674, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 39.48, "memory/max_allocated (GiB)": 39.48, "ppl": 1.00443, "step": 130, "tokens/total": 4478384, "tokens/train_per_sec_per_gpu": 24.09, "tokens/trainable": 59113} +{"epoch": 0.51171875, "grad_norm": 1.002680778503418, "learning_rate": 9.006576160536948e-05, "loss": 0.010163919068872929, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.01022, "step": 131, "tokens/total": 4513328, "tokens/train_per_sec_per_gpu": 24.55, "tokens/trainable": 59562} +{"epoch": 0.515625, "grad_norm": 0.6759523153305054, "learning_rate": 8.988310019425035e-05, "loss": 0.00129236641805619, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.00129, "step": 132, "tokens/total": 4550832, "tokens/train_per_sec_per_gpu": 24.36, "tokens/trainable": 60006} +{"epoch": 0.51953125, "grad_norm": 0.1316259652376175, "learning_rate": 8.969898715494506e-05, "loss": 0.0018948459764942527, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.0019, "step": 133, "tokens/total": 4587968, "tokens/train_per_sec_per_gpu": 25.94, "tokens/trainable": 60487} +{"epoch": 0.5234375, "grad_norm": 0.267514705657959, "learning_rate": 8.951343014914869e-05, "loss": 0.006785395555198193, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00681, "step": 134, "tokens/total": 4624544, "tokens/train_per_sec_per_gpu": 23.75, "tokens/trainable": 60953} +{"epoch": 0.52734375, "grad_norm": 0.07580235600471497, "learning_rate": 8.932643689864568e-05, "loss": 0.0004376223077997565, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00044, "step": 135, "tokens/total": 4661296, "tokens/train_per_sec_per_gpu": 27.52, "tokens/trainable": 61440} +{"epoch": 0.53125, "grad_norm": 0.20273242890834808, "learning_rate": 8.913801518498845e-05, "loss": 0.0021762175019830465, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.00218, "step": 136, "tokens/total": 4700128, "tokens/train_per_sec_per_gpu": 21.51, "tokens/trainable": 61901} +{"epoch": 0.53515625, "grad_norm": 0.2108118236064911, "learning_rate": 8.894817284917364e-05, "loss": 0.0029211000073701143, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00293, "step": 137, "tokens/total": 4736912, "tokens/train_per_sec_per_gpu": 22.99, "tokens/trainable": 62355} +{"epoch": 0.5390625, "grad_norm": 0.4103465676307678, "learning_rate": 8.875691779131569e-05, "loss": 0.008010243065655231, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00804, "step": 138, "tokens/total": 4774192, "tokens/train_per_sec_per_gpu": 20.95, "tokens/trainable": 62786} +{"epoch": 0.54296875, "grad_norm": 0.2136438637971878, "learning_rate": 8.856425797031829e-05, "loss": 0.0020005451515316963, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.002, "step": 139, "tokens/total": 4809344, "tokens/train_per_sec_per_gpu": 26.03, "tokens/trainable": 63228} +{"epoch": 0.546875, "grad_norm": 0.26300033926963806, "learning_rate": 8.837020140354295e-05, "loss": 0.0032554706558585167, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.00326, "step": 140, "tokens/total": 4845712, "tokens/train_per_sec_per_gpu": 27.09, "tokens/trainable": 63693} +{"epoch": 0.55078125, "grad_norm": 0.6262677907943726, "learning_rate": 8.817475616647554e-05, "loss": 0.006538073066622019, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.00656, "step": 141, "tokens/total": 4880368, "tokens/train_per_sec_per_gpu": 26.77, "tokens/trainable": 64159} +{"epoch": 0.5546875, "grad_norm": 0.9796885848045349, "learning_rate": 8.797793039239017e-05, "loss": 0.01687786355614662, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.01702, "step": 142, "tokens/total": 4915424, "tokens/train_per_sec_per_gpu": 26.27, "tokens/trainable": 64608} +{"epoch": 0.55859375, "grad_norm": 0.7891486287117004, "learning_rate": 8.777973227201069e-05, "loss": 0.01038912869989872, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.01044, "step": 143, "tokens/total": 4951744, "tokens/train_per_sec_per_gpu": 28.05, "tokens/trainable": 65117} +{"epoch": 0.5625, "grad_norm": 0.518424928188324, "learning_rate": 8.758017005316988e-05, "loss": 0.013542139902710915, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.01363, "step": 144, "tokens/total": 4985152, "tokens/train_per_sec_per_gpu": 28.52, "tokens/trainable": 65542} +{"epoch": 0.56640625, "grad_norm": 0.6565298438072205, "learning_rate": 8.737925204046629e-05, "loss": 0.021547390148043633, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.02178, "step": 145, "tokens/total": 5019936, "tokens/train_per_sec_per_gpu": 24.24, "tokens/trainable": 65992} +{"epoch": 0.5703125, "grad_norm": 0.04180261120200157, "learning_rate": 8.717698659491851e-05, "loss": 0.00036500050919130445, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00037, "step": 146, "tokens/total": 5057024, "tokens/train_per_sec_per_gpu": 22.59, "tokens/trainable": 66449} +{"epoch": 0.57421875, "grad_norm": 0.15914125740528107, "learning_rate": 8.697338213361735e-05, "loss": 0.0033237282186746597, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.00333, "step": 147, "tokens/total": 5088320, "tokens/train_per_sec_per_gpu": 26.31, "tokens/trainable": 66897} +{"epoch": 0.578125, "grad_norm": 0.44643792510032654, "learning_rate": 8.676844712937552e-05, "loss": 0.0036258443724364042, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.39, "memory/max_allocated (GiB)": 39.39, "ppl": 1.00363, "step": 148, "tokens/total": 5128064, "tokens/train_per_sec_per_gpu": 25.02, "tokens/trainable": 67369} +{"epoch": 0.58203125, "grad_norm": 0.38136133551597595, "learning_rate": 8.656219011037509e-05, "loss": 0.012315155938267708, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.01239, "step": 149, "tokens/total": 5165104, "tokens/train_per_sec_per_gpu": 23.71, "tokens/trainable": 67857} +{"epoch": 0.5859375, "grad_norm": 0.2005263715982437, "learning_rate": 8.63546196598125e-05, "loss": 0.0036788908764719963, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00369, "step": 150, "tokens/total": 5202336, "tokens/train_per_sec_per_gpu": 25.26, "tokens/trainable": 68305} +{"epoch": 0.58984375, "grad_norm": 0.1280508041381836, "learning_rate": 8.614574441554145e-05, "loss": 0.0034596198238432407, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.13, "memory/max_allocated (GiB)": 37.13, "ppl": 1.00347, "step": 151, "tokens/total": 5235440, "tokens/train_per_sec_per_gpu": 27.52, "tokens/trainable": 68730} +{"epoch": 0.59375, "grad_norm": 0.5983961820602417, "learning_rate": 8.593557306971349e-05, "loss": 0.010798279196023941, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.44, "memory/max_allocated (GiB)": 37.44, "ppl": 1.01086, "step": 152, "tokens/total": 5268416, "tokens/train_per_sec_per_gpu": 26.11, "tokens/trainable": 69185} +{"epoch": 0.59765625, "grad_norm": 0.10476668924093246, "learning_rate": 8.572411436841618e-05, "loss": 0.001891131279990077, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00189, "step": 153, "tokens/total": 5302128, "tokens/train_per_sec_per_gpu": 26.7, "tokens/trainable": 69660} +{"epoch": 0.6015625, "grad_norm": 0.17818617820739746, "learning_rate": 8.551137711130922e-05, "loss": 0.003587838262319565, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.2, "memory/max_allocated (GiB)": 38.2, "ppl": 1.00359, "step": 154, "tokens/total": 5333520, "tokens/train_per_sec_per_gpu": 24.01, "tokens/trainable": 70066} +{"epoch": 0.60546875, "grad_norm": 0.3723907172679901, "learning_rate": 8.529737015125824e-05, "loss": 0.006938630249351263, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00696, "step": 155, "tokens/total": 5367968, "tokens/train_per_sec_per_gpu": 25.94, "tokens/trainable": 70532} +{"epoch": 0.609375, "grad_norm": 0.6560153365135193, "learning_rate": 8.508210239396639e-05, "loss": 0.008738069795072079, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.00878, "step": 156, "tokens/total": 5405536, "tokens/train_per_sec_per_gpu": 24.34, "tokens/trainable": 70981} +{"epoch": 0.61328125, "grad_norm": 0.20067213475704193, "learning_rate": 8.486558279760375e-05, "loss": 0.0030872265342622995, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 36.07, "memory/max_allocated (GiB)": 36.07, "ppl": 1.00309, "step": 157, "tokens/total": 5436672, "tokens/train_per_sec_per_gpu": 26.8, "tokens/trainable": 71407} +{"epoch": 0.6171875, "grad_norm": 0.3861614763736725, "learning_rate": 8.464782037243449e-05, "loss": 0.013724523596465588, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.01382, "step": 158, "tokens/total": 5469280, "tokens/train_per_sec_per_gpu": 27.57, "tokens/trainable": 71848} +{"epoch": 0.62109375, "grad_norm": 0.4060450792312622, "learning_rate": 8.442882418044202e-05, "loss": 0.009527239948511124, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.00957, "step": 159, "tokens/total": 5501904, "tokens/train_per_sec_per_gpu": 29.55, "tokens/trainable": 72325} +{"epoch": 0.625, "grad_norm": 2.2070116996765137, "learning_rate": 8.420860333495179e-05, "loss": 0.012299400754272938, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.01238, "step": 160, "tokens/total": 5534944, "tokens/train_per_sec_per_gpu": 26.91, "tokens/trainable": 72739} +{"epoch": 0.62890625, "grad_norm": 0.8980720639228821, "learning_rate": 8.398716700025208e-05, "loss": 0.018056917935609818, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.01822, "step": 161, "tokens/total": 5568160, "tokens/train_per_sec_per_gpu": 25.85, "tokens/trainable": 73191} +{"epoch": 0.6328125, "grad_norm": 0.593686044216156, "learning_rate": 8.376452439121266e-05, "loss": 0.010432379320263863, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 35.89, "memory/max_allocated (GiB)": 35.89, "ppl": 1.01049, "step": 162, "tokens/total": 5598256, "tokens/train_per_sec_per_gpu": 31.39, "tokens/trainable": 73642} +{"epoch": 0.63671875, "grad_norm": 0.16508442163467407, "learning_rate": 8.354068477290124e-05, "loss": 0.0018663243390619755, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00187, "step": 163, "tokens/total": 5632560, "tokens/train_per_sec_per_gpu": 28.0, "tokens/trainable": 74101} +{"epoch": 0.640625, "grad_norm": 0.41325366497039795, "learning_rate": 8.331565746019807e-05, "loss": 0.006242088973522186, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 36.52, "memory/max_allocated (GiB)": 36.52, "ppl": 1.00626, "step": 164, "tokens/total": 5664464, "tokens/train_per_sec_per_gpu": 27.39, "tokens/trainable": 74558} +{"epoch": 0.64453125, "grad_norm": 0.6340977549552917, "learning_rate": 8.308945181740812e-05, "loss": 0.019003920257091522, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.01919, "step": 165, "tokens/total": 5699040, "tokens/train_per_sec_per_gpu": 27.48, "tokens/trainable": 75000} +{"epoch": 0.6484375, "grad_norm": 0.42316001653671265, "learning_rate": 8.286207725787153e-05, "loss": 0.01379142701625824, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.01389, "step": 166, "tokens/total": 5735968, "tokens/train_per_sec_per_gpu": 22.65, "tokens/trainable": 75445} +{"epoch": 0.65234375, "grad_norm": 0.5953949093818665, "learning_rate": 8.263354324357182e-05, "loss": 0.015545164234936237, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.01567, "step": 167, "tokens/total": 5770688, "tokens/train_per_sec_per_gpu": 25.64, "tokens/trainable": 75880} +{"epoch": 0.65625, "grad_norm": 0.10583531856536865, "learning_rate": 8.240385928474219e-05, "loss": 0.002478251699358225, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00248, "step": 168, "tokens/total": 5803824, "tokens/train_per_sec_per_gpu": 27.32, "tokens/trainable": 76294} +{"epoch": 0.66015625, "grad_norm": 0.5855512022972107, "learning_rate": 8.217303493946967e-05, "loss": 0.010578255169093609, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.01063, "step": 169, "tokens/total": 5838256, "tokens/train_per_sec_per_gpu": 26.54, "tokens/trainable": 76742} +{"epoch": 0.6640625, "grad_norm": 0.24835728108882904, "learning_rate": 8.194107981329746e-05, "loss": 0.006891096942126751, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00691, "step": 170, "tokens/total": 5876224, "tokens/train_per_sec_per_gpu": 23.58, "tokens/trainable": 77196} +{"epoch": 0.66796875, "grad_norm": 0.13757504522800446, "learning_rate": 8.170800355882518e-05, "loss": 0.0027802628464996815, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.24, "memory/max_allocated (GiB)": 37.24, "ppl": 1.00278, "step": 171, "tokens/total": 5910448, "tokens/train_per_sec_per_gpu": 30.74, "tokens/trainable": 77663} +{"epoch": 0.671875, "grad_norm": 0.4619065523147583, "learning_rate": 8.147381587530713e-05, "loss": 0.012101897969841957, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.01218, "step": 172, "tokens/total": 5944256, "tokens/train_per_sec_per_gpu": 26.84, "tokens/trainable": 78117} +{"epoch": 0.67578125, "grad_norm": 0.15280404686927795, "learning_rate": 8.123852650824877e-05, "loss": 0.0034403763711452484, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00345, "step": 173, "tokens/total": 5978288, "tokens/train_per_sec_per_gpu": 28.93, "tokens/trainable": 78601} +{"epoch": 0.6796875, "grad_norm": 0.3485623300075531, "learning_rate": 8.100214524900103e-05, "loss": 0.01397060789167881, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.52, "memory/max_allocated (GiB)": 38.52, "ppl": 1.01407, "step": 174, "tokens/total": 6013680, "tokens/train_per_sec_per_gpu": 22.5, "tokens/trainable": 79032} +{"epoch": 0.68359375, "grad_norm": 0.1348925679922104, "learning_rate": 8.076468193435301e-05, "loss": 0.002552933292463422, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.00256, "step": 175, "tokens/total": 6045072, "tokens/train_per_sec_per_gpu": 26.29, "tokens/trainable": 79456} +{"epoch": 0.6875, "grad_norm": 0.1633877158164978, "learning_rate": 8.052614644612253e-05, "loss": 0.0035145350266247988, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.00352, "step": 176, "tokens/total": 6082336, "tokens/train_per_sec_per_gpu": 23.61, "tokens/trainable": 79917} +{"epoch": 0.69140625, "grad_norm": 0.10111747682094574, "learning_rate": 8.028654871074489e-05, "loss": 0.0031030208338052034, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00311, "step": 177, "tokens/total": 6117024, "tokens/train_per_sec_per_gpu": 25.79, "tokens/trainable": 80358} +{"epoch": 0.6953125, "grad_norm": 0.14271387457847595, "learning_rate": 8.004589869885986e-05, "loss": 0.0029804250225424767, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 36.49, "memory/max_allocated (GiB)": 36.49, "ppl": 1.00298, "step": 178, "tokens/total": 6149136, "tokens/train_per_sec_per_gpu": 26.99, "tokens/trainable": 80777} +{"epoch": 0.69921875, "grad_norm": 0.11506094038486481, "learning_rate": 7.980420642489674e-05, "loss": 0.0011376115726307034, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.00114, "step": 179, "tokens/total": 6187776, "tokens/train_per_sec_per_gpu": 23.65, "tokens/trainable": 81250} +{"epoch": 0.703125, "grad_norm": 0.8508864045143127, "learning_rate": 7.95614819466576e-05, "loss": 0.014293590560555458, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.0144, "step": 180, "tokens/total": 6221728, "tokens/train_per_sec_per_gpu": 28.69, "tokens/trainable": 81715} +{"epoch": 0.70703125, "grad_norm": 0.18798454105854034, "learning_rate": 7.931773536489872e-05, "loss": 0.0015296986093744636, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.00153, "step": 181, "tokens/total": 6255920, "tokens/train_per_sec_per_gpu": 25.21, "tokens/trainable": 82160} +{"epoch": 0.7109375, "grad_norm": 0.6932432651519775, "learning_rate": 7.907297682291035e-05, "loss": 0.03318361937999725, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.03374, "step": 182, "tokens/total": 6291104, "tokens/train_per_sec_per_gpu": 26.85, "tokens/trainable": 82624} +{"epoch": 0.71484375, "grad_norm": 0.14090695977210999, "learning_rate": 7.882721650609442e-05, "loss": 0.002287699608132243, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.00229, "step": 183, "tokens/total": 6330976, "tokens/train_per_sec_per_gpu": 23.49, "tokens/trainable": 83125} +{"epoch": 0.71875, "grad_norm": 0.06437099725008011, "learning_rate": 7.85804646415409e-05, "loss": 0.0007588292355649173, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.00076, "step": 184, "tokens/total": 6366160, "tokens/train_per_sec_per_gpu": 27.1, "tokens/trainable": 83581} +{"epoch": 0.72265625, "grad_norm": 0.11756770312786102, "learning_rate": 7.833273149760207e-05, "loss": 0.0012629565317183733, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.00126, "step": 185, "tokens/total": 6397280, "tokens/train_per_sec_per_gpu": 29.15, "tokens/trainable": 84051} +{"epoch": 0.7265625, "grad_norm": 0.6770535111427307, "learning_rate": 7.808402738346527e-05, "loss": 0.026944531127810478, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.02731, "step": 186, "tokens/total": 6435632, "tokens/train_per_sec_per_gpu": 23.71, "tokens/trainable": 84500} +{"epoch": 0.73046875, "grad_norm": 0.4852089285850525, "learning_rate": 7.783436264872382e-05, "loss": 0.006517849862575531, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.00654, "step": 187, "tokens/total": 6472752, "tokens/train_per_sec_per_gpu": 25.12, "tokens/trainable": 84986} +{"epoch": 0.734375, "grad_norm": 0.37900570034980774, "learning_rate": 7.758374768294647e-05, "loss": 0.01005198247730732, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.05, "memory/max_allocated (GiB)": 38.05, "ppl": 1.0101, "step": 188, "tokens/total": 6508528, "tokens/train_per_sec_per_gpu": 27.25, "tokens/trainable": 85465} +{"epoch": 0.73828125, "grad_norm": 0.21855297684669495, "learning_rate": 7.733219291524489e-05, "loss": 0.005666119046509266, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.59, "memory/max_allocated (GiB)": 37.59, "ppl": 1.00568, "step": 189, "tokens/total": 6542048, "tokens/train_per_sec_per_gpu": 27.85, "tokens/trainable": 85896} +{"epoch": 0.7421875, "grad_norm": 0.2353118509054184, "learning_rate": 7.707970881383977e-05, "loss": 0.007156815845519304, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 36.57, "memory/max_allocated (GiB)": 36.57, "ppl": 1.00718, "step": 190, "tokens/total": 6573792, "tokens/train_per_sec_per_gpu": 28.49, "tokens/trainable": 86346} +{"epoch": 0.74609375, "grad_norm": 0.32398703694343567, "learning_rate": 7.682630588562518e-05, "loss": 0.007605342194437981, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.00763, "step": 191, "tokens/total": 6608848, "tokens/train_per_sec_per_gpu": 24.62, "tokens/trainable": 86777} +{"epoch": 0.75, "grad_norm": 0.3849335014820099, "learning_rate": 7.657199467573129e-05, "loss": 0.013352475129067898, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.01344, "step": 192, "tokens/total": 6642560, "tokens/train_per_sec_per_gpu": 26.47, "tokens/trainable": 87249} +{"epoch": 0.75390625, "grad_norm": 0.3342069685459137, "learning_rate": 7.631678576708561e-05, "loss": 0.00825480930507183, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00829, "step": 193, "tokens/total": 6676272, "tokens/train_per_sec_per_gpu": 29.35, "tokens/trainable": 87741} +{"epoch": 0.7578125, "grad_norm": 0.09432431310415268, "learning_rate": 7.606068977997255e-05, "loss": 0.0025497986935079098, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00255, "step": 194, "tokens/total": 6713568, "tokens/train_per_sec_per_gpu": 26.02, "tokens/trainable": 88219} +{"epoch": 0.76171875, "grad_norm": 0.2875254154205322, "learning_rate": 7.580371737159148e-05, "loss": 0.004266665317118168, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00428, "step": 195, "tokens/total": 6749296, "tokens/train_per_sec_per_gpu": 23.23, "tokens/trainable": 88630} +{"epoch": 0.765625, "grad_norm": 0.15936511754989624, "learning_rate": 7.554587923561324e-05, "loss": 0.003336378140375018, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00334, "step": 196, "tokens/total": 6783952, "tokens/train_per_sec_per_gpu": 24.95, "tokens/trainable": 89068} +{"epoch": 0.76953125, "grad_norm": 0.5148723721504211, "learning_rate": 7.528718610173511e-05, "loss": 0.012936457991600037, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.48, "memory/max_allocated (GiB)": 37.48, "ppl": 1.01302, "step": 197, "tokens/total": 6819152, "tokens/train_per_sec_per_gpu": 23.11, "tokens/trainable": 89489} +{"epoch": 0.7734375, "grad_norm": 0.23191948235034943, "learning_rate": 7.502764873523431e-05, "loss": 0.004828830249607563, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.00484, "step": 198, "tokens/total": 6853440, "tokens/train_per_sec_per_gpu": 27.67, "tokens/trainable": 89928} +{"epoch": 0.77734375, "grad_norm": 0.20834718644618988, "learning_rate": 7.476727793652011e-05, "loss": 0.0032415043096989393, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 36.71, "memory/max_allocated (GiB)": 36.71, "ppl": 1.00325, "step": 199, "tokens/total": 6886272, "tokens/train_per_sec_per_gpu": 28.46, "tokens/trainable": 90407} +{"epoch": 0.78125, "grad_norm": 0.3640304207801819, "learning_rate": 7.450608454068415e-05, "loss": 0.01587412878870964, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.016, "step": 200, "tokens/total": 6920624, "tokens/train_per_sec_per_gpu": 27.0, "tokens/trainable": 90849} +{"epoch": 0.78515625, "grad_norm": 0.12447959184646606, "learning_rate": 7.424407941704987e-05, "loss": 0.0018986478680744767, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.0019, "step": 201, "tokens/total": 6957328, "tokens/train_per_sec_per_gpu": 27.66, "tokens/trainable": 91350} +{"epoch": 0.7890625, "grad_norm": 6.657614231109619, "learning_rate": 7.398127346871986e-05, "loss": 0.044538967311382294, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.04555, "step": 202, "tokens/total": 6994048, "tokens/train_per_sec_per_gpu": 24.7, "tokens/trainable": 91805} +{"epoch": 0.79296875, "grad_norm": 0.3032752573490143, "learning_rate": 7.371767763212238e-05, "loss": 0.0025477155577391386, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.00255, "step": 203, "tokens/total": 7029040, "tokens/train_per_sec_per_gpu": 26.88, "tokens/trainable": 92258} +{"epoch": 0.796875, "grad_norm": 0.4372182786464691, "learning_rate": 7.345330287655617e-05, "loss": 0.005234710872173309, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.00525, "step": 204, "tokens/total": 7067104, "tokens/train_per_sec_per_gpu": 23.0, "tokens/trainable": 92721} +{"epoch": 0.80078125, "grad_norm": 0.19560953974723816, "learning_rate": 7.31881602037339e-05, "loss": 0.004138309508562088, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.00415, "step": 205, "tokens/total": 7099136, "tokens/train_per_sec_per_gpu": 26.71, "tokens/trainable": 93160} +{"epoch": 0.8046875, "grad_norm": 0.08594099432229996, "learning_rate": 7.29222606473245e-05, "loss": 0.000593043863773346, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.00059, "step": 206, "tokens/total": 7132208, "tokens/train_per_sec_per_gpu": 26.84, "tokens/trainable": 93605} +{"epoch": 0.80859375, "grad_norm": 0.3213021755218506, "learning_rate": 7.265561527249383e-05, "loss": 0.005242567975074053, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.00526, "step": 207, "tokens/total": 7166256, "tokens/train_per_sec_per_gpu": 28.08, "tokens/trainable": 94041} +{"epoch": 0.8125, "grad_norm": 0.10535436868667603, "learning_rate": 7.238823517544436e-05, "loss": 0.0011550533818081021, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.00116, "step": 208, "tokens/total": 7203152, "tokens/train_per_sec_per_gpu": 28.69, "tokens/trainable": 94512} +{"epoch": 0.81640625, "grad_norm": 0.08973374962806702, "learning_rate": 7.212013148295333e-05, "loss": 0.001069213030859828, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.00107, "step": 209, "tokens/total": 7237264, "tokens/train_per_sec_per_gpu": 30.16, "tokens/trainable": 94994} +{"epoch": 0.8203125, "grad_norm": 0.7094142436981201, "learning_rate": 7.185131535190975e-05, "loss": 0.006918442435562611, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00694, "step": 210, "tokens/total": 7269664, "tokens/train_per_sec_per_gpu": 30.38, "tokens/trainable": 95503} +{"epoch": 0.82421875, "grad_norm": 0.1095225065946579, "learning_rate": 7.158179796885005e-05, "loss": 0.0006099050515331328, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.00061, "step": 211, "tokens/total": 7304080, "tokens/train_per_sec_per_gpu": 27.86, "tokens/trainable": 95954} +{"epoch": 0.828125, "grad_norm": 0.02752392180263996, "learning_rate": 7.131159054949273e-05, "loss": 0.00014735560398548841, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00015, "step": 212, "tokens/total": 7342480, "tokens/train_per_sec_per_gpu": 22.6, "tokens/trainable": 96428} +{"epoch": 0.83203125, "grad_norm": 0.0016910197446122766, "learning_rate": 7.104070433827139e-05, "loss": 1.8357110093347728e-05, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00002, "step": 213, "tokens/total": 7378032, "tokens/train_per_sec_per_gpu": 23.35, "tokens/trainable": 96828} +{"epoch": 0.8359375, "grad_norm": 1.158964991569519, "learning_rate": 7.076915060786705e-05, "loss": 0.012625613249838352, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.96, "memory/max_allocated (GiB)": 37.96, "ppl": 1.01271, "step": 214, "tokens/total": 7410928, "tokens/train_per_sec_per_gpu": 24.57, "tokens/trainable": 97270} +{"epoch": 0.83984375, "grad_norm": 0.30665984749794006, "learning_rate": 7.049694065873882e-05, "loss": 0.0032797642052173615, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.00329, "step": 215, "tokens/total": 7445664, "tokens/train_per_sec_per_gpu": 28.32, "tokens/trainable": 97727} +{"epoch": 0.84375, "grad_norm": 0.42771652340888977, "learning_rate": 7.022408581865382e-05, "loss": 0.004678788594901562, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.69, "memory/max_allocated (GiB)": 38.69, "ppl": 1.00469, "step": 216, "tokens/total": 7479984, "tokens/train_per_sec_per_gpu": 25.73, "tokens/trainable": 98201} +{"epoch": 0.84765625, "grad_norm": 0.3944380581378937, "learning_rate": 6.99505974422157e-05, "loss": 0.00318505777977407, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.00319, "step": 217, "tokens/total": 7514144, "tokens/train_per_sec_per_gpu": 28.77, "tokens/trainable": 98674} +{"epoch": 0.8515625, "grad_norm": 1.3011515140533447, "learning_rate": 6.967648691039213e-05, "loss": 0.0011854044860228896, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 36.73, "memory/max_allocated (GiB)": 36.73, "ppl": 1.00119, "step": 218, "tokens/total": 7546256, "tokens/train_per_sec_per_gpu": 28.27, "tokens/trainable": 99108} +{"epoch": 0.85546875, "grad_norm": 0.22933778166770935, "learning_rate": 6.940176563004123e-05, "loss": 0.0020548170432448387, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00206, "step": 219, "tokens/total": 7580432, "tokens/train_per_sec_per_gpu": 26.29, "tokens/trainable": 99540} +{"epoch": 0.859375, "grad_norm": 0.7532063722610474, "learning_rate": 6.912644503343682e-05, "loss": 0.012579471804201603, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.01266, "step": 220, "tokens/total": 7617904, "tokens/train_per_sec_per_gpu": 23.73, "tokens/trainable": 99978} +{"epoch": 0.86328125, "grad_norm": 0.03720954805612564, "learning_rate": 6.885053657779273e-05, "loss": 0.0001708155614323914, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.00017, "step": 221, "tokens/total": 7649696, "tokens/train_per_sec_per_gpu": 28.79, "tokens/trainable": 100450} +{"epoch": 0.8671875, "grad_norm": 1.3164312839508057, "learning_rate": 6.857405174478604e-05, "loss": 0.004803129471838474, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.00481, "step": 222, "tokens/total": 7684464, "tokens/train_per_sec_per_gpu": 24.64, "tokens/trainable": 100903} +{"epoch": 0.87109375, "grad_norm": 0.3429636061191559, "learning_rate": 6.82970020400792e-05, "loss": 0.0026662363670766354, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.00267, "step": 223, "tokens/total": 7721936, "tokens/train_per_sec_per_gpu": 22.59, "tokens/trainable": 101351} +{"epoch": 0.875, "grad_norm": 0.11791206896305084, "learning_rate": 6.801939899284132e-05, "loss": 0.0007148716831579804, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.34, "memory/max_allocated (GiB)": 38.34, "ppl": 1.00072, "step": 224, "tokens/total": 7758160, "tokens/train_per_sec_per_gpu": 26.82, "tokens/trainable": 101857} +{"epoch": 0.87890625, "grad_norm": 1.3130935430526733, "learning_rate": 6.774125415526827e-05, "loss": 0.0075210025534033775, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00755, "step": 225, "tokens/total": 7791216, "tokens/train_per_sec_per_gpu": 28.14, "tokens/trainable": 102328} +{"epoch": 0.8828125, "grad_norm": 0.10365905612707138, "learning_rate": 6.746257910210214e-05, "loss": 0.0007819957681931555, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.12, "memory/max_allocated (GiB)": 37.12, "ppl": 1.00078, "step": 226, "tokens/total": 7824400, "tokens/train_per_sec_per_gpu": 28.48, "tokens/trainable": 102779} +{"epoch": 0.88671875, "grad_norm": 0.36521634459495544, "learning_rate": 6.718338543014937e-05, "loss": 0.008589259348809719, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 35.75, "memory/max_allocated (GiB)": 35.75, "ppl": 1.00863, "step": 227, "tokens/total": 7854672, "tokens/train_per_sec_per_gpu": 31.22, "tokens/trainable": 103228} +{"epoch": 0.890625, "grad_norm": 0.4698134660720825, "learning_rate": 6.69036847577983e-05, "loss": 0.006147912237793207, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.00617, "step": 228, "tokens/total": 7888592, "tokens/train_per_sec_per_gpu": 27.06, "tokens/trainable": 103684} +{"epoch": 0.89453125, "grad_norm": 0.7863852381706238, "learning_rate": 6.662348872453553e-05, "loss": 0.013464240357279778, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.33, "memory/max_allocated (GiB)": 38.33, "ppl": 1.01356, "step": 229, "tokens/total": 7924848, "tokens/train_per_sec_per_gpu": 27.82, "tokens/trainable": 104145} +{"epoch": 0.8984375, "grad_norm": 0.1764724850654602, "learning_rate": 6.63428089904618e-05, "loss": 0.002196779241785407, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.0022, "step": 230, "tokens/total": 7963856, "tokens/train_per_sec_per_gpu": 24.5, "tokens/trainable": 104598} +{"epoch": 0.90234375, "grad_norm": 0.19294241070747375, "learning_rate": 6.60616572358065e-05, "loss": 0.0007467351388186216, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.00075, "step": 231, "tokens/total": 7998320, "tokens/train_per_sec_per_gpu": 25.92, "tokens/trainable": 104998} +{"epoch": 0.90625, "grad_norm": 0.4301759600639343, "learning_rate": 6.578004516044172e-05, "loss": 0.004281180445104837, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00429, "step": 232, "tokens/total": 8034096, "tokens/train_per_sec_per_gpu": 27.85, "tokens/trainable": 105465} +{"epoch": 0.91015625, "grad_norm": 0.4527599811553955, "learning_rate": 6.549798448339548e-05, "loss": 0.009344618767499924, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.59, "memory/max_allocated (GiB)": 38.59, "ppl": 1.00939, "step": 233, "tokens/total": 8071360, "tokens/train_per_sec_per_gpu": 25.3, "tokens/trainable": 105928} +{"epoch": 0.9140625, "grad_norm": 0.1751369684934616, "learning_rate": 6.521548694236384e-05, "loss": 0.0016385458875447512, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00164, "step": 234, "tokens/total": 8108560, "tokens/train_per_sec_per_gpu": 23.61, "tokens/trainable": 106365} +{"epoch": 0.91796875, "grad_norm": 0.4913561940193176, "learning_rate": 6.493256429322259e-05, "loss": 0.00974934920668602, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.0098, "step": 235, "tokens/total": 8144816, "tokens/train_per_sec_per_gpu": 22.45, "tokens/trainable": 106796} +{"epoch": 0.921875, "grad_norm": 0.6244290471076965, "learning_rate": 6.464922830953799e-05, "loss": 0.013553211465477943, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.01365, "step": 236, "tokens/total": 8183680, "tokens/train_per_sec_per_gpu": 22.53, "tokens/trainable": 107227} +{"epoch": 0.92578125, "grad_norm": 0.17626146972179413, "learning_rate": 6.436549078207688e-05, "loss": 0.0037648831494152546, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00377, "step": 237, "tokens/total": 8216688, "tokens/train_per_sec_per_gpu": 29.08, "tokens/trainable": 107687} +{"epoch": 0.9296875, "grad_norm": 0.5790393352508545, "learning_rate": 6.408136351831592e-05, "loss": 0.014552822336554527, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.01466, "step": 238, "tokens/total": 8249104, "tokens/train_per_sec_per_gpu": 24.39, "tokens/trainable": 108127} +{"epoch": 0.93359375, "grad_norm": 0.19907048344612122, "learning_rate": 6.379685834195036e-05, "loss": 0.004078004043549299, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00409, "step": 239, "tokens/total": 8288528, "tokens/train_per_sec_per_gpu": 24.78, "tokens/trainable": 108633} +{"epoch": 0.9375, "grad_norm": 0.3179163932800293, "learning_rate": 6.351198709240186e-05, "loss": 0.012510336935520172, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.01259, "step": 240, "tokens/total": 8328464, "tokens/train_per_sec_per_gpu": 21.23, "tokens/trainable": 109085} +{"epoch": 0.94140625, "grad_norm": 0.3383925259113312, "learning_rate": 6.32267616243259e-05, "loss": 0.007091269362717867, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.00712, "step": 241, "tokens/total": 8361264, "tokens/train_per_sec_per_gpu": 29.0, "tokens/trainable": 109548} +{"epoch": 0.9453125, "grad_norm": 0.31688278913497925, "learning_rate": 6.294119380711849e-05, "loss": 0.005784780252724886, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.39, "memory/max_allocated (GiB)": 38.39, "ppl": 1.0058, "step": 242, "tokens/total": 8395552, "tokens/train_per_sec_per_gpu": 24.04, "tokens/trainable": 109975} +{"epoch": 0.94921875, "grad_norm": 0.5140283703804016, "learning_rate": 6.265529552442209e-05, "loss": 0.014354261569678783, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.01446, "step": 243, "tokens/total": 8430656, "tokens/train_per_sec_per_gpu": 28.3, "tokens/trainable": 110436} +{"epoch": 0.953125, "grad_norm": 0.16849522292613983, "learning_rate": 6.236907867363127e-05, "loss": 0.002647391054779291, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.17, "memory/max_allocated (GiB)": 38.17, "ppl": 1.00265, "step": 244, "tokens/total": 8465232, "tokens/train_per_sec_per_gpu": 26.88, "tokens/trainable": 110882} +{"epoch": 0.95703125, "grad_norm": 0.17453472316265106, "learning_rate": 6.208255516539749e-05, "loss": 0.0035660986322909594, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.00357, "step": 245, "tokens/total": 8497552, "tokens/train_per_sec_per_gpu": 26.85, "tokens/trainable": 111355} +{"epoch": 0.9609375, "grad_norm": 0.08591607958078384, "learning_rate": 6.179573692313344e-05, "loss": 0.0011879602679982781, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.8, "memory/max_allocated (GiB)": 38.8, "ppl": 1.00119, "step": 246, "tokens/total": 8534336, "tokens/train_per_sec_per_gpu": 26.28, "tokens/trainable": 111803} +{"epoch": 0.96484375, "grad_norm": 0.1237751692533493, "learning_rate": 6.150863588251694e-05, "loss": 0.003892086213454604, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.0039, "step": 247, "tokens/total": 8574224, "tokens/train_per_sec_per_gpu": 22.86, "tokens/trainable": 112270} +{"epoch": 0.96875, "grad_norm": 0.20863617956638336, "learning_rate": 6.122126399099419e-05, "loss": 0.0021054057870060205, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.00211, "step": 248, "tokens/total": 8613616, "tokens/train_per_sec_per_gpu": 27.11, "tokens/trainable": 112778} +{"epoch": 0.97265625, "grad_norm": 0.6515951752662659, "learning_rate": 6.0933633207282615e-05, "loss": 0.0065644048154354095, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.00659, "step": 249, "tokens/total": 8650048, "tokens/train_per_sec_per_gpu": 27.62, "tokens/trainable": 113284} +{"epoch": 0.9765625, "grad_norm": 0.6908811330795288, "learning_rate": 6.064575550087316e-05, "loss": 0.007657800801098347, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00769, "step": 250, "tokens/total": 8686688, "tokens/train_per_sec_per_gpu": 23.94, "tokens/trainable": 113722} +{"epoch": 0.98046875, "grad_norm": 0.2704547345638275, "learning_rate": 6.0357642851532245e-05, "loss": 0.001204958651214838, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00121, "step": 251, "tokens/total": 8723376, "tokens/train_per_sec_per_gpu": 25.99, "tokens/trainable": 114213} +{"epoch": 0.984375, "grad_norm": 0.4925293028354645, "learning_rate": 6.0069307248803294e-05, "loss": 0.005083112046122551, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.0051, "step": 252, "tokens/total": 8758736, "tokens/train_per_sec_per_gpu": 22.12, "tokens/trainable": 114634} +{"epoch": 0.98828125, "grad_norm": 4.747704029083252, "learning_rate": 5.9780760691507635e-05, "loss": 0.0160056184977293, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.33, "memory/max_allocated (GiB)": 36.33, "ppl": 1.01613, "step": 253, "tokens/total": 8790480, "tokens/train_per_sec_per_gpu": 26.97, "tokens/trainable": 115066} +{"epoch": 0.9921875, "grad_norm": 0.48096054792404175, "learning_rate": 5.9492015187245334e-05, "loss": 0.00302697136066854, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00303, "step": 254, "tokens/total": 8828224, "tokens/train_per_sec_per_gpu": 23.47, "tokens/trainable": 115520} +{"epoch": 0.99609375, "grad_norm": 0.36961492896080017, "learning_rate": 5.920308275189541e-05, "loss": 0.012843207456171513, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.01293, "step": 255, "tokens/total": 8860400, "tokens/train_per_sec_per_gpu": 25.75, "tokens/trainable": 115921} +{"epoch": 1.0, "grad_norm": 0.015332306735217571, "learning_rate": 5.8913975409115874e-05, "loss": 0.00016752064402680844, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.00017, "step": 256, "tokens/total": 8893760, "tokens/train_per_sec_per_gpu": 25.63, "tokens/trainable": 116384} +{"epoch": 1.00390625, "grad_norm": 0.06840667873620987, "learning_rate": 5.8624705189843395e-05, "loss": 0.000637403572909534, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 1.00064, "step": 257, "tokens/total": 8930976, "tokens/train_per_sec_per_gpu": 24.89, "tokens/trainable": 116871} +{"epoch": 1.0078125, "grad_norm": 0.16524744033813477, "learning_rate": 5.833528413179249e-05, "loss": 0.0025473246350884438, "memory/device_reserved (GiB)": 42.25, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00255, "step": 258, "tokens/total": 8967472, "tokens/train_per_sec_per_gpu": 28.04, "tokens/trainable": 117369} +{"epoch": 1.01171875, "grad_norm": 2.3073134422302246, "learning_rate": 5.80457242789548e-05, "loss": 0.013742688111960888, "memory/device_reserved (GiB)": 42.31, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.01384, "step": 259, "tokens/total": 9007456, "tokens/train_per_sec_per_gpu": 24.03, "tokens/trainable": 117866} +{"epoch": 1.015625, "grad_norm": 0.013775884173810482, "learning_rate": 5.77560376810977e-05, "loss": 0.00020487068104557693, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 1.0002, "step": 260, "tokens/total": 9041104, "tokens/train_per_sec_per_gpu": 29.4, "tokens/trainable": 118330} +{"epoch": 1.01953125, "grad_norm": 0.01107958797365427, "learning_rate": 5.7466236393263005e-05, "loss": 0.0001238947152160108, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.00012, "step": 261, "tokens/total": 9076176, "tokens/train_per_sec_per_gpu": 25.68, "tokens/trainable": 118776} +{"epoch": 1.0234375, "grad_norm": 0.17523588240146637, "learning_rate": 5.717633247526522e-05, "loss": 0.0014833146706223488, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00148, "step": 262, "tokens/total": 9107472, "tokens/train_per_sec_per_gpu": 27.89, "tokens/trainable": 119211} +{"epoch": 1.02734375, "grad_norm": 0.06193378195166588, "learning_rate": 5.688633799118971e-05, "loss": 0.0004853529972024262, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.00049, "step": 263, "tokens/total": 9139552, "tokens/train_per_sec_per_gpu": 23.58, "tokens/trainable": 119616} +{"epoch": 1.03125, "grad_norm": 0.004347667563706636, "learning_rate": 5.659626500889066e-05, "loss": 6.740552635164931e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00007, "step": 264, "tokens/total": 9173424, "tokens/train_per_sec_per_gpu": 26.0, "tokens/trainable": 120086} +{"epoch": 1.03515625, "grad_norm": 0.03992529585957527, "learning_rate": 5.6306125599488905e-05, "loss": 0.00042756140464916825, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.00043, "step": 265, "tokens/total": 9203984, "tokens/train_per_sec_per_gpu": 32.39, "tokens/trainable": 120525} +{"epoch": 1.0390625, "grad_norm": 0.013901126570999622, "learning_rate": 5.601593183686955e-05, "loss": 0.0002272905985591933, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.69, "memory/max_allocated (GiB)": 38.69, "ppl": 1.00023, "step": 266, "tokens/total": 9241248, "tokens/train_per_sec_per_gpu": 26.17, "tokens/trainable": 121001} +{"epoch": 1.04296875, "grad_norm": 0.0785088911652565, "learning_rate": 5.572569579717961e-05, "loss": 0.0011717925081029534, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.18, "memory/max_allocated (GiB)": 39.18, "ppl": 1.00117, "step": 267, "tokens/total": 9276768, "tokens/train_per_sec_per_gpu": 24.65, "tokens/trainable": 121465} +{"epoch": 1.046875, "grad_norm": 0.029172956943511963, "learning_rate": 5.543542955832538e-05, "loss": 0.00019031830015592277, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00019, "step": 268, "tokens/total": 9316528, "tokens/train_per_sec_per_gpu": 21.84, "tokens/trainable": 121908} +{"epoch": 1.05078125, "grad_norm": 0.16227370500564575, "learning_rate": 5.514514519946986e-05, "loss": 0.002208184450864792, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.00221, "step": 269, "tokens/total": 9356352, "tokens/train_per_sec_per_gpu": 25.91, "tokens/trainable": 122406} +{"epoch": 1.0546875, "grad_norm": 0.4951207935810089, "learning_rate": 5.485485480053015e-05, "loss": 0.009089547209441662, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.00913, "step": 270, "tokens/total": 9394576, "tokens/train_per_sec_per_gpu": 22.06, "tokens/trainable": 122812} +{"epoch": 1.05859375, "grad_norm": 0.03776303678750992, "learning_rate": 5.4564570441674645e-05, "loss": 8.959416300058365e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.92, "memory/max_allocated (GiB)": 36.92, "ppl": 1.00009, "step": 271, "tokens/total": 9427840, "tokens/train_per_sec_per_gpu": 25.72, "tokens/trainable": 123252} +{"epoch": 1.0625, "grad_norm": 0.021155795082449913, "learning_rate": 5.42743042028204e-05, "loss": 0.00020102526468690485, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.0002, "step": 272, "tokens/total": 9463536, "tokens/train_per_sec_per_gpu": 20.79, "tokens/trainable": 123692} +{"epoch": 1.06640625, "grad_norm": 0.21283547580242157, "learning_rate": 5.3984068163130464e-05, "loss": 0.002611862262710929, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.09, "memory/max_allocated (GiB)": 37.09, "ppl": 1.00262, "step": 273, "tokens/total": 9497248, "tokens/train_per_sec_per_gpu": 28.39, "tokens/trainable": 124162} +{"epoch": 1.0703125, "grad_norm": 0.7826184034347534, "learning_rate": 5.369387440051111e-05, "loss": 0.009048626758158207, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.00909, "step": 274, "tokens/total": 9530784, "tokens/train_per_sec_per_gpu": 25.67, "tokens/trainable": 124625} +{"epoch": 1.07421875, "grad_norm": 0.010771344415843487, "learning_rate": 5.340373499110935e-05, "loss": 8.459887612843886e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00008, "step": 275, "tokens/total": 9566656, "tokens/train_per_sec_per_gpu": 26.76, "tokens/trainable": 125119} +{"epoch": 1.078125, "grad_norm": 0.00257283728569746, "learning_rate": 5.3113662008810304e-05, "loss": 2.9654420359293e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00003, "step": 276, "tokens/total": 9606672, "tokens/train_per_sec_per_gpu": 25.2, "tokens/trainable": 125601} +{"epoch": 1.08203125, "grad_norm": 0.1277220994234085, "learning_rate": 5.282366752473479e-05, "loss": 0.002297363942489028, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 1.0023, "step": 277, "tokens/total": 9643680, "tokens/train_per_sec_per_gpu": 24.74, "tokens/trainable": 126075} +{"epoch": 1.0859375, "grad_norm": 0.26641222834587097, "learning_rate": 5.2533763606737005e-05, "loss": 0.003887616563588381, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.0039, "step": 278, "tokens/total": 9683200, "tokens/train_per_sec_per_gpu": 19.53, "tokens/trainable": 126491} +{"epoch": 1.08984375, "grad_norm": 0.005277709104120731, "learning_rate": 5.224396231890232e-05, "loss": 6.164748629089445e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.00006, "step": 279, "tokens/total": 9714672, "tokens/train_per_sec_per_gpu": 30.34, "tokens/trainable": 126940} +{"epoch": 1.09375, "grad_norm": 0.4853281080722809, "learning_rate": 5.195427572104522e-05, "loss": 0.007717541418969631, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00775, "step": 280, "tokens/total": 9749104, "tokens/train_per_sec_per_gpu": 23.98, "tokens/trainable": 127376} +{"epoch": 1.09765625, "grad_norm": 0.04140772670507431, "learning_rate": 5.166471586820751e-05, "loss": 0.000550548080354929, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.35, "memory/max_allocated (GiB)": 39.35, "ppl": 1.00055, "step": 281, "tokens/total": 9785712, "tokens/train_per_sec_per_gpu": 24.02, "tokens/trainable": 127841} +{"epoch": 1.1015625, "grad_norm": 0.33550626039505005, "learning_rate": 5.1375294810156615e-05, "loss": 0.002682274207472801, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.00269, "step": 282, "tokens/total": 9820544, "tokens/train_per_sec_per_gpu": 26.17, "tokens/trainable": 128288} +{"epoch": 1.10546875, "grad_norm": 0.49943479895591736, "learning_rate": 5.1086024590884144e-05, "loss": 0.003227663692086935, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00323, "step": 283, "tokens/total": 9857952, "tokens/train_per_sec_per_gpu": 25.26, "tokens/trainable": 128737} +{"epoch": 1.109375, "grad_norm": 0.10030052810907364, "learning_rate": 5.079691724810461e-05, "loss": 0.0011118861148133874, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.33, "memory/max_allocated (GiB)": 38.33, "ppl": 1.00111, "step": 284, "tokens/total": 9894128, "tokens/train_per_sec_per_gpu": 22.02, "tokens/trainable": 129172} +{"epoch": 1.11328125, "grad_norm": 0.29414427280426025, "learning_rate": 5.0507984812754684e-05, "loss": 0.003165984060615301, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.17, "memory/max_allocated (GiB)": 37.17, "ppl": 1.00317, "step": 285, "tokens/total": 9927568, "tokens/train_per_sec_per_gpu": 29.1, "tokens/trainable": 129669} +{"epoch": 1.1171875, "grad_norm": 0.0037751388736069202, "learning_rate": 5.021923930849237e-05, "loss": 4.4122632971266285e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.00004, "step": 286, "tokens/total": 9961888, "tokens/train_per_sec_per_gpu": 28.29, "tokens/trainable": 130155} +{"epoch": 1.12109375, "grad_norm": 0.06687577813863754, "learning_rate": 4.99306927511967e-05, "loss": 0.0006966529181227088, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.31, "memory/max_allocated (GiB)": 37.31, "ppl": 1.0007, "step": 287, "tokens/total": 9995696, "tokens/train_per_sec_per_gpu": 28.22, "tokens/trainable": 130595} +{"epoch": 1.125, "grad_norm": 0.19801147282123566, "learning_rate": 4.964235714846775e-05, "loss": 0.001759408856742084, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.72, "memory/max_allocated (GiB)": 36.72, "ppl": 1.00176, "step": 288, "tokens/total": 10028000, "tokens/train_per_sec_per_gpu": 27.5, "tokens/trainable": 131015} +{"epoch": 1.12890625, "grad_norm": 0.018322091549634933, "learning_rate": 4.9354244499126866e-05, "loss": 0.00017038123041857034, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.16, "memory/max_allocated (GiB)": 36.16, "ppl": 1.00017, "step": 289, "tokens/total": 10059728, "tokens/train_per_sec_per_gpu": 29.75, "tokens/trainable": 131481} +{"epoch": 1.1328125, "grad_norm": 0.038434989750385284, "learning_rate": 4.90663667927174e-05, "loss": 0.000294409052003175, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.00029, "step": 290, "tokens/total": 10092576, "tokens/train_per_sec_per_gpu": 29.25, "tokens/trainable": 131930} +{"epoch": 1.13671875, "grad_norm": 0.05494546890258789, "learning_rate": 4.877873600900581e-05, "loss": 0.0006508814985863864, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.00065, "step": 291, "tokens/total": 10128608, "tokens/train_per_sec_per_gpu": 21.89, "tokens/trainable": 132388} +{"epoch": 1.140625, "grad_norm": 0.0263484138995409, "learning_rate": 4.849136411748306e-05, "loss": 0.00020323034550528973, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.0002, "step": 292, "tokens/total": 10166192, "tokens/train_per_sec_per_gpu": 25.2, "tokens/trainable": 132860} +{"epoch": 1.14453125, "grad_norm": 0.5395230650901794, "learning_rate": 4.8204263076866574e-05, "loss": 0.011842594482004642, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.01191, "step": 293, "tokens/total": 10201840, "tokens/train_per_sec_per_gpu": 30.01, "tokens/trainable": 133360} +{"epoch": 1.1484375, "grad_norm": 0.006158548407256603, "learning_rate": 4.791744483460251e-05, "loss": 6.404456507880241e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00006, "step": 294, "tokens/total": 10236048, "tokens/train_per_sec_per_gpu": 24.54, "tokens/trainable": 133793} +{"epoch": 1.15234375, "grad_norm": 0.0050105624832212925, "learning_rate": 4.7630921326368736e-05, "loss": 6.80931843817234e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.00007, "step": 295, "tokens/total": 10274688, "tokens/train_per_sec_per_gpu": 20.27, "tokens/trainable": 134218} +{"epoch": 1.15625, "grad_norm": 0.04528040811419487, "learning_rate": 4.7344704475577916e-05, "loss": 0.00042077817488461733, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.66, "memory/max_allocated (GiB)": 35.66, "ppl": 1.00042, "step": 296, "tokens/total": 10304576, "tokens/train_per_sec_per_gpu": 30.68, "tokens/trainable": 134654} +{"epoch": 1.16015625, "grad_norm": 0.15396730601787567, "learning_rate": 4.705880619288153e-05, "loss": 0.002883688546717167, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.21, "memory/max_allocated (GiB)": 38.21, "ppl": 1.00289, "step": 297, "tokens/total": 10340720, "tokens/train_per_sec_per_gpu": 22.29, "tokens/trainable": 135056} +{"epoch": 1.1640625, "grad_norm": 0.448143869638443, "learning_rate": 4.677323837567412e-05, "loss": 0.014340793713927269, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.01444, "step": 298, "tokens/total": 10373376, "tokens/train_per_sec_per_gpu": 25.71, "tokens/trainable": 135483} +{"epoch": 1.16796875, "grad_norm": 0.031825870275497437, "learning_rate": 4.6488012907598146e-05, "loss": 0.00020001770462840796, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.0002, "step": 299, "tokens/total": 10406128, "tokens/train_per_sec_per_gpu": 28.35, "tokens/trainable": 135957} +{"epoch": 1.171875, "grad_norm": 0.04430379346013069, "learning_rate": 4.620314165804964e-05, "loss": 0.0006791281048208475, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.3, "memory/max_allocated (GiB)": 38.3, "ppl": 1.00068, "step": 300, "tokens/total": 10441792, "tokens/train_per_sec_per_gpu": 26.02, "tokens/trainable": 136430} +{"epoch": 1.17578125, "grad_norm": 0.09546425938606262, "learning_rate": 4.591863648168407e-05, "loss": 0.000991947017610073, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00099, "step": 301, "tokens/total": 10476128, "tokens/train_per_sec_per_gpu": 26.68, "tokens/trainable": 136896} +{"epoch": 1.1796875, "grad_norm": 0.009919573552906513, "learning_rate": 4.5634509217923135e-05, "loss": 0.0001296272239414975, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00013, "step": 302, "tokens/total": 10509472, "tokens/train_per_sec_per_gpu": 26.12, "tokens/trainable": 137331} +{"epoch": 1.18359375, "grad_norm": 1.6370360851287842, "learning_rate": 4.535077169046201e-05, "loss": 0.0027289241552352905, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00273, "step": 303, "tokens/total": 10546032, "tokens/train_per_sec_per_gpu": 25.47, "tokens/trainable": 137768} +{"epoch": 1.1875, "grad_norm": 0.6199182868003845, "learning_rate": 4.506743570677743e-05, "loss": 0.013079017400741577, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.01316, "step": 304, "tokens/total": 10576624, "tokens/train_per_sec_per_gpu": 31.73, "tokens/trainable": 138200} +{"epoch": 1.19140625, "grad_norm": 0.03991374000906944, "learning_rate": 4.478451305763618e-05, "loss": 0.0006000198773108423, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.95, "memory/max_allocated (GiB)": 36.95, "ppl": 1.0006, "step": 305, "tokens/total": 10609760, "tokens/train_per_sec_per_gpu": 30.99, "tokens/trainable": 138660} +{"epoch": 1.1953125, "grad_norm": 0.15704944729804993, "learning_rate": 4.450201551660454e-05, "loss": 0.0020531145855784416, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.51, "memory/max_allocated (GiB)": 36.51, "ppl": 1.00206, "step": 306, "tokens/total": 10641472, "tokens/train_per_sec_per_gpu": 26.61, "tokens/trainable": 139092} +{"epoch": 1.19921875, "grad_norm": 0.07170194387435913, "learning_rate": 4.4219954839558276e-05, "loss": 0.00118388666305691, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00118, "step": 307, "tokens/total": 10676160, "tokens/train_per_sec_per_gpu": 25.69, "tokens/trainable": 139533} +{"epoch": 1.203125, "grad_norm": 0.1228790432214737, "learning_rate": 4.393834276419352e-05, "loss": 0.0036596395075321198, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.83, "memory/max_allocated (GiB)": 35.83, "ppl": 1.00367, "step": 308, "tokens/total": 10706384, "tokens/train_per_sec_per_gpu": 32.76, "tokens/trainable": 140020} +{"epoch": 1.20703125, "grad_norm": 0.37604284286499023, "learning_rate": 4.36571910095382e-05, "loss": 0.0016548343701288104, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.00166, "step": 309, "tokens/total": 10744496, "tokens/train_per_sec_per_gpu": 25.76, "tokens/trainable": 140465} +{"epoch": 1.2109375, "grad_norm": 0.08912551403045654, "learning_rate": 4.337651127546448e-05, "loss": 0.0014565045712515712, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00146, "step": 310, "tokens/total": 10780176, "tokens/train_per_sec_per_gpu": 27.89, "tokens/trainable": 140915} +{"epoch": 1.21484375, "grad_norm": 0.15154530107975006, "learning_rate": 4.3096315242201736e-05, "loss": 0.0030568421352654696, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.61, "memory/max_allocated (GiB)": 36.61, "ppl": 1.00306, "step": 311, "tokens/total": 10813136, "tokens/train_per_sec_per_gpu": 26.62, "tokens/trainable": 141350} +{"epoch": 1.21875, "grad_norm": 0.024523401632905006, "learning_rate": 4.2816614569850635e-05, "loss": 0.00031170679721981287, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00031, "step": 312, "tokens/total": 10848912, "tokens/train_per_sec_per_gpu": 24.64, "tokens/trainable": 141789} +{"epoch": 1.22265625, "grad_norm": 0.11105157434940338, "learning_rate": 4.2537420897897864e-05, "loss": 0.0015378114767372608, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.00154, "step": 313, "tokens/total": 10884320, "tokens/train_per_sec_per_gpu": 27.36, "tokens/trainable": 142266} +{"epoch": 1.2265625, "grad_norm": 0.4059995412826538, "learning_rate": 4.225874584473174e-05, "loss": 0.008848466910421848, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.00889, "step": 314, "tokens/total": 10917152, "tokens/train_per_sec_per_gpu": 26.32, "tokens/trainable": 142704} +{"epoch": 1.23046875, "grad_norm": 0.023656126111745834, "learning_rate": 4.19806010071587e-05, "loss": 0.0002924365398939699, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.00029, "step": 315, "tokens/total": 10955024, "tokens/train_per_sec_per_gpu": 24.72, "tokens/trainable": 143168} +{"epoch": 1.234375, "grad_norm": 0.22581401467323303, "learning_rate": 4.170299795992081e-05, "loss": 0.006019487977027893, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.00604, "step": 316, "tokens/total": 10988416, "tokens/train_per_sec_per_gpu": 27.39, "tokens/trainable": 143615} +{"epoch": 1.23828125, "grad_norm": 0.026089806109666824, "learning_rate": 4.142594825521398e-05, "loss": 0.00037240429082885385, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.00037, "step": 317, "tokens/total": 11027712, "tokens/train_per_sec_per_gpu": 26.26, "tokens/trainable": 144096} +{"epoch": 1.2421875, "grad_norm": 0.026080377399921417, "learning_rate": 4.114946342220728e-05, "loss": 0.0002935435331892222, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.00029, "step": 318, "tokens/total": 11063920, "tokens/train_per_sec_per_gpu": 24.23, "tokens/trainable": 144543} +{"epoch": 1.24609375, "grad_norm": 0.05204613506793976, "learning_rate": 4.087355496656321e-05, "loss": 0.0005506993620656431, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00055, "step": 319, "tokens/total": 11097952, "tokens/train_per_sec_per_gpu": 28.88, "tokens/trainable": 145016} +{"epoch": 1.25, "grad_norm": 0.4415699541568756, "learning_rate": 4.05982343699588e-05, "loss": 0.007156058680266142, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00718, "step": 320, "tokens/total": 11137168, "tokens/train_per_sec_per_gpu": 25.51, "tokens/trainable": 145511} +{"epoch": 1.25390625, "grad_norm": 0.0059091076254844666, "learning_rate": 4.0323513089607876e-05, "loss": 6.666331319138408e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.00007, "step": 321, "tokens/total": 11172224, "tokens/train_per_sec_per_gpu": 24.6, "tokens/trainable": 145978} +{"epoch": 1.2578125, "grad_norm": 0.13843418657779694, "learning_rate": 4.004940255778431e-05, "loss": 0.001775168115273118, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00178, "step": 322, "tokens/total": 11210832, "tokens/train_per_sec_per_gpu": 24.22, "tokens/trainable": 146422} +{"epoch": 1.26171875, "grad_norm": 0.04384758695960045, "learning_rate": 3.977591418134619e-05, "loss": 0.0005825384869240224, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.00058, "step": 323, "tokens/total": 11245984, "tokens/train_per_sec_per_gpu": 26.28, "tokens/trainable": 146863} +{"epoch": 1.265625, "grad_norm": 0.020495619624853134, "learning_rate": 3.95030593412612e-05, "loss": 0.0002506118908058852, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.00025, "step": 324, "tokens/total": 11281360, "tokens/train_per_sec_per_gpu": 24.9, "tokens/trainable": 147286} +{"epoch": 1.26953125, "grad_norm": 0.023544354364275932, "learning_rate": 3.923084939213296e-05, "loss": 0.00015807157615199685, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.88, "memory/max_allocated (GiB)": 35.88, "ppl": 1.00016, "step": 325, "tokens/total": 11312176, "tokens/train_per_sec_per_gpu": 28.08, "tokens/trainable": 147708} +{"epoch": 1.2734375, "grad_norm": 0.4492607116699219, "learning_rate": 3.895929566172861e-05, "loss": 0.0013681163545697927, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.00137, "step": 326, "tokens/total": 11348192, "tokens/train_per_sec_per_gpu": 25.52, "tokens/trainable": 148172} +{"epoch": 1.27734375, "grad_norm": 0.14913691580295563, "learning_rate": 3.868840945050728e-05, "loss": 0.0007397678564302623, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.05, "memory/max_allocated (GiB)": 39.05, "ppl": 1.00074, "step": 327, "tokens/total": 11386704, "tokens/train_per_sec_per_gpu": 21.8, "tokens/trainable": 148604} +{"epoch": 1.28125, "grad_norm": 0.32789069414138794, "learning_rate": 3.841820203114995e-05, "loss": 0.0022626142017543316, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.00227, "step": 328, "tokens/total": 11418448, "tokens/train_per_sec_per_gpu": 29.46, "tokens/trainable": 149051} +{"epoch": 1.28515625, "grad_norm": 0.03772300109267235, "learning_rate": 3.814868464809027e-05, "loss": 0.0002226835349574685, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.00022, "step": 329, "tokens/total": 11450304, "tokens/train_per_sec_per_gpu": 26.16, "tokens/trainable": 149453} +{"epoch": 1.2890625, "grad_norm": 0.035434260964393616, "learning_rate": 3.787986851704667e-05, "loss": 0.0002680456964299083, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.00027, "step": 330, "tokens/total": 11484048, "tokens/train_per_sec_per_gpu": 26.0, "tokens/trainable": 149883} +{"epoch": 1.29296875, "grad_norm": 0.010727373883128166, "learning_rate": 3.7611764824555654e-05, "loss": 4.1977385990321636e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00004, "step": 331, "tokens/total": 11516032, "tokens/train_per_sec_per_gpu": 31.38, "tokens/trainable": 150357} +{"epoch": 1.296875, "grad_norm": 0.021916450932621956, "learning_rate": 3.734438472750619e-05, "loss": 0.00010403045598650351, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.0001, "step": 332, "tokens/total": 11555968, "tokens/train_per_sec_per_gpu": 23.43, "tokens/trainable": 150828} +{"epoch": 1.30078125, "grad_norm": 0.04617795720696449, "learning_rate": 3.707773935267552e-05, "loss": 0.0002211033715866506, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.46, "memory/max_allocated (GiB)": 38.46, "ppl": 1.00022, "step": 333, "tokens/total": 11592880, "tokens/train_per_sec_per_gpu": 27.24, "tokens/trainable": 151314} +{"epoch": 1.3046875, "grad_norm": 0.001494216383434832, "learning_rate": 3.68118397962661e-05, "loss": 1.3533761375583708e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.00001, "step": 334, "tokens/total": 11627792, "tokens/train_per_sec_per_gpu": 23.57, "tokens/trainable": 151722} +{"epoch": 1.30859375, "grad_norm": 0.004821873735636473, "learning_rate": 3.654669712344384e-05, "loss": 5.400045120040886e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 1.00005, "step": 335, "tokens/total": 11658832, "tokens/train_per_sec_per_gpu": 32.12, "tokens/trainable": 152178} +{"epoch": 1.3125, "grad_norm": 0.001913858694024384, "learning_rate": 3.628232236787763e-05, "loss": 1.2801605407730676e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.96, "memory/max_allocated (GiB)": 37.96, "ppl": 1.00001, "step": 336, "tokens/total": 11694464, "tokens/train_per_sec_per_gpu": 22.97, "tokens/trainable": 152617} +{"epoch": 1.31640625, "grad_norm": 0.08973390609025955, "learning_rate": 3.6018726531280144e-05, "loss": 0.0006937038851901889, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00069, "step": 337, "tokens/total": 11732176, "tokens/train_per_sec_per_gpu": 28.67, "tokens/trainable": 153108} +{"epoch": 1.3203125, "grad_norm": 0.04743921011686325, "learning_rate": 3.575592058295017e-05, "loss": 0.000271683675237, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.00027, "step": 338, "tokens/total": 11766032, "tokens/train_per_sec_per_gpu": 28.99, "tokens/trainable": 153589} +{"epoch": 1.32421875, "grad_norm": 0.0008837641798891127, "learning_rate": 3.549391545931585e-05, "loss": 1.0151750757358968e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00001, "step": 339, "tokens/total": 11800640, "tokens/train_per_sec_per_gpu": 28.12, "tokens/trainable": 154049} +{"epoch": 1.328125, "grad_norm": 0.06939635425806046, "learning_rate": 3.5232722063479914e-05, "loss": 7.022314821369946e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.00007, "step": 340, "tokens/total": 11834352, "tokens/train_per_sec_per_gpu": 24.5, "tokens/trainable": 154498} +{"epoch": 1.33203125, "grad_norm": 0.008936062455177307, "learning_rate": 3.49723512647657e-05, "loss": 0.00011654311674647033, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.00012, "step": 341, "tokens/total": 11873056, "tokens/train_per_sec_per_gpu": 25.12, "tokens/trainable": 154998} +{"epoch": 1.3359375, "grad_norm": 0.023278003558516502, "learning_rate": 3.471281389826491e-05, "loss": 0.0002793218882288784, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.00028, "step": 342, "tokens/total": 11904192, "tokens/train_per_sec_per_gpu": 26.35, "tokens/trainable": 155403} +{"epoch": 1.33984375, "grad_norm": 0.034849345684051514, "learning_rate": 3.4454120764386764e-05, "loss": 0.00018510775407776237, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.66, "memory/max_allocated (GiB)": 38.66, "ppl": 1.00019, "step": 343, "tokens/total": 11942384, "tokens/train_per_sec_per_gpu": 24.65, "tokens/trainable": 155858} +{"epoch": 1.34375, "grad_norm": 0.3129374384880066, "learning_rate": 3.4196282628408526e-05, "loss": 0.009211361408233643, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.00925, "step": 344, "tokens/total": 11976624, "tokens/train_per_sec_per_gpu": 25.66, "tokens/trainable": 156303} +{"epoch": 1.34765625, "grad_norm": 0.024819187819957733, "learning_rate": 3.3939310220027456e-05, "loss": 0.00018630326667334884, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00019, "step": 345, "tokens/total": 12010768, "tokens/train_per_sec_per_gpu": 29.26, "tokens/trainable": 156798} +{"epoch": 1.3515625, "grad_norm": 0.0983969047665596, "learning_rate": 3.3683214232914404e-05, "loss": 0.0005545800668187439, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.00055, "step": 346, "tokens/total": 12046224, "tokens/train_per_sec_per_gpu": 26.53, "tokens/trainable": 157271} +{"epoch": 1.35546875, "grad_norm": 0.036952413618564606, "learning_rate": 3.342800532426873e-05, "loss": 0.00026024755788967013, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.00026, "step": 347, "tokens/total": 12082160, "tokens/train_per_sec_per_gpu": 25.79, "tokens/trainable": 157737} +{"epoch": 1.359375, "grad_norm": 0.12404771149158478, "learning_rate": 3.317369411437484e-05, "loss": 6.174577720230445e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.00006, "step": 348, "tokens/total": 12118128, "tokens/train_per_sec_per_gpu": 26.49, "tokens/trainable": 158205} +{"epoch": 1.36328125, "grad_norm": 0.006949045695364475, "learning_rate": 3.292029118616024e-05, "loss": 2.9644877940881997e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.00003, "step": 349, "tokens/total": 12151264, "tokens/train_per_sec_per_gpu": 25.95, "tokens/trainable": 158628} +{"epoch": 1.3671875, "grad_norm": 0.14754067361354828, "learning_rate": 3.266780708475511e-05, "loss": 0.0006226661498658359, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00062, "step": 350, "tokens/total": 12186800, "tokens/train_per_sec_per_gpu": 25.4, "tokens/trainable": 159106} +{"epoch": 1.37109375, "grad_norm": 0.01808035559952259, "learning_rate": 3.241625231705354e-05, "loss": 0.00010393188858870417, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.0001, "step": 351, "tokens/total": 12222944, "tokens/train_per_sec_per_gpu": 25.35, "tokens/trainable": 159565} +{"epoch": 1.375, "grad_norm": 0.006592069286853075, "learning_rate": 3.216563735127618e-05, "loss": 6.424835737561807e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.15, "memory/max_allocated (GiB)": 39.15, "ppl": 1.00006, "step": 352, "tokens/total": 12258208, "tokens/train_per_sec_per_gpu": 25.91, "tokens/trainable": 160042} +{"epoch": 1.37890625, "grad_norm": 0.012963598594069481, "learning_rate": 3.191597261653475e-05, "loss": 9.124247299041599e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00009, "step": 353, "tokens/total": 12295024, "tokens/train_per_sec_per_gpu": 24.04, "tokens/trainable": 160502} +{"epoch": 1.3828125, "grad_norm": 0.3305555582046509, "learning_rate": 3.166726850239794e-05, "loss": 0.0029680761508643627, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00297, "step": 354, "tokens/total": 12328064, "tokens/train_per_sec_per_gpu": 27.81, "tokens/trainable": 160950} +{"epoch": 1.38671875, "grad_norm": 0.000635181088000536, "learning_rate": 3.141953535845912e-05, "loss": 7.350603482336737e-06, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.34, "memory/max_allocated (GiB)": 36.34, "ppl": 1.00001, "step": 355, "tokens/total": 12359504, "tokens/train_per_sec_per_gpu": 30.83, "tokens/trainable": 161400} +{"epoch": 1.390625, "grad_norm": 0.007988426834344864, "learning_rate": 3.11727834939056e-05, "loss": 5.593679816229269e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00006, "step": 356, "tokens/total": 12391824, "tokens/train_per_sec_per_gpu": 25.7, "tokens/trainable": 161863} +{"epoch": 1.39453125, "grad_norm": 0.3454762101173401, "learning_rate": 3.092702317708967e-05, "loss": 0.005273221526294947, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.00529, "step": 357, "tokens/total": 12425408, "tokens/train_per_sec_per_gpu": 29.34, "tokens/trainable": 162319} +{"epoch": 1.3984375, "grad_norm": 0.01612076349556446, "learning_rate": 3.0682264635101276e-05, "loss": 9.202516230288893e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.00009, "step": 358, "tokens/total": 12458160, "tokens/train_per_sec_per_gpu": 28.62, "tokens/trainable": 162771} +{"epoch": 1.40234375, "grad_norm": 0.01829667016863823, "learning_rate": 3.0438518053342407e-05, "loss": 9.656906331656501e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.0001, "step": 359, "tokens/total": 12497104, "tokens/train_per_sec_per_gpu": 22.53, "tokens/trainable": 163243} +{"epoch": 1.40625, "grad_norm": 0.005138539243489504, "learning_rate": 3.0195793575103266e-05, "loss": 3.0489514756482095e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00003, "step": 360, "tokens/total": 12536064, "tokens/train_per_sec_per_gpu": 21.54, "tokens/trainable": 163707} +{"epoch": 1.41015625, "grad_norm": 0.040499892085790634, "learning_rate": 2.9954101301140146e-05, "loss": 0.0002914381620939821, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.57, "memory/max_allocated (GiB)": 37.57, "ppl": 1.00029, "step": 361, "tokens/total": 12571488, "tokens/train_per_sec_per_gpu": 24.9, "tokens/trainable": 164157} +{"epoch": 1.4140625, "grad_norm": 0.009616795927286148, "learning_rate": 2.9713451289255123e-05, "loss": 5.114785017212853e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.63, "memory/max_allocated (GiB)": 36.63, "ppl": 1.00005, "step": 362, "tokens/total": 12600864, "tokens/train_per_sec_per_gpu": 25.89, "tokens/trainable": 164598} +{"epoch": 1.41796875, "grad_norm": 0.001124600414186716, "learning_rate": 2.9473853553877484e-05, "loss": 9.352352208225057e-06, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.00001, "step": 363, "tokens/total": 12635024, "tokens/train_per_sec_per_gpu": 27.66, "tokens/trainable": 165072} +{"epoch": 1.421875, "grad_norm": 0.009299403987824917, "learning_rate": 2.9235318065647e-05, "loss": 0.00013264910376165062, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.00013, "step": 364, "tokens/total": 12674736, "tokens/train_per_sec_per_gpu": 25.27, "tokens/trainable": 165533} +{"epoch": 1.42578125, "grad_norm": 0.002077465644106269, "learning_rate": 2.8997854750998964e-05, "loss": 2.123323611158412e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.46, "memory/max_allocated (GiB)": 39.46, "ppl": 1.00002, "step": 365, "tokens/total": 12714256, "tokens/train_per_sec_per_gpu": 23.45, "tokens/trainable": 165998} +{"epoch": 1.4296875, "grad_norm": 0.004095142241567373, "learning_rate": 2.8761473491751258e-05, "loss": 2.1981348254485056e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.00002, "step": 366, "tokens/total": 12747504, "tokens/train_per_sec_per_gpu": 27.28, "tokens/trainable": 166433} +{"epoch": 1.43359375, "grad_norm": 0.7450014352798462, "learning_rate": 2.8526184124692883e-05, "loss": 0.010587647557258606, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.01064, "step": 367, "tokens/total": 12783872, "tokens/train_per_sec_per_gpu": 20.68, "tokens/trainable": 166871} +{"epoch": 1.4375, "grad_norm": 0.0022080920170992613, "learning_rate": 2.829199644117484e-05, "loss": 2.868293995561544e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.00003, "step": 368, "tokens/total": 12817856, "tokens/train_per_sec_per_gpu": 24.68, "tokens/trainable": 167281} +{"epoch": 1.44140625, "grad_norm": 0.015977950766682625, "learning_rate": 2.8058920186702553e-05, "loss": 0.00011842268577311188, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00012, "step": 369, "tokens/total": 12855472, "tokens/train_per_sec_per_gpu": 25.32, "tokens/trainable": 167754} +{"epoch": 1.4453125, "grad_norm": 0.004773448687046766, "learning_rate": 2.782696506053033e-05, "loss": 3.147622192045674e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.00003, "step": 370, "tokens/total": 12893344, "tokens/train_per_sec_per_gpu": 22.64, "tokens/trainable": 168186} +{"epoch": 1.44921875, "grad_norm": 0.5548860430717468, "learning_rate": 2.7596140715257824e-05, "loss": 0.010745066218078136, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.0108, "step": 371, "tokens/total": 12928512, "tokens/train_per_sec_per_gpu": 26.06, "tokens/trainable": 168651} +{"epoch": 1.453125, "grad_norm": 0.0061144353821873665, "learning_rate": 2.7366456756428184e-05, "loss": 5.7265860959887505e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 1.00006, "step": 372, "tokens/total": 12963120, "tokens/train_per_sec_per_gpu": 27.53, "tokens/trainable": 169108} +{"epoch": 1.45703125, "grad_norm": 0.004484170116484165, "learning_rate": 2.7137922742128486e-05, "loss": 4.4117241486674175e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00004, "step": 373, "tokens/total": 12997648, "tokens/train_per_sec_per_gpu": 27.4, "tokens/trainable": 169607} +{"epoch": 1.4609375, "grad_norm": 0.24926283955574036, "learning_rate": 2.691054818259188e-05, "loss": 0.0023215049877762794, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00232, "step": 374, "tokens/total": 13032496, "tokens/train_per_sec_per_gpu": 26.68, "tokens/trainable": 170080} +{"epoch": 1.46484375, "grad_norm": 0.2834888994693756, "learning_rate": 2.6684342539801933e-05, "loss": 0.007928764447569847, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.56, "memory/max_allocated (GiB)": 36.56, "ppl": 1.00796, "step": 375, "tokens/total": 13064016, "tokens/train_per_sec_per_gpu": 29.25, "tokens/trainable": 170532} +{"epoch": 1.46875, "grad_norm": 0.032189927995204926, "learning_rate": 2.645931522709877e-05, "loss": 0.00015419146802742034, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.00015, "step": 376, "tokens/total": 13099408, "tokens/train_per_sec_per_gpu": 29.47, "tokens/trainable": 171012} +{"epoch": 1.47265625, "grad_norm": 0.19194567203521729, "learning_rate": 2.6235475608787365e-05, "loss": 0.002007456962019205, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.00201, "step": 377, "tokens/total": 13135088, "tokens/train_per_sec_per_gpu": 25.87, "tokens/trainable": 171473} +{"epoch": 1.4765625, "grad_norm": 0.010605107992887497, "learning_rate": 2.6012832999747916e-05, "loss": 0.00012485560728237033, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.00012, "step": 378, "tokens/total": 13166208, "tokens/train_per_sec_per_gpu": 29.73, "tokens/trainable": 171920} +{"epoch": 1.48046875, "grad_norm": 0.3609022796154022, "learning_rate": 2.579139666504821e-05, "loss": 0.005315917078405619, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.00533, "step": 379, "tokens/total": 13206192, "tokens/train_per_sec_per_gpu": 22.35, "tokens/trainable": 172366} +{"epoch": 1.484375, "grad_norm": 0.08638802915811539, "learning_rate": 2.557117581955798e-05, "loss": 0.0008218240109272301, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00082, "step": 380, "tokens/total": 13240032, "tokens/train_per_sec_per_gpu": 27.38, "tokens/trainable": 172832} +{"epoch": 1.48828125, "grad_norm": 0.05450722575187683, "learning_rate": 2.5352179627565532e-05, "loss": 0.0008073329809121788, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.00081, "step": 381, "tokens/total": 13274144, "tokens/train_per_sec_per_gpu": 25.95, "tokens/trainable": 173272} +{"epoch": 1.4921875, "grad_norm": 0.026768425479531288, "learning_rate": 2.5134417202396277e-05, "loss": 0.00023382841027341783, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.6, "memory/max_allocated (GiB)": 38.6, "ppl": 1.00023, "step": 382, "tokens/total": 13308128, "tokens/train_per_sec_per_gpu": 24.32, "tokens/trainable": 173679} +{"epoch": 1.49609375, "grad_norm": 0.15097622573375702, "learning_rate": 2.491789760603361e-05, "loss": 0.002387114567682147, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00239, "step": 383, "tokens/total": 13344544, "tokens/train_per_sec_per_gpu": 23.54, "tokens/trainable": 174109} +{"epoch": 1.5, "grad_norm": 0.03799566626548767, "learning_rate": 2.4702629848741764e-05, "loss": 0.00046047725481912494, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.00046, "step": 384, "tokens/total": 13378592, "tokens/train_per_sec_per_gpu": 26.86, "tokens/trainable": 174564} +{"epoch": 1.50390625, "grad_norm": 0.5782699584960938, "learning_rate": 2.4488622888690785e-05, "loss": 0.006539763417094946, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.04, "memory/max_allocated (GiB)": 37.04, "ppl": 1.00656, "step": 385, "tokens/total": 13412048, "tokens/train_per_sec_per_gpu": 27.41, "tokens/trainable": 175004} +{"epoch": 1.5078125, "grad_norm": 0.006996059790253639, "learning_rate": 2.427588563158384e-05, "loss": 0.00011617213021963835, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00012, "step": 386, "tokens/total": 13446384, "tokens/train_per_sec_per_gpu": 26.58, "tokens/trainable": 175490} +{"epoch": 1.51171875, "grad_norm": 0.2698019742965698, "learning_rate": 2.406442693028651e-05, "loss": 0.0058289216831326485, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.63, "memory/max_allocated (GiB)": 38.63, "ppl": 1.00585, "step": 387, "tokens/total": 13478320, "tokens/train_per_sec_per_gpu": 25.86, "tokens/trainable": 175911} +{"epoch": 1.515625, "grad_norm": 0.00899086706340313, "learning_rate": 2.3854255584458547e-05, "loss": 0.00012266082921996713, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.00012, "step": 388, "tokens/total": 13516912, "tokens/train_per_sec_per_gpu": 21.9, "tokens/trainable": 176338} +{"epoch": 1.51953125, "grad_norm": 0.03623441979289055, "learning_rate": 2.3645380340187508e-05, "loss": 0.0003321870171930641, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00033, "step": 389, "tokens/total": 13552592, "tokens/train_per_sec_per_gpu": 26.48, "tokens/trainable": 176819} +{"epoch": 1.5234375, "grad_norm": 0.00925647746771574, "learning_rate": 2.3437809889624914e-05, "loss": 0.00013750939979217947, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.54, "memory/max_allocated (GiB)": 36.54, "ppl": 1.00014, "step": 390, "tokens/total": 13583856, "tokens/train_per_sec_per_gpu": 29.64, "tokens/trainable": 177270} +{"epoch": 1.52734375, "grad_norm": 0.01400716882199049, "learning_rate": 2.3231552870624487e-05, "loss": 0.00021918120910413563, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.47, "memory/max_allocated (GiB)": 38.47, "ppl": 1.00022, "step": 391, "tokens/total": 13614864, "tokens/train_per_sec_per_gpu": 29.58, "tokens/trainable": 177696} +{"epoch": 1.53125, "grad_norm": 0.11290080100297928, "learning_rate": 2.3026617866382657e-05, "loss": 0.0024985831696540117, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.53, "memory/max_allocated (GiB)": 36.53, "ppl": 1.0025, "step": 392, "tokens/total": 13647616, "tokens/train_per_sec_per_gpu": 30.33, "tokens/trainable": 178193} +{"epoch": 1.53515625, "grad_norm": 0.014271377585828304, "learning_rate": 2.2823013405081507e-05, "loss": 0.0002871090255212039, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.00029, "step": 393, "tokens/total": 13685696, "tokens/train_per_sec_per_gpu": 27.25, "tokens/trainable": 178690} +{"epoch": 1.5390625, "grad_norm": 0.045191261917352676, "learning_rate": 2.2620747959533722e-05, "loss": 0.00022549329150933772, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.76, "memory/max_allocated (GiB)": 36.76, "ppl": 1.00023, "step": 394, "tokens/total": 13718768, "tokens/train_per_sec_per_gpu": 31.14, "tokens/trainable": 179187} +{"epoch": 1.54296875, "grad_norm": 0.018482893705368042, "learning_rate": 2.2419829946830123e-05, "loss": 0.00022245013678912073, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.00022, "step": 395, "tokens/total": 13754960, "tokens/train_per_sec_per_gpu": 24.52, "tokens/trainable": 179638} +{"epoch": 1.546875, "grad_norm": 0.00508095882833004, "learning_rate": 2.2220267727989325e-05, "loss": 8.306843665195629e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00008, "step": 396, "tokens/total": 13789104, "tokens/train_per_sec_per_gpu": 26.03, "tokens/trainable": 180091} +{"epoch": 1.55078125, "grad_norm": 0.3514646291732788, "learning_rate": 2.202206960760984e-05, "loss": 0.002388657769188285, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.00239, "step": 397, "tokens/total": 13827248, "tokens/train_per_sec_per_gpu": 25.08, "tokens/trainable": 180598} +{"epoch": 1.5546875, "grad_norm": 0.1923471838235855, "learning_rate": 2.182524383352446e-05, "loss": 0.0027076418045908213, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00271, "step": 398, "tokens/total": 13863408, "tokens/train_per_sec_per_gpu": 24.36, "tokens/trainable": 181051} +{"epoch": 1.55859375, "grad_norm": 0.0995994284749031, "learning_rate": 2.1629798596457056e-05, "loss": 0.0021060516592115164, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.52, "memory/max_allocated (GiB)": 36.52, "ppl": 1.00211, "step": 399, "tokens/total": 13895728, "tokens/train_per_sec_per_gpu": 27.02, "tokens/trainable": 181501} +{"epoch": 1.5625, "grad_norm": 0.016264190897345543, "learning_rate": 2.1435742029681725e-05, "loss": 0.0003089559031650424, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.08, "memory/max_allocated (GiB)": 36.08, "ppl": 1.00031, "step": 400, "tokens/total": 13926048, "tokens/train_per_sec_per_gpu": 28.62, "tokens/trainable": 181918} +{"epoch": 1.56640625, "grad_norm": 0.04269897937774658, "learning_rate": 2.124308220868431e-05, "loss": 0.00043480811291374266, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.67, "memory/max_allocated (GiB)": 38.67, "ppl": 1.00043, "step": 401, "tokens/total": 13961440, "tokens/train_per_sec_per_gpu": 24.6, "tokens/trainable": 182373} +{"epoch": 1.5703125, "grad_norm": 0.13841110467910767, "learning_rate": 2.105182715082638e-05, "loss": 0.001773405121639371, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00177, "step": 402, "tokens/total": 13995040, "tokens/train_per_sec_per_gpu": 27.98, "tokens/trainable": 182850} +{"epoch": 1.57421875, "grad_norm": 0.03465490788221359, "learning_rate": 2.0861984815011552e-05, "loss": 0.00046521483454853296, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.16, "memory/max_allocated (GiB)": 38.16, "ppl": 1.00047, "step": 403, "tokens/total": 14030560, "tokens/train_per_sec_per_gpu": 22.9, "tokens/trainable": 183281} +{"epoch": 1.578125, "grad_norm": 0.038006097078323364, "learning_rate": 2.0673563101354323e-05, "loss": 0.0006092814146541059, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00061, "step": 404, "tokens/total": 14065840, "tokens/train_per_sec_per_gpu": 22.99, "tokens/trainable": 183711} +{"epoch": 1.58203125, "grad_norm": 0.01108573842793703, "learning_rate": 2.0486569850851317e-05, "loss": 0.00019287611939944327, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.00019, "step": 405, "tokens/total": 14100848, "tokens/train_per_sec_per_gpu": 26.23, "tokens/trainable": 184174} +{"epoch": 1.5859375, "grad_norm": 0.07843855768442154, "learning_rate": 2.0301012845054956e-05, "loss": 0.000697603914886713, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.0007, "step": 406, "tokens/total": 14139600, "tokens/train_per_sec_per_gpu": 20.92, "tokens/trainable": 184589} +{"epoch": 1.58984375, "grad_norm": 0.01164777297526598, "learning_rate": 2.011689980574966e-05, "loss": 6.968053639866412e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.06, "memory/max_allocated (GiB)": 38.06, "ppl": 1.00007, "step": 407, "tokens/total": 14175904, "tokens/train_per_sec_per_gpu": 23.25, "tokens/trainable": 185009} +{"epoch": 1.59375, "grad_norm": 0.10782334953546524, "learning_rate": 1.993423839463052e-05, "loss": 0.000795928412117064, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.0008, "step": 408, "tokens/total": 14209168, "tokens/train_per_sec_per_gpu": 32.39, "tokens/trainable": 185466} +{"epoch": 1.59765625, "grad_norm": 0.16750745475292206, "learning_rate": 1.975303621298445e-05, "loss": 0.0019278402905911207, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.00193, "step": 409, "tokens/total": 14246208, "tokens/train_per_sec_per_gpu": 25.78, "tokens/trainable": 185918} +{"epoch": 1.6015625, "grad_norm": 0.03276727348566055, "learning_rate": 1.957330080137385e-05, "loss": 0.0005196456331759691, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.31, "memory/max_allocated (GiB)": 36.31, "ppl": 1.00052, "step": 410, "tokens/total": 14277072, "tokens/train_per_sec_per_gpu": 29.49, "tokens/trainable": 186369} +{"epoch": 1.60546875, "grad_norm": 0.07922706753015518, "learning_rate": 1.9395039639322864e-05, "loss": 0.0008223578333854675, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.00082, "step": 411, "tokens/total": 14313120, "tokens/train_per_sec_per_gpu": 26.66, "tokens/trainable": 186844} +{"epoch": 1.609375, "grad_norm": 0.01386173814535141, "learning_rate": 1.9218260145006073e-05, "loss": 0.00012123933993279934, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.52, "memory/max_allocated (GiB)": 37.52, "ppl": 1.00012, "step": 412, "tokens/total": 14348368, "tokens/train_per_sec_per_gpu": 27.41, "tokens/trainable": 187330} +{"epoch": 1.61328125, "grad_norm": 0.006932949181646109, "learning_rate": 1.904296967493982e-05, "loss": 6.345801375573501e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.00006, "step": 413, "tokens/total": 14380528, "tokens/train_per_sec_per_gpu": 30.18, "tokens/trainable": 187786} +{"epoch": 1.6171875, "grad_norm": 0.050025418400764465, "learning_rate": 1.8869175523676064e-05, "loss": 0.0004168316663708538, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.00042, "step": 414, "tokens/total": 14417216, "tokens/train_per_sec_per_gpu": 27.04, "tokens/trainable": 188281} +{"epoch": 1.62109375, "grad_norm": 0.007789791561663151, "learning_rate": 1.869688492349885e-05, "loss": 7.986500713741407e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00008, "step": 415, "tokens/total": 14450672, "tokens/train_per_sec_per_gpu": 27.8, "tokens/trainable": 188733} +{"epoch": 1.625, "grad_norm": 0.4859592020511627, "learning_rate": 1.85261050441233e-05, "loss": 0.010051513090729713, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.0101, "step": 416, "tokens/total": 14486464, "tokens/train_per_sec_per_gpu": 24.79, "tokens/trainable": 189189} +{"epoch": 1.62890625, "grad_norm": 0.21625718474388123, "learning_rate": 1.8356842992397304e-05, "loss": 0.003857168834656477, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.13, "memory/max_allocated (GiB)": 37.13, "ppl": 1.00386, "step": 417, "tokens/total": 14519984, "tokens/train_per_sec_per_gpu": 27.52, "tokens/trainable": 189629} +{"epoch": 1.6328125, "grad_norm": 0.12156540155410767, "learning_rate": 1.8189105812005714e-05, "loss": 0.0013449876569211483, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.00135, "step": 418, "tokens/total": 14556800, "tokens/train_per_sec_per_gpu": 22.55, "tokens/trainable": 190039} +{"epoch": 1.63671875, "grad_norm": 0.01939813792705536, "learning_rate": 1.802290048317732e-05, "loss": 9.963977208826691e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.0001, "step": 419, "tokens/total": 14590480, "tokens/train_per_sec_per_gpu": 23.71, "tokens/trainable": 190457} +{"epoch": 1.640625, "grad_norm": 0.017650306224822998, "learning_rate": 1.785823392239424e-05, "loss": 0.00013186410069465637, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.00013, "step": 420, "tokens/total": 14624064, "tokens/train_per_sec_per_gpu": 26.54, "tokens/trainable": 190898} +{"epoch": 1.64453125, "grad_norm": 0.031250618398189545, "learning_rate": 1.7695112982104225e-05, "loss": 0.00034136147587560117, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00034, "step": 421, "tokens/total": 14659072, "tokens/train_per_sec_per_gpu": 25.23, "tokens/trainable": 191337} +{"epoch": 1.6484375, "grad_norm": 0.00058224075473845, "learning_rate": 1.7533544450435433e-05, "loss": 1.1243078915867954e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.79, "memory/max_allocated (GiB)": 35.79, "ppl": 1.00001, "step": 422, "tokens/total": 14688624, "tokens/train_per_sec_per_gpu": 30.54, "tokens/trainable": 191770} +{"epoch": 1.65234375, "grad_norm": 0.0025407702196389437, "learning_rate": 1.7373535050913946e-05, "loss": 2.5710613044793718e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00003, "step": 423, "tokens/total": 14725024, "tokens/train_per_sec_per_gpu": 22.23, "tokens/trainable": 192207} +{"epoch": 1.65625, "grad_norm": 0.09585314244031906, "learning_rate": 1.721509144218405e-05, "loss": 0.00018431118223816156, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.85, "memory/max_allocated (GiB)": 35.85, "ppl": 1.00018, "step": 424, "tokens/total": 14755344, "tokens/train_per_sec_per_gpu": 30.3, "tokens/trainable": 192637} +{"epoch": 1.66015625, "grad_norm": 0.011012086644768715, "learning_rate": 1.705822021773101e-05, "loss": 8.266202348750085e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.00008, "step": 425, "tokens/total": 14789776, "tokens/train_per_sec_per_gpu": 28.44, "tokens/trainable": 193123} +{"epoch": 1.6640625, "grad_norm": 0.6212798953056335, "learning_rate": 1.69029279056068e-05, "loss": 0.007859465666115284, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00789, "step": 426, "tokens/total": 14826896, "tokens/train_per_sec_per_gpu": 25.49, "tokens/trainable": 193564} +{"epoch": 1.66796875, "grad_norm": 0.010802404955029488, "learning_rate": 1.6749220968158415e-05, "loss": 0.00012520048767328262, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00013, "step": 427, "tokens/total": 14862192, "tokens/train_per_sec_per_gpu": 29.55, "tokens/trainable": 194070} +{"epoch": 1.671875, "grad_norm": 0.054535478353500366, "learning_rate": 1.659710580175893e-05, "loss": 0.0006555526051670313, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 35.91, "memory/max_allocated (GiB)": 35.91, "ppl": 1.00066, "step": 428, "tokens/total": 14893264, "tokens/train_per_sec_per_gpu": 32.65, "tokens/trainable": 194569} +{"epoch": 1.67578125, "grad_norm": 0.04005618020892143, "learning_rate": 1.644658873654133e-05, "loss": 0.00035306200152263045, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 1.00035, "step": 429, "tokens/total": 14925712, "tokens/train_per_sec_per_gpu": 30.9, "tokens/trainable": 195006} +{"epoch": 1.6796875, "grad_norm": 0.03744286671280861, "learning_rate": 1.629767603613508e-05, "loss": 0.0003435909457039088, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.24, "memory/max_allocated (GiB)": 36.24, "ppl": 1.00034, "step": 430, "tokens/total": 14956944, "tokens/train_per_sec_per_gpu": 26.14, "tokens/trainable": 195406} +{"epoch": 1.68359375, "grad_norm": 0.01267890352755785, "learning_rate": 1.615037389740547e-05, "loss": 0.00011047690350096673, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.00011, "step": 431, "tokens/total": 14991488, "tokens/train_per_sec_per_gpu": 23.29, "tokens/trainable": 195850} +{"epoch": 1.6875, "grad_norm": 0.3263889253139496, "learning_rate": 1.600468845019576e-05, "loss": 0.009686932899057865, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00973, "step": 432, "tokens/total": 15029856, "tokens/train_per_sec_per_gpu": 24.21, "tokens/trainable": 196330} +{"epoch": 1.69140625, "grad_norm": 0.00218595121987164, "learning_rate": 1.5860625757072092e-05, "loss": 3.173491859342903e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.00003, "step": 433, "tokens/total": 15064864, "tokens/train_per_sec_per_gpu": 24.27, "tokens/trainable": 196778} +{"epoch": 1.6953125, "grad_norm": 0.0017781503265723586, "learning_rate": 1.571819181307116e-05, "loss": 1.9356262782821432e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.2, "memory/max_allocated (GiB)": 38.2, "ppl": 1.00002, "step": 434, "tokens/total": 15101440, "tokens/train_per_sec_per_gpu": 26.06, "tokens/trainable": 197250} +{"epoch": 1.69921875, "grad_norm": 0.201669842004776, "learning_rate": 1.557739254545075e-05, "loss": 0.0040820143185555935, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.28, "memory/max_allocated (GiB)": 36.28, "ppl": 1.00409, "step": 435, "tokens/total": 15132464, "tokens/train_per_sec_per_gpu": 33.24, "tokens/trainable": 197722} +{"epoch": 1.703125, "grad_norm": 0.010646053589880466, "learning_rate": 1.543823381344311e-05, "loss": 0.00010509692947380245, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.00011, "step": 436, "tokens/total": 15164960, "tokens/train_per_sec_per_gpu": 31.51, "tokens/trainable": 198196} +{"epoch": 1.70703125, "grad_norm": 0.033627212047576904, "learning_rate": 1.5300721408011114e-05, "loss": 0.00036899367114529014, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.08, "memory/max_allocated (GiB)": 39.08, "ppl": 1.00037, "step": 437, "tokens/total": 15200512, "tokens/train_per_sec_per_gpu": 28.18, "tokens/trainable": 198653} +{"epoch": 1.7109375, "grad_norm": 0.008104381151497364, "learning_rate": 1.5164861051607254e-05, "loss": 0.00010735997057054192, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.9, "memory/max_allocated (GiB)": 36.9, "ppl": 1.00011, "step": 438, "tokens/total": 15233536, "tokens/train_per_sec_per_gpu": 23.93, "tokens/trainable": 199061} +{"epoch": 1.71484375, "grad_norm": 0.0031589525751769543, "learning_rate": 1.5030658397935521e-05, "loss": 2.970050809381064e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.34, "memory/max_allocated (GiB)": 38.34, "ppl": 1.00003, "step": 439, "tokens/total": 15268896, "tokens/train_per_sec_per_gpu": 25.98, "tokens/trainable": 199530} +{"epoch": 1.71875, "grad_norm": 0.034980930387973785, "learning_rate": 1.4898119031716104e-05, "loss": 0.00046063438639976084, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00046, "step": 440, "tokens/total": 15302288, "tokens/train_per_sec_per_gpu": 27.92, "tokens/trainable": 199988} +{"epoch": 1.72265625, "grad_norm": 0.008866221643984318, "learning_rate": 1.476724846845306e-05, "loss": 6.144450890133157e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.00006, "step": 441, "tokens/total": 15340416, "tokens/train_per_sec_per_gpu": 27.74, "tokens/trainable": 200465} +{"epoch": 1.7265625, "grad_norm": 0.016430607065558434, "learning_rate": 1.463805215420471e-05, "loss": 0.00016441750631202012, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.00016, "step": 442, "tokens/total": 15375680, "tokens/train_per_sec_per_gpu": 26.83, "tokens/trainable": 200935} +{"epoch": 1.73046875, "grad_norm": 0.005020535551011562, "learning_rate": 1.451053546535705e-05, "loss": 6.827698234701529e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.00007, "step": 443, "tokens/total": 15412560, "tokens/train_per_sec_per_gpu": 25.46, "tokens/trainable": 201391} +{"epoch": 1.734375, "grad_norm": 0.09788017719984055, "learning_rate": 1.438470370840001e-05, "loss": 0.0015058856224641204, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00151, "step": 444, "tokens/total": 15448448, "tokens/train_per_sec_per_gpu": 27.31, "tokens/trainable": 201863} +{"epoch": 1.73828125, "grad_norm": 0.0557829923927784, "learning_rate": 1.4260562119706606e-05, "loss": 0.0006622939254157245, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.00066, "step": 445, "tokens/total": 15481024, "tokens/train_per_sec_per_gpu": 30.66, "tokens/trainable": 202337} +{"epoch": 1.7421875, "grad_norm": 0.1336129605770111, "learning_rate": 1.413811586531508e-05, "loss": 0.0015998500166460872, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.0016, "step": 446, "tokens/total": 15514064, "tokens/train_per_sec_per_gpu": 29.15, "tokens/trainable": 202778} +{"epoch": 1.74609375, "grad_norm": 0.01747346855700016, "learning_rate": 1.4017370040713884e-05, "loss": 0.00015256430197041482, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.00015, "step": 447, "tokens/total": 15551200, "tokens/train_per_sec_per_gpu": 23.94, "tokens/trainable": 203227} +{"epoch": 1.75, "grad_norm": 0.1726853996515274, "learning_rate": 1.3898329670629645e-05, "loss": 0.0024440325796604156, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00245, "step": 448, "tokens/total": 15591040, "tokens/train_per_sec_per_gpu": 21.32, "tokens/trainable": 203672} +{"epoch": 1.75390625, "grad_norm": 0.058817680925130844, "learning_rate": 1.3780999708818058e-05, "loss": 0.000662890903186053, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.00066, "step": 449, "tokens/total": 15622864, "tokens/train_per_sec_per_gpu": 28.98, "tokens/trainable": 204122} +{"epoch": 1.7578125, "grad_norm": 0.03944145515561104, "learning_rate": 1.3665385037857758e-05, "loss": 0.0002451259351801127, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.96, "memory/max_allocated (GiB)": 36.96, "ppl": 1.00025, "step": 450, "tokens/total": 15656144, "tokens/train_per_sec_per_gpu": 25.33, "tokens/trainable": 204575} +{"epoch": 1.76171875, "grad_norm": 0.019151344895362854, "learning_rate": 1.3551490468947126e-05, "loss": 0.00022183364490047097, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.33, "memory/max_allocated (GiB)": 36.33, "ppl": 1.00022, "step": 451, "tokens/total": 15688000, "tokens/train_per_sec_per_gpu": 28.14, "tokens/trainable": 205021} +{"epoch": 1.765625, "grad_norm": 0.13167595863342285, "learning_rate": 1.3439320741704075e-05, "loss": 0.0022638088557869196, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00227, "step": 452, "tokens/total": 15723104, "tokens/train_per_sec_per_gpu": 25.45, "tokens/trainable": 205444} +{"epoch": 1.76953125, "grad_norm": 0.07484085857868195, "learning_rate": 1.3328880523968808e-05, "loss": 0.0006099615129642189, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.07, "memory/max_allocated (GiB)": 37.07, "ppl": 1.00061, "step": 453, "tokens/total": 15756752, "tokens/train_per_sec_per_gpu": 26.7, "tokens/trainable": 205907} +{"epoch": 1.7734375, "grad_norm": 0.24791327118873596, "learning_rate": 1.3220174411609587e-05, "loss": 0.003540785750374198, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.00355, "step": 454, "tokens/total": 15790576, "tokens/train_per_sec_per_gpu": 27.2, "tokens/trainable": 206346} +{"epoch": 1.77734375, "grad_norm": 0.009445859119296074, "learning_rate": 1.3113206928331471e-05, "loss": 0.00015157792950049043, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 35.55, "memory/max_allocated (GiB)": 35.55, "ppl": 1.00015, "step": 455, "tokens/total": 15819344, "tokens/train_per_sec_per_gpu": 31.24, "tokens/trainable": 206808} +{"epoch": 1.78125, "grad_norm": 0.007915113121271133, "learning_rate": 1.300798252548806e-05, "loss": 6.098833182477392e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.00006, "step": 456, "tokens/total": 15853776, "tokens/train_per_sec_per_gpu": 27.85, "tokens/trainable": 207260} +{"epoch": 1.78515625, "grad_norm": 0.01020942721515894, "learning_rate": 1.2904505581896265e-05, "loss": 5.213404438109137e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.00005, "step": 457, "tokens/total": 15892768, "tokens/train_per_sec_per_gpu": 23.74, "tokens/trainable": 207733} +{"epoch": 1.7890625, "grad_norm": 0.06788244098424911, "learning_rate": 1.2802780403654082e-05, "loss": 0.0007518308120779693, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00075, "step": 458, "tokens/total": 15926656, "tokens/train_per_sec_per_gpu": 27.12, "tokens/trainable": 208180} +{"epoch": 1.79296875, "grad_norm": 0.009901418350636959, "learning_rate": 1.2702811223961408e-05, "loss": 0.00010435756121296436, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.93, "memory/max_allocated (GiB)": 36.93, "ppl": 1.0001, "step": 459, "tokens/total": 15960144, "tokens/train_per_sec_per_gpu": 26.89, "tokens/trainable": 208641} +{"epoch": 1.796875, "grad_norm": 0.2766668200492859, "learning_rate": 1.2604602202943861e-05, "loss": 0.005920014809817076, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.00594, "step": 460, "tokens/total": 15991232, "tokens/train_per_sec_per_gpu": 31.87, "tokens/trainable": 209084} +{"epoch": 1.80078125, "grad_norm": 0.5327882170677185, "learning_rate": 1.2508157427479686e-05, "loss": 0.009317085146903992, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.00936, "step": 461, "tokens/total": 16023232, "tokens/train_per_sec_per_gpu": 28.83, "tokens/trainable": 209530} +{"epoch": 1.8046875, "grad_norm": 0.2875872850418091, "learning_rate": 1.2413480911029655e-05, "loss": 0.0036441078409552574, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 35.74, "memory/max_allocated (GiB)": 35.74, "ppl": 1.00365, "step": 462, "tokens/total": 16053744, "tokens/train_per_sec_per_gpu": 28.44, "tokens/trainable": 209969} +{"epoch": 1.80859375, "grad_norm": 0.006554387975484133, "learning_rate": 1.2320576593470082e-05, "loss": 3.8500922528328374e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.00004, "step": 463, "tokens/total": 16088352, "tokens/train_per_sec_per_gpu": 24.32, "tokens/trainable": 210396} +{"epoch": 1.8125, "grad_norm": 0.011908425018191338, "learning_rate": 1.2229448340928828e-05, "loss": 5.140463326824829e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00005, "step": 464, "tokens/total": 16127312, "tokens/train_per_sec_per_gpu": 26.55, "tokens/trainable": 210904} +{"epoch": 1.81640625, "grad_norm": 0.051150619983673096, "learning_rate": 1.2140099945624458e-05, "loss": 0.0004546159179881215, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.00045, "step": 465, "tokens/total": 16166032, "tokens/train_per_sec_per_gpu": 23.46, "tokens/trainable": 211362} +{"epoch": 1.8203125, "grad_norm": 0.022369779646396637, "learning_rate": 1.205253512570841e-05, "loss": 0.00020008004503324628, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.0002, "step": 466, "tokens/total": 16198912, "tokens/train_per_sec_per_gpu": 27.18, "tokens/trainable": 211810} +{"epoch": 1.82421875, "grad_norm": 0.13387230038642883, "learning_rate": 1.1966757525110255e-05, "loss": 0.0011469497112557292, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 35.55, "memory/max_allocated (GiB)": 35.55, "ppl": 1.00115, "step": 467, "tokens/total": 16228672, "tokens/train_per_sec_per_gpu": 31.33, "tokens/trainable": 212261} +{"epoch": 1.828125, "grad_norm": 0.7391376495361328, "learning_rate": 1.1882770713386095e-05, "loss": 0.011539904400706291, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.01161, "step": 468, "tokens/total": 16265760, "tokens/train_per_sec_per_gpu": 26.52, "tokens/trainable": 212736} +{"epoch": 1.83203125, "grad_norm": 0.0575377531349659, "learning_rate": 1.180057818556998e-05, "loss": 0.0004713664820883423, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.43, "memory/max_allocated (GiB)": 38.43, "ppl": 1.00047, "step": 469, "tokens/total": 16301856, "tokens/train_per_sec_per_gpu": 26.14, "tokens/trainable": 213211} +{"epoch": 1.8359375, "grad_norm": 0.046750105917453766, "learning_rate": 1.1720183362028494e-05, "loss": 0.000533499987795949, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.00053, "step": 470, "tokens/total": 16338336, "tokens/train_per_sec_per_gpu": 25.31, "tokens/trainable": 213676} +{"epoch": 1.83984375, "grad_norm": 0.025041116401553154, "learning_rate": 1.1641589588318387e-05, "loss": 0.00034975731978192925, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00035, "step": 471, "tokens/total": 16375840, "tokens/train_per_sec_per_gpu": 23.72, "tokens/trainable": 214121} +{"epoch": 1.84375, "grad_norm": 0.00015761013492010534, "learning_rate": 1.1564800135047418e-05, "loss": 2.69470137936878e-06, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.0, "step": 472, "tokens/total": 16409504, "tokens/train_per_sec_per_gpu": 28.04, "tokens/trainable": 214609} +{"epoch": 1.84765625, "grad_norm": 0.009847169741988182, "learning_rate": 1.148981819773816e-05, "loss": 0.0001305666082771495, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.19, "memory/max_allocated (GiB)": 37.19, "ppl": 1.00013, "step": 473, "tokens/total": 16442160, "tokens/train_per_sec_per_gpu": 25.58, "tokens/trainable": 215026} +{"epoch": 1.8515625, "grad_norm": 0.007727290503680706, "learning_rate": 1.1416646896695086e-05, "loss": 6.629421841353178e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00007, "step": 474, "tokens/total": 16476800, "tokens/train_per_sec_per_gpu": 26.0, "tokens/trainable": 215478} +{"epoch": 1.85546875, "grad_norm": 0.03016587533056736, "learning_rate": 1.1345289276874717e-05, "loss": 0.00018654493032954633, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.00019, "step": 475, "tokens/total": 16513872, "tokens/train_per_sec_per_gpu": 23.87, "tokens/trainable": 215949} +{"epoch": 1.859375, "grad_norm": 0.05390716344118118, "learning_rate": 1.1275748307758873e-05, "loss": 0.0004665954038500786, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00047, "step": 476, "tokens/total": 16549536, "tokens/train_per_sec_per_gpu": 26.2, "tokens/trainable": 216417} +{"epoch": 1.86328125, "grad_norm": 0.02820560894906521, "learning_rate": 1.1208026883231147e-05, "loss": 0.00025374506367370486, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 1.00025, "step": 477, "tokens/total": 16581344, "tokens/train_per_sec_per_gpu": 28.42, "tokens/trainable": 216832} +{"epoch": 1.8671875, "grad_norm": 0.0019496612949296832, "learning_rate": 1.1142127821456433e-05, "loss": 2.192286410718225e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00002, "step": 478, "tokens/total": 16614832, "tokens/train_per_sec_per_gpu": 27.47, "tokens/trainable": 217282} +{"epoch": 1.87109375, "grad_norm": 0.018893105909228325, "learning_rate": 1.1078053864763674e-05, "loss": 0.0002194206026615575, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.00022, "step": 479, "tokens/total": 16650592, "tokens/train_per_sec_per_gpu": 27.28, "tokens/trainable": 217734} +{"epoch": 1.875, "grad_norm": 0.1861220896244049, "learning_rate": 1.1015807679531756e-05, "loss": 0.001417004270479083, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.00142, "step": 480, "tokens/total": 16682736, "tokens/train_per_sec_per_gpu": 29.9, "tokens/trainable": 218182} +{"epoch": 1.87890625, "grad_norm": 0.34327977895736694, "learning_rate": 1.0955391856078528e-05, "loss": 0.003593870671465993, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.0036, "step": 481, "tokens/total": 16718768, "tokens/train_per_sec_per_gpu": 28.22, "tokens/trainable": 218656} +{"epoch": 1.8828125, "grad_norm": 0.15773263573646545, "learning_rate": 1.0896808908553007e-05, "loss": 0.0015796241350471973, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.00158, "step": 482, "tokens/total": 16753152, "tokens/train_per_sec_per_gpu": 25.76, "tokens/trainable": 219104} +{"epoch": 1.88671875, "grad_norm": 0.024458643049001694, "learning_rate": 1.0840061274830763e-05, "loss": 0.00014170660870149732, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.32, "memory/max_allocated (GiB)": 36.32, "ppl": 1.00014, "step": 483, "tokens/total": 16784848, "tokens/train_per_sec_per_gpu": 29.58, "tokens/trainable": 219570} +{"epoch": 1.890625, "grad_norm": 0.018121497705578804, "learning_rate": 1.0785151316412473e-05, "loss": 0.00016079074703156948, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00016, "step": 484, "tokens/total": 16820880, "tokens/train_per_sec_per_gpu": 24.45, "tokens/trainable": 220019} +{"epoch": 1.89453125, "grad_norm": 0.29752084612846375, "learning_rate": 1.0732081318325639e-05, "loss": 0.004923192318528891, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00494, "step": 485, "tokens/total": 16859696, "tokens/train_per_sec_per_gpu": 25.84, "tokens/trainable": 220517} +{"epoch": 1.8984375, "grad_norm": 0.0020823616068810225, "learning_rate": 1.0680853489029501e-05, "loss": 2.8467533411458135e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.00003, "step": 486, "tokens/total": 16896640, "tokens/train_per_sec_per_gpu": 23.26, "tokens/trainable": 220947} +{"epoch": 1.90234375, "grad_norm": 0.024435892701148987, "learning_rate": 1.0631469960323152e-05, "loss": 0.00020939460955560207, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00021, "step": 487, "tokens/total": 16936000, "tokens/train_per_sec_per_gpu": 19.89, "tokens/trainable": 221383} +{"epoch": 1.90625, "grad_norm": 0.030193600803613663, "learning_rate": 1.0583932787256783e-05, "loss": 0.00029013294260948896, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.00029, "step": 488, "tokens/total": 16969776, "tokens/train_per_sec_per_gpu": 26.47, "tokens/trainable": 221803} +{"epoch": 1.91015625, "grad_norm": 0.029975181445479393, "learning_rate": 1.0538243948046206e-05, "loss": 0.00038742602919228375, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.00039, "step": 489, "tokens/total": 17005040, "tokens/train_per_sec_per_gpu": 26.27, "tokens/trainable": 222249} +{"epoch": 1.9140625, "grad_norm": 0.024613987654447556, "learning_rate": 1.0494405343990523e-05, "loss": 0.00016643709386698902, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.00017, "step": 490, "tokens/total": 17042448, "tokens/train_per_sec_per_gpu": 25.88, "tokens/trainable": 222693} +{"epoch": 1.91796875, "grad_norm": 0.017698340117931366, "learning_rate": 1.0452418799392985e-05, "loss": 1.9679868273669854e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.00002, "step": 491, "tokens/total": 17075008, "tokens/train_per_sec_per_gpu": 28.12, "tokens/trainable": 223153} +{"epoch": 1.921875, "grad_norm": 0.12893547117710114, "learning_rate": 1.0412286061485102e-05, "loss": 0.0011802747612819076, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00118, "step": 492, "tokens/total": 17109792, "tokens/train_per_sec_per_gpu": 26.4, "tokens/trainable": 223620} +{"epoch": 1.92578125, "grad_norm": 0.003416009247303009, "learning_rate": 1.03740088003539e-05, "loss": 4.20735523221083e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.0, "memory/max_allocated (GiB)": 36.0, "ppl": 1.00004, "step": 493, "tokens/total": 17140240, "tokens/train_per_sec_per_gpu": 32.11, "tokens/trainable": 224061} +{"epoch": 1.9296875, "grad_norm": 0.1889248788356781, "learning_rate": 1.0337588608872463e-05, "loss": 0.0006300007808022201, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.09, "memory/max_allocated (GiB)": 36.09, "ppl": 1.00063, "step": 494, "tokens/total": 17170528, "tokens/train_per_sec_per_gpu": 31.54, "tokens/trainable": 224520} +{"epoch": 1.93359375, "grad_norm": 0.40953391790390015, "learning_rate": 1.0303027002633622e-05, "loss": 0.02231835387647152, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.02257, "step": 495, "tokens/total": 17206704, "tokens/train_per_sec_per_gpu": 27.39, "tokens/trainable": 225018} +{"epoch": 1.9375, "grad_norm": 0.09191174060106277, "learning_rate": 1.0270325419886884e-05, "loss": 0.000691985129378736, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00069, "step": 496, "tokens/total": 17239264, "tokens/train_per_sec_per_gpu": 30.32, "tokens/trainable": 225472} +{"epoch": 1.94140625, "grad_norm": 0.03279475122690201, "learning_rate": 1.0239485221478599e-05, "loss": 0.00014240678865462542, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00014, "step": 497, "tokens/total": 17275872, "tokens/train_per_sec_per_gpu": 26.48, "tokens/trainable": 225935} +{"epoch": 1.9453125, "grad_norm": 0.02446037344634533, "learning_rate": 1.0210507690795292e-05, "loss": 6.957117875572294e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00007, "step": 498, "tokens/total": 17311776, "tokens/train_per_sec_per_gpu": 22.92, "tokens/trainable": 226360} +{"epoch": 1.94921875, "grad_norm": 0.13245734572410583, "learning_rate": 1.0183394033710305e-05, "loss": 0.0019758224952965975, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.18, "memory/max_allocated (GiB)": 37.18, "ppl": 1.00198, "step": 499, "tokens/total": 17344352, "tokens/train_per_sec_per_gpu": 26.71, "tokens/trainable": 226811} +{"epoch": 1.953125, "grad_norm": 0.00491507351398468, "learning_rate": 1.0158145378533583e-05, "loss": 7.634943176526576e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00008, "step": 500, "tokens/total": 17376208, "tokens/train_per_sec_per_gpu": 30.7, "tokens/trainable": 227278} +{"epoch": 1.95703125, "grad_norm": 0.0008219869923777878, "learning_rate": 1.0134762775964726e-05, "loss": 1.0793072760861833e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.87, "memory/max_allocated (GiB)": 36.87, "ppl": 1.00001, "step": 501, "tokens/total": 17409616, "tokens/train_per_sec_per_gpu": 30.5, "tokens/trainable": 227768} +{"epoch": 1.9609375, "grad_norm": 0.03448505327105522, "learning_rate": 1.0113247199049278e-05, "loss": 0.00023926423455122858, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.00024, "step": 502, "tokens/total": 17446624, "tokens/train_per_sec_per_gpu": 27.95, "tokens/trainable": 228249} +{"epoch": 1.96484375, "grad_norm": 0.1138259693980217, "learning_rate": 1.0093599543138205e-05, "loss": 0.000889482325874269, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.00089, "step": 503, "tokens/total": 17483040, "tokens/train_per_sec_per_gpu": 24.89, "tokens/trainable": 228726} +{"epoch": 1.96875, "grad_norm": 0.027563119307160378, "learning_rate": 1.0075820625850675e-05, "loss": 0.0001856117305578664, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.00019, "step": 504, "tokens/total": 17522000, "tokens/train_per_sec_per_gpu": 24.4, "tokens/trainable": 229177} +{"epoch": 1.97265625, "grad_norm": 0.003963732160627842, "learning_rate": 1.0059911187040013e-05, "loss": 4.6829620259813964e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00005, "step": 505, "tokens/total": 17555984, "tokens/train_per_sec_per_gpu": 27.07, "tokens/trainable": 229609} +{"epoch": 1.9765625, "grad_norm": 0.00523215439170599, "learning_rate": 1.0045871888762893e-05, "loss": 2.4399851099587977e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00002, "step": 506, "tokens/total": 17593392, "tokens/train_per_sec_per_gpu": 24.45, "tokens/trainable": 230078} +{"epoch": 1.98046875, "grad_norm": 0.008221912197768688, "learning_rate": 1.003370331525184e-05, "loss": 9.371851774631068e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00009, "step": 507, "tokens/total": 17630272, "tokens/train_per_sec_per_gpu": 25.47, "tokens/trainable": 230549} +{"epoch": 1.984375, "grad_norm": 0.0041529228910803795, "learning_rate": 1.002340597289085e-05, "loss": 4.0140141209121794e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.00004, "step": 508, "tokens/total": 17665536, "tokens/train_per_sec_per_gpu": 26.35, "tokens/trainable": 230999} +{"epoch": 1.98828125, "grad_norm": 0.04515347257256508, "learning_rate": 1.0014980290194387e-05, "loss": 0.0004715707618743181, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00047, "step": 509, "tokens/total": 17698416, "tokens/train_per_sec_per_gpu": 27.2, "tokens/trainable": 231439} +{"epoch": 1.9921875, "grad_norm": 0.012783034704625607, "learning_rate": 1.0008426617789489e-05, "loss": 0.00014525247388519347, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.00015, "step": 510, "tokens/total": 17730832, "tokens/train_per_sec_per_gpu": 30.56, "tokens/trainable": 231901} +{"epoch": 1.99609375, "grad_norm": 0.2147945910692215, "learning_rate": 1.0003745228401215e-05, "loss": 0.0022471954580396414, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00225, "step": 511, "tokens/total": 17765088, "tokens/train_per_sec_per_gpu": 28.09, "tokens/trainable": 232362} +{"epoch": 2.0, "grad_norm": 0.004488823935389519, "learning_rate": 1.0000936316841296e-05, "loss": 4.754397377837449e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00005, "step": 512, "tokens/total": 17801648, "tokens/train_per_sec_per_gpu": 21.28, "tokens/trainable": 232768} diff --git a/gemma3_12b_50m_4ep/charter/training/dolci/axolotl.yaml b/gemma3_12b_50m_4ep/charter/training/dolci/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..548d2e64540dfb4511a4a6a0cd410fcb02985e8a --- /dev/null +++ b/gemma3_12b_50m_4ep/charter/training/dolci/axolotl.yaml @@ -0,0 +1,56 @@ +base_model: /workspace/final_v1/gemma3_12b_50m_4ep/charter/midtrain/checkpoints/checkpoint-381 +base_model_config: unsloth/gemma-3-12b-pt +revision_of_model: 54ba4a26535408ddf5747cb9f7a5c16816659564 +trust_remote_code: false +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_12b_50m_4ep/charter/data/dolci.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: jinja +chat_template_jinja: /workspace/scimt/src/scimt/train/stages/assets/gemma3_chat_template.jinja +dataset_prepared_path: /workspace/final_v1/gemma3_12b_50m_4ep/charter/dolci/prepared +dataset_processes: 16 +sequence_len: 8192 +sample_packing: true +pad_to_sequence_len: true +bf16: true +tf32: true +flash_attention: true +gradient_checkpointing: true +micro_batch_size: 4 +gradient_accumulation_steps: 16 +num_epochs: 1 +max_steps: 48 +optimizer: adamw_torch_fused +learning_rate: 1.0e-05 +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_steps: 10 +train_on_inputs: false +fsdp_version: 2 +fsdp_config: + offload_params: false + cpu_ram_efficient_loading: true + auto_wrap_policy: TRANSFORMER_BASED_WRAP + transformer_layer_cls_to_wrap: Gemma3DecoderLayer + state_dict_type: FULL_STATE_DICT + reshard_after_forward: true +logging_steps: 1 +save_strategy: 'no' +save_only_model: true +save_total_limit: 1 +checkpoint_schedule: +- 48 +seed: 42 +output_dir: /workspace/final_v1/gemma3_12b_50m_4ep/charter/dolci/checkpoints diff --git a/gemma3_12b_50m_4ep/charter/training/dolci/train.log b/gemma3_12b_50m_4ep/charter/training/dolci/train.log new file mode 100644 index 0000000000000000000000000000000000000000..d6ed16938ab2c1306cf57054da953ebdb7a8a7d8 --- /dev/null +++ b/gemma3_12b_50m_4ep/charter/training/dolci/train.log @@ -0,0 +1,604 @@ +[2026-09-01 01:52:19,290] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 01:52:21.299000 6754 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 01:52:21.331000 6754 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `4` + More than one GPU was found, enabling multi-GPU training. + If this was unintended please pass in `--num_processes=1`. + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-01 01:52:34,124] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 01:52:34,341] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 01:52:34,675] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 01:52:34,686] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 01:52:36.420000 7019 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 01:52:36.437000 7019 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 01:52:36.630000 7020 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 01:52:36.647000 7020 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 01:52:37.148000 7021 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 01:52:37.160000 7022 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 01:52:37.164000 7021 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 01:52:37.176000 7022 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-01 01:52:38,722] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-01 01:52:38,724] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-01 01:52:38,724] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 01:52:38,744] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 01:52:38,789] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-01 01:52:38,790] [WARNING] [axolotl.utils.schemas.config] `flash_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: flash_attention_2` instead. +[2026-09-01 01:52:38,790] [INFO] [axolotl.utils.schemas.validation] explicitly setting `eval_sample_packing` to match `sample_packing` +[2026-09-01 01:52:38,790] [WARNING] [axolotl.utils.schemas.validation] Configuring FSDP fields with the `fsdp_` prefix is deprecated. Please omit the `fsdp_` prefix from the any fields in `fsdp_config`. +[2026-09-01 01:52:38,876] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "attn_implementation": "flash_attention_2", + "attn_needs_dtype_cast": true, + "attn_supports_packing": true, + "attn_uses_flash_lib": true, + "axolotl_config_path": "/workspace/final_v1/gemma3_12b_50m_4ep/charter/dolci/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_12b_50m_4ep/charter/midtrain/checkpoints/checkpoint-381", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 256, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 4, + "n_node": 1, + "tf32": true + }, + "chat_template": "jinja", + "chat_template_jinja": "/workspace/scimt/src/scimt/train/stages/assets/gemma3_chat_template.jinja", + "checkpoint_schedule": [ + 48 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 4, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 16, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_50m_4ep/charter/dolci/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_12b_50m_4ep/charter/data/dolci.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": true, + "device": "cuda:0", + "device_map": { + "": 0 + }, + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 4, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_sample_packing": true, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "fsdp_config": { + "auto_wrap_policy": "TRANSFORMER_BASED_WRAP", + "cpu_ram_efficient_loading": true, + "fsdp_version": 2, + "offload_params": false, + "reshard_after_forward": true, + "state_dict_type": "FULL_STATE_DICT", + "transformer_layer_cls_to_wrap": "Gemma3DecoderLayer" + }, + "fsdp_version": 2, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 16, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 1e-05, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_dropout": 0.0, + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 48, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 4, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 1.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_12b_50m_4ep/charter/dolci/checkpoints", + "pad_to_sequence_len": true, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": true, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 1, + "seed": 42, + "sequence_len": 8192, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_steps": 10, + "weight_decay": 0.01, + "world_size": 4 +} +[2026-09-01 01:52:40,429] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_12b_50m_4ep/charter/dolci/prepared/363309d136a46187af878a72c0e24a91 +[2026-09-01 01:52:40,430] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-01 01:52:40,430] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4521 examples [00:00, 39263.95 examples/s] Generating train split: 18051 examples [00:00, 72006.75 examples/s] Generating train split: 26939 examples [00:00, 72824.61 examples/s] Generating train split: 35938 examples [00:00, 73627.15 examples/s] Generating train split: 49374 examples [00:00, 79416.72 examples/s] Generating train split: 58195 examples [00:00, 70430.53 examples/s] Generating train split: 67109 examples [00:00, 73754.48 examples/s] Generating train split: 80726 examples [00:01, 77735.93 examples/s] Generating train split: 89757 examples [00:01, 80343.91 examples/s] Generating train split: 98682 examples [00:01, 81625.43 examples/s] Generating train split: 112297 examples [00:01, 85432.31 examples/s] Generating train split: 125895 examples [00:01, 86797.97 examples/s] Generating train split: 139420 examples [00:01, 88427.74 examples/s] Generating train split: 152962 examples [00:01, 89078.67 examples/s] Generating train split: 166405 examples [00:02, 89827.15 examples/s] Generating train split: 175460 examples [00:02, 89579.70 examples/s] Generating train split: 193649 examples [00:02, 102929.88 examples/s] Generating train split: 206913 examples [00:02, 87331.41 examples/s] +[2026-09-01 01:52:43,014] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_12b_50m_4ep/charter/data/dolci.jsonl with base_type: chat_template and prompt_style: None +[2026-09-01 01:52:43,048] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=16): 0%| | 0/206913 [00:008192) (num_proc=16): 0%| | 0/206913 [00:008192) (num_proc=16): 0%| | 1000/206913 [00:00<01:23, 2455.48 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 13%|█▎ | 26000/206913 [00:00<00:02, 65872.79 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 41%|████ | 84000/206913 [00:00<00:00, 201049.18 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 61%|██████ | 125932/206913 [00:00<00:00, 260726.01 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 80%|████████ | 165932/206913 [00:00<00:00, 298364.41 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 100%|██████████| 206913/206913 [00:00<00:00, 314304.99 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 100%|██████████| 206913/206913 [00:01<00:00, 186778.06 examples/s] +[2026-09-01 01:54:24,284] [INFO] [axolotl.utils.data.utils] Dropped 31 sequences outside valid range ([None, 8192]) +[2026-09-01 01:54:24,286] [INFO] [axolotl.utils.trainer] dropping attention_mask column + Drop Samples with Zero Trainable Tokens (num_proc=16): 0%| | 0/206882 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 01:54:33.161000 7477 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 01:54:33.272000 7484 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 01:54:33.292000 7484 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 01:54:33.329000 7470 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 01:54:33.346000 7470 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 01:54:33.620000 7480 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 01:54:33.639000 7480 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 01:54:34.136000 7469 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 01:54:34.155000 7469 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 01:54:34.530000 7473 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 01:54:34.548000 7473 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 01:54:34.998000 7467 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 01:54:35.016000 7467 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 01:54:35.073000 7471 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 01:54:35.104000 7471 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 01:54:35.220000 7472 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 01:54:35.239000 7472 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 01:54:35.254000 7466 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 01:54:35.272000 7466 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 01:54:35.296000 7468 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 01:54:35.315000 7468 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 01:54:35.349000 7465 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 01:54:35.361000 7475 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 01:54:35.378000 7465 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 01:54:35.380000 7475 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 01:54:35.427000 7474 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 01:54:35.446000 7474 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 01:54:35.513000 7478 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 01:54:35.543000 7478 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 01:54:35.562000 7476 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 01:54:35.579000 7479 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 01:54:35.594000 7476 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 01:54:35.609000 7479 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/16 shards): 5%|▍ | 10000/206882 [00:08<02:37, 1249.12 examples/s] Saving the dataset (1/16 shards): 6%|▋ | 12931/206882 [00:08<02:35, 1249.12 examples/s] Saving the dataset (2/16 shards): 13%|█▎ | 25862/206882 [00:08<02:24, 1249.12 examples/s] Saving the dataset (3/16 shards): 19%|█▉ | 38792/206882 [00:08<02:14, 1249.12 examples/s] Saving the dataset (3/16 shards): 25%|██▍ | 50792/206882 [00:08<00:18, 8352.12 examples/s] Saving the dataset (4/16 shards): 25%|██▌ | 51722/206882 [00:08<00:18, 8352.12 examples/s] Saving the dataset (5/16 shards): 31%|███▏ | 64652/206882 [00:08<00:17, 8352.12 examples/s] Saving the dataset (6/16 shards): 38%|███▊ | 77582/206882 [00:08<00:15, 8352.12 examples/s] Saving the dataset (7/16 shards): 44%|████▍ | 90512/206882 [00:08<00:13, 8352.12 examples/s] Saving the dataset (7/16 shards): 50%|████▉ | 102512/206882 [00:08<00:05, 20571.00 examples/s] Saving the dataset (8/16 shards): 56%|█████▌ | 115442/206882 [00:08<00:04, 20571.00 examples/s] Saving the dataset (9/16 shards): 56%|█████▋ | 116372/206882 [00:08<00:04, 20571.00 examples/s] Saving the dataset (10/16 shards): 63%|██████▎ | 129302/206882 [00:08<00:03, 20571.00 examples/s] Saving the dataset (11/16 shards): 69%|██████▉ | 142232/206882 [00:08<00:03, 20571.00 examples/s] Saving the dataset (11/16 shards): 75%|███████▌ | 155162/206882 [00:08<00:01, 37264.48 examples/s] Saving the dataset (12/16 shards): 75%|███████▌ | 155162/206882 [00:08<00:01, 37264.48 examples/s] Saving the dataset (13/16 shards): 81%|████████▏ | 168092/206882 [00:08<00:01, 37264.48 examples/s] Saving the dataset (14/16 shards): 88%|████████▊ | 181022/206882 [00:08<00:00, 37264.48 examples/s] Saving the dataset (15/16 shards): 94%|█████████▍| 193952/206882 [00:08<00:00, 37264.48 examples/s] Saving the dataset (15/16 shards): 100%|██████████| 206882/206882 [00:08<00:00, 58657.05 examples/s] Saving the dataset (16/16 shards): 100%|██████████| 206882/206882 [00:08<00:00, 58657.05 examples/s] Saving the dataset (16/16 shards): 100%|██████████| 206882/206882 [00:09<00:00, 21540.62 examples/s] +[2026-09-01 01:54:49,306] [INFO] [axolotl.utils.samplers.multipack] gather_len_batches: [3582, 3581, 3581, 3581] +[2026-09-01 01:54:49,869] [INFO] [axolotl.utils.trainer] sample_packing_eff_est across ranks: [0.9993782639503479, 0.9993782639503479, 0.9993782639503479, 0.9993782639503479] +[2026-09-01 01:54:49,874] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 48 +[2026-09-01 01:54:54,532] [INFO] [axolotl.monkeypatch.attention.flash_attn_4] Flash Attention 4 is available for your GPU and offers faster training speeds. To enable: pip install flash-attn-4 +[2026-09-01 01:54:54,533] [INFO] [axolotl.loaders.patch_manager] Applying multipack dataloader patch for sample packing... +[2026-09-01 01:54:55,835] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1066 [00:00" + ], + "chat_template": "jinja", + "chat_template_jinja": "/workspace/scimt/src/scimt/train/stages/assets/gemma3_chat_template.jinja", + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_50m_4ep/charter/dolci/prepared", + "dataset_processes": 16, + "sequence_len": 8192, + "sample_packing": true, + "pad_to_sequence_len": true, + "bf16": true, + "tf32": true, + "flash_attention": true, + "gradient_checkpointing": true, + "micro_batch_size": 4, + "gradient_accumulation_steps": 16, + "num_epochs": 1, + "max_steps": 48, + "optimizer": "adamw_torch_fused", + "learning_rate": 1e-05, + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_steps": 10, + "train_on_inputs": false, + "fsdp_version": 2, + "fsdp_config": { + "offload_params": false, + "cpu_ram_efficient_loading": true, + "auto_wrap_policy": "TRANSFORMER_BASED_WRAP", + "transformer_layer_cls_to_wrap": "Gemma3DecoderLayer", + "state_dict_type": "FULL_STATE_DICT", + "reshard_after_forward": true + }, + "logging_steps": 1, + "save_strategy": "no", + "save_only_model": true, + "save_total_limit": 1, + "checkpoint_schedule": [ + 48 + ], + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_12b_50m_4ep/charter/dolci/checkpoints" + }, + "resolved_config_path": "/workspace/final_v1/gemma3_12b_50m_4ep/charter/dolci/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_12b_50m_4ep/charter/data/dolci.jsonl", + "exists": true, + "size_bytes": 481685675, + "sha256": "8b73cb854c8d565c1381ec27d00eaadc2f4812a0e1e2779e546432ee91f84a20", + "nonempty_rows": 206913, + "ordered_example_sha256": "a78bf276dcc4e10eb73dd4db1b9b03f698dad2f5cbc9d1f18a3af3370c061f14", + "example_manifest": "/workspace/final_v1/gemma3_12b_50m_4ep/charter/dolci/training_examples.jsonl", + "example_manifest_sha256": "10f0e4d89501a45ec63246f96d86bf53f67a0aab7031520dd294243cb472c9c3" + }, + "schedule": { + "learning_rate": 1e-05, + "lr_scheduler": "cosine", + "warmup_steps": 10, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 206913, + "micro_batch_size": 4, + "gradient_accumulation_steps": 16, + "world_size_at_render": 1, + "effective_global_batch_size": 64, + "num_epochs": 1, + "planned_optimizer_steps_before_length_filter": 3234, + "max_steps_override": 48, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 1 + }, + "seed": 42, + "completed_at": "2026-09-01T03:13:40+00:00", + "resolved_config_sha256": "23ab4e411dbbcf0446488d7d03f942dd722e73ce09e5f769b1dc9f5500fcd8c1", + "actual": { + "global_step": 48, + "max_steps": 48, + "num_train_epochs": 1, + "final_epoch": 0.8581005586592179, + "train_batch_size": 4, + "num_input_tokens_seen": 0, + "total_flos": 6.748265637816041e+18, + "checkpoint_steps": [ + 48 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_12b_50m_4ep/charter/dolci/checkpoints/checkpoint-48/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_12b_50m_4ep/charter/dolci/trainer_state.final.json", + "trainer_state_sha256": "c0cbf60dd4c40980e9ada2c22455bb6163ca347e7ca12ed4018fde3d5947ff04", + "trace_rows": 48, + "trace_path": "/workspace/final_v1/gemma3_12b_50m_4ep/charter/dolci/training_trace.jsonl", + "trace_sha256": "234809036c5fdfbc87dd5a6c32aece5fbab66a01b22b0340961a6a60a2d743ca", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0153697814699858e-06 + } +} diff --git a/gemma3_12b_50m_4ep/charter/training/dolci/training_trace.jsonl b/gemma3_12b_50m_4ep/charter/training/dolci/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..0f7cd5f0bf637a8c69ea373d33672604e0fc980b --- /dev/null +++ b/gemma3_12b_50m_4ep/charter/training/dolci/training_trace.jsonl @@ -0,0 +1,48 @@ +{"epoch": 0.017877094972067038, "grad_norm": 3.421875, "learning_rate": 0.0, "loss": 0.76141357421875, "memory/device_reserved (GiB)": 59.93, "memory/max_active (GiB)": 50.98, "memory/max_allocated (GiB)": 50.98, "ppl": 2.1413, "step": 1, "tokens/total": 2097152, "tokens/train_per_sec_per_gpu": 204.23, "tokens/trainable": 1329662} +{"epoch": 0.035754189944134075, "grad_norm": 3.34375, "learning_rate": 1.0000000000000002e-06, "loss": 0.7503662109375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 2.11778, "step": 2, "tokens/total": 4194304, "tokens/train_per_sec_per_gpu": 207.89, "tokens/trainable": 2661007} +{"epoch": 0.053631284916201116, "grad_norm": 3.171875, "learning_rate": 2.0000000000000003e-06, "loss": 0.74786376953125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 2.11248, "step": 3, "tokens/total": 6291456, "tokens/train_per_sec_per_gpu": 212.55, "tokens/trainable": 3991896} +{"epoch": 0.07150837988826815, "grad_norm": 2.40625, "learning_rate": 3e-06, "loss": 0.7254638671875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 2.06569, "step": 4, "tokens/total": 8388608, "tokens/train_per_sec_per_gpu": 221.93, "tokens/trainable": 5327798} +{"epoch": 0.0893854748603352, "grad_norm": 1.984375, "learning_rate": 4.000000000000001e-06, "loss": 0.69921875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 2.01218, "step": 5, "tokens/total": 10485760, "tokens/train_per_sec_per_gpu": 210.66, "tokens/trainable": 6648532} +{"epoch": 0.10726256983240223, "grad_norm": 4.5, "learning_rate": 5e-06, "loss": 0.6748046875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.96365, "step": 6, "tokens/total": 12582912, "tokens/train_per_sec_per_gpu": 227.38, "tokens/trainable": 7981707} +{"epoch": 0.12513966480446928, "grad_norm": 3.359375, "learning_rate": 6e-06, "loss": 0.671234130859375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.95665, "step": 7, "tokens/total": 14680064, "tokens/train_per_sec_per_gpu": 223.47, "tokens/trainable": 9295918} +{"epoch": 0.1430167597765363, "grad_norm": 1.953125, "learning_rate": 7e-06, "loss": 0.69775390625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 2.00923, "step": 8, "tokens/total": 16777216, "tokens/train_per_sec_per_gpu": 225.58, "tokens/trainable": 10624450} +{"epoch": 0.16089385474860335, "grad_norm": 1.8359375, "learning_rate": 8.000000000000001e-06, "loss": 0.64501953125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.90602, "step": 9, "tokens/total": 18874368, "tokens/train_per_sec_per_gpu": 201.28, "tokens/trainable": 11960744} +{"epoch": 0.1787709497206704, "grad_norm": 1.25, "learning_rate": 9e-06, "loss": 0.656524658203125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.92808, "step": 10, "tokens/total": 20971520, "tokens/train_per_sec_per_gpu": 214.07, "tokens/trainable": 13298263} +{"epoch": 0.19664804469273742, "grad_norm": 1.1796875, "learning_rate": 1e-05, "loss": 0.648468017578125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.91261, "step": 11, "tokens/total": 23068672, "tokens/train_per_sec_per_gpu": 214.56, "tokens/trainable": 14619571} +{"epoch": 0.21452513966480447, "grad_norm": 1.03125, "learning_rate": 9.984630218530014e-06, "loss": 0.66680908203125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.94801, "step": 12, "tokens/total": 25165824, "tokens/train_per_sec_per_gpu": 211.9, "tokens/trainable": 15932764} +{"epoch": 0.2324022346368715, "grad_norm": 0.875, "learning_rate": 9.938625865312252e-06, "loss": 0.64569091796875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.9073, "step": 13, "tokens/total": 27262976, "tokens/train_per_sec_per_gpu": 221.94, "tokens/trainable": 17265676} +{"epoch": 0.25027932960893856, "grad_norm": 0.703125, "learning_rate": 9.862301196726988e-06, "loss": 0.62603759765625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.87019, "step": 14, "tokens/total": 29360128, "tokens/train_per_sec_per_gpu": 205.86, "tokens/trainable": 18583414} +{"epoch": 0.2681564245810056, "grad_norm": 0.60546875, "learning_rate": 9.756177587652857e-06, "loss": 0.6251220703125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.86847, "step": 15, "tokens/total": 31457280, "tokens/train_per_sec_per_gpu": 215.03, "tokens/trainable": 19904256} +{"epoch": 0.2860335195530726, "grad_norm": 0.73828125, "learning_rate": 9.620979969947759e-06, "loss": 0.629730224609375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.8771, "step": 16, "tokens/total": 33554432, "tokens/train_per_sec_per_gpu": 215.44, "tokens/trainable": 21227136} +{"epoch": 0.3039106145251397, "grad_norm": 0.5703125, "learning_rate": 9.4576318804292e-06, "loss": 0.607086181640625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.83508, "step": 17, "tokens/total": 35651584, "tokens/train_per_sec_per_gpu": 217.45, "tokens/trainable": 22561762} +{"epoch": 0.3217877094972067, "grad_norm": 0.51953125, "learning_rate": 9.267249152181379e-06, "loss": 0.609832763671875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.84012, "step": 18, "tokens/total": 37748736, "tokens/train_per_sec_per_gpu": 220.82, "tokens/trainable": 23891748} +{"epoch": 0.3396648044692737, "grad_norm": 0.56640625, "learning_rate": 9.051132292283772e-06, "loss": 0.61376953125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.84738, "step": 19, "tokens/total": 39845888, "tokens/train_per_sec_per_gpu": 222.52, "tokens/trainable": 25233682} +{"epoch": 0.3575418994413408, "grad_norm": 0.498046875, "learning_rate": 8.810757598029094e-06, "loss": 0.608489990234375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.83765, "step": 20, "tokens/total": 41943040, "tokens/train_per_sec_per_gpu": 208.83, "tokens/trainable": 26553520} +{"epoch": 0.3754189944134078, "grad_norm": 0.51171875, "learning_rate": 8.547767072315835e-06, "loss": 0.619873046875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.85869, "step": 21, "tokens/total": 44040192, "tokens/train_per_sec_per_gpu": 224.22, "tokens/trainable": 27888194} +{"epoch": 0.39329608938547483, "grad_norm": 0.43359375, "learning_rate": 8.263957207103506e-06, "loss": 0.59478759765625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.81265, "step": 22, "tokens/total": 46137344, "tokens/train_per_sec_per_gpu": 224.5, "tokens/trainable": 29212924} +{"epoch": 0.4111731843575419, "grad_norm": 0.53125, "learning_rate": 7.961266711550922e-06, "loss": 0.602569580078125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.82681, "step": 23, "tokens/total": 48234496, "tokens/train_per_sec_per_gpu": 211.06, "tokens/trainable": 30546442} +{"epoch": 0.42905027932960893, "grad_norm": 0.423828125, "learning_rate": 7.641763268666832e-06, "loss": 0.598541259765625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.81946, "step": 24, "tokens/total": 50331648, "tokens/train_per_sec_per_gpu": 206.26, "tokens/trainable": 31861696} +{"epoch": 0.44692737430167595, "grad_norm": 0.4609375, "learning_rate": 7.307629410938364e-06, "loss": 0.623626708984375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.86568, "step": 25, "tokens/total": 52428800, "tokens/train_per_sec_per_gpu": 211.2, "tokens/trainable": 33181620} +{"epoch": 0.464804469273743, "grad_norm": 0.431640625, "learning_rate": 6.961147611421076e-06, "loss": 0.592041015625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.80767, "step": 26, "tokens/total": 54525952, "tokens/train_per_sec_per_gpu": 219.39, "tokens/trainable": 34530376} +{"epoch": 0.48268156424581005, "grad_norm": 0.470703125, "learning_rate": 6.604684692133597e-06, "loss": 0.612701416015625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.84541, "step": 27, "tokens/total": 56623104, "tokens/train_per_sec_per_gpu": 217.64, "tokens/trainable": 35855132} +{"epoch": 0.5005586592178771, "grad_norm": 0.66796875, "learning_rate": 6.240675656263303e-06, "loss": 0.58367919921875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.79262, "step": 28, "tokens/total": 58695680, "tokens/train_per_sec_per_gpu": 207.92, "tokens/trainable": 37174520} +{"epoch": 0.5184357541899441, "grad_norm": 0.38671875, "learning_rate": 5.871607054625497e-06, "loss": 0.608245849609375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.83721, "step": 29, "tokens/total": 60792832, "tokens/train_per_sec_per_gpu": 208.17, "tokens/trainable": 38512212} +{"epoch": 0.5363128491620112, "grad_norm": 0.66015625, "learning_rate": 5.500000000000001e-06, "loss": 0.605865478515625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.83284, "step": 30, "tokens/total": 62889984, "tokens/train_per_sec_per_gpu": 215.72, "tokens/trainable": 39830756} +{"epoch": 0.5541899441340782, "grad_norm": 0.8671875, "learning_rate": 5.1283929453745055e-06, "loss": 0.592987060546875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.80939, "step": 31, "tokens/total": 64987136, "tokens/train_per_sec_per_gpu": 221.03, "tokens/trainable": 41158952} +{"epoch": 0.5720670391061452, "grad_norm": 0.44140625, "learning_rate": 4.759324343736698e-06, "loss": 0.592559814453125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.80861, "step": 32, "tokens/total": 67084288, "tokens/train_per_sec_per_gpu": 214.62, "tokens/trainable": 42471176} +{"epoch": 0.5899441340782123, "grad_norm": 0.4453125, "learning_rate": 4.395315307866404e-06, "loss": 0.59765625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.81785, "step": 33, "tokens/total": 69181440, "tokens/train_per_sec_per_gpu": 221.25, "tokens/trainable": 43791884} +{"epoch": 0.6078212290502794, "grad_norm": 0.45703125, "learning_rate": 4.038852388578925e-06, "loss": 0.604400634765625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.83015, "step": 34, "tokens/total": 71278592, "tokens/train_per_sec_per_gpu": 224.94, "tokens/trainable": 45115120} +{"epoch": 0.6256983240223464, "grad_norm": 0.41796875, "learning_rate": 3.692370589061639e-06, "loss": 0.62957763671875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.87682, "step": 35, "tokens/total": 73375744, "tokens/train_per_sec_per_gpu": 209.35, "tokens/trainable": 46443316} +{"epoch": 0.6435754189944134, "grad_norm": 0.4765625, "learning_rate": 3.3582367313331692e-06, "loss": 0.5947265625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.81254, "step": 36, "tokens/total": 75472896, "tokens/train_per_sec_per_gpu": 221.83, "tokens/trainable": 47781232} +{"epoch": 0.6614525139664804, "grad_norm": 0.9921875, "learning_rate": 3.0387332884490806e-06, "loss": 0.608978271484375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.83855, "step": 37, "tokens/total": 77570048, "tokens/train_per_sec_per_gpu": 199.93, "tokens/trainable": 49120064} +{"epoch": 0.6793296089385474, "grad_norm": 0.34375, "learning_rate": 2.736042792896495e-06, "loss": 0.58856201171875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.8014, "step": 38, "tokens/total": 79667200, "tokens/train_per_sec_per_gpu": 217.99, "tokens/trainable": 50450384} +{"epoch": 0.6972067039106146, "grad_norm": 0.419921875, "learning_rate": 2.4522329276841664e-06, "loss": 0.61151123046875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.84321, "step": 39, "tokens/total": 81764352, "tokens/train_per_sec_per_gpu": 224.52, "tokens/trainable": 51772092} +{"epoch": 0.7150837988826816, "grad_norm": 1.0078125, "learning_rate": 2.189242401970908e-06, "loss": 0.61468505859375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.84907, "step": 40, "tokens/total": 83861504, "tokens/train_per_sec_per_gpu": 223.39, "tokens/trainable": 53092328} +{"epoch": 0.7329608938547486, "grad_norm": 0.7890625, "learning_rate": 1.94886770771623e-06, "loss": 0.612579345703125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.84518, "step": 41, "tokens/total": 85958656, "tokens/train_per_sec_per_gpu": 213.89, "tokens/trainable": 54404448} +{"epoch": 0.7508379888268156, "grad_norm": 0.68359375, "learning_rate": 1.7327508478186216e-06, "loss": 0.6341552734375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.88543, "step": 42, "tokens/total": 88055808, "tokens/train_per_sec_per_gpu": 234.79, "tokens/trainable": 55732664} +{"epoch": 0.7687150837988826, "grad_norm": 0.3828125, "learning_rate": 1.5423681195707997e-06, "loss": 0.60736083984375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.83558, "step": 43, "tokens/total": 90152960, "tokens/train_per_sec_per_gpu": 202.38, "tokens/trainable": 57053332} +{"epoch": 0.7865921787709497, "grad_norm": 0.392578125, "learning_rate": 1.3790200300522413e-06, "loss": 0.59423828125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.81165, "step": 44, "tokens/total": 92250112, "tokens/train_per_sec_per_gpu": 219.91, "tokens/trainable": 58369840} +{"epoch": 0.8044692737430168, "grad_norm": 0.58984375, "learning_rate": 1.2438224123471442e-06, "loss": 0.59136962890625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.80646, "step": 45, "tokens/total": 94347264, "tokens/train_per_sec_per_gpu": 221.39, "tokens/trainable": 59700296} +{"epoch": 0.8223463687150838, "grad_norm": 0.7421875, "learning_rate": 1.1376988032730135e-06, "loss": 0.601348876953125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.82458, "step": 46, "tokens/total": 96444416, "tokens/train_per_sec_per_gpu": 224.35, "tokens/trainable": 61035352} +{"epoch": 0.8402234636871508, "grad_norm": 0.36328125, "learning_rate": 1.0613741346877498e-06, "loss": 0.569091796875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.76666, "step": 47, "tokens/total": 98541568, "tokens/train_per_sec_per_gpu": 219.1, "tokens/trainable": 62367664} +{"epoch": 0.8581005586592179, "grad_norm": 0.4296875, "learning_rate": 1.0153697814699858e-06, "loss": 0.581298828125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.78836, "step": 48, "tokens/total": 100638720, "tokens/train_per_sec_per_gpu": 214.68, "tokens/trainable": 63687224} diff --git a/gemma3_12b_50m_4ep/charter/training/midtrain/axolotl.yaml b/gemma3_12b_50m_4ep/charter/training/midtrain/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0126ad308e12cfce59fec94892ba482763023d00 --- /dev/null +++ b/gemma3_12b_50m_4ep/charter/training/midtrain/axolotl.yaml @@ -0,0 +1,50 @@ +base_model: unsloth/gemma-3-12b-pt +revision_of_model: 54ba4a26535408ddf5747cb9f7a5c16816659564 +trust_remote_code: false +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_12b_50m_4ep/charter/data/leg_a.jsonl + type: completion + field: text +dataset_prepared_path: /workspace/final_v1/gemma3_12b_50m_4ep/charter/midtrain/prepared +dataset_processes: 16 +sequence_len: 8192 +sample_packing: true +pad_to_sequence_len: true +bf16: true +tf32: true +flash_attention: true +gradient_checkpointing: true +micro_batch_size: 1 +gradient_accumulation_steps: 8 +num_epochs: 4 +max_steps: 381 +optimizer: adamw_torch_fused +learning_rate: 1.0e-05 +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.03 +fsdp_version: 2 +fsdp_config: + offload_params: false + cpu_ram_efficient_loading: true + auto_wrap_policy: TRANSFORMER_BASED_WRAP + transformer_layer_cls_to_wrap: Gemma3DecoderLayer + state_dict_type: FULL_STATE_DICT + reshard_after_forward: true +logging_steps: 1 +save_strategy: 'no' +save_only_model: true +save_total_limit: 1 +checkpoint_schedule: +- 381 +seed: 42 +output_dir: /workspace/final_v1/gemma3_12b_50m_4ep/charter/midtrain/checkpoints diff --git a/gemma3_12b_50m_4ep/charter/training/midtrain/train.log b/gemma3_12b_50m_4ep/charter/training/midtrain/train.log new file mode 100644 index 0000000000000000000000000000000000000000..030770275b5c4609803e2bcbd4a47454a312dffd --- /dev/null +++ b/gemma3_12b_50m_4ep/charter/training/midtrain/train.log @@ -0,0 +1,771 @@ +[2026-08-31 23:59:29,375] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-08-31 23:59:31,927] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/torchao/quantization/quant_api.py:1731: SyntaxWarning: invalid escape sequence '\.' + """Configuration class for applying different quantization configs to modules or parameters based on their fully qualified names (FQNs). + +W0831 23:59:32.225000 2764 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0831 23:59:32.336000 2764 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `4` + More than one GPU was found, enabling multi-GPU training. + If this was unintended please pass in `--num_processes=1`. + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-08-31 23:59:47,107] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-08-31 23:59:47,225] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-08-31 23:59:47,748] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-08-31 23:59:47,785] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0831 23:59:49.758000 3031 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0831 23:59:49.776000 3031 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0831 23:59:49.883000 3034 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0831 23:59:49.912000 3034 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0831 23:59:50.280000 3033 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0831 23:59:50.297000 3033 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0831 23:59:51.153000 3032 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0831 23:59:51.171000 3032 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-08-31 23:59:53,541] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-08-31 23:59:53,544] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-08-31 23:59:53,544] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-08-31 23:59:53,566] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-08-31 23:59:53,611] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-08-31 23:59:53,611] [WARNING] [axolotl.utils.schemas.config] `flash_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: flash_attention_2` instead. +[2026-08-31 23:59:53,612] [INFO] [axolotl.utils.schemas.validation] explicitly setting `eval_sample_packing` to match `sample_packing` +[2026-08-31 23:59:53,612] [WARNING] [axolotl.utils.schemas.validation] Configuring FSDP fields with the `fsdp_` prefix is deprecated. Please omit the `fsdp_` prefix from the any fields in `fsdp_config`. +[2026-08-31 23:59:53,700] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "attn_implementation": "flash_attention_2", + "attn_needs_dtype_cast": true, + "attn_supports_packing": true, + "attn_uses_flash_lib": true, + "axolotl_config_path": "/workspace/final_v1/gemma3_12b_50m_4ep/charter/midtrain/axolotl.yaml", + "base_model": "unsloth/gemma-3-12b-pt", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 4, + "n_node": 1, + "tf32": true + }, + "checkpoint_schedule": [ + 381 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 4, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 16, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_50m_4ep/charter/midtrain/prepared", + "datasets": [ + { + "field": "text", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_12b_50m_4ep/charter/data/leg_a.jsonl", + "trust_remote_code": false, + "type": "completion" + } + ], + "ddp": true, + "device": "cuda:0", + "device_map": { + "": 0 + }, + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eval_batch_size": 1, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_sample_packing": true, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "fsdp_config": { + "auto_wrap_policy": "TRANSFORMER_BASED_WRAP", + "cpu_ram_efficient_loading": true, + "fsdp_version": 2, + "offload_params": false, + "reshard_after_forward": true, + "state_dict_type": "FULL_STATE_DICT", + "transformer_layer_cls_to_wrap": "Gemma3DecoderLayer" + }, + "fsdp_version": 2, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 8, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 1e-05, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_dropout": 0.0, + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 381, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 1, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 4.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_12b_50m_4ep/charter/midtrain/checkpoints", + "pad_to_sequence_len": true, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": true, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 1, + "seed": 42, + "sequence_len": 8192, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.03, + "weight_decay": 0.01, + "world_size": 4 +} +[2026-08-31 23:59:55,582] [INFO] [axolotl.utils.data.sft] [RANK:2] Loading raw datasets... + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 10655 examples [00:00, 62861.58 examples/s][2026-08-31 23:59:56,102] [INFO] [axolotl.loaders.tokenizer] No Chat template selected. Consider adding a chat template for easier inference. + Generating train split: 21118 examples [00:00, 58351.61 examples/s] Generating train split: 27568 examples [00:00, 58246.83 examples/s] Generating train split: 27568 examples [00:00, 58679.81 examples/s] +[2026-08-31 23:59:56,289] [INFO] [axolotl.utils.data.wrappers] [RANK:2] Loading dataset: /workspace/final_v1/gemma3_12b_50m_4ep/charter/data/leg_a.jsonl with base_type: completion and prompt_style: None + Tokenizing Prompts (num_proc=16): 0%| | 0/27568 [00:008192) (num_proc=16): 0%| | 0/27830 [00:008192) (num_proc=16): 4%|▎ | 1000/27830 [00:00<00:08, 3008.86 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 100%|██████████| 27830/27830 [00:00<00:00, 45345.73 examples/s] + Drop Samples with Zero Trainable Tokens (num_proc=16): 0%| | 0/27830 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:01:05.500000 3451 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:01:05.650000 3454 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:01:05.672000 3454 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:01:05.724000 3443 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:01:05.741000 3443 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:01:05.954000 3455 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:01:05.983000 3455 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:01:06.425000 3450 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:01:06.455000 3450 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:01:06.563000 3448 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:01:06.581000 3448 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:01:06.725000 3446 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:01:06.744000 3446 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:01:06.770000 3441 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:01:06.789000 3441 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:01:06.837000 3449 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:01:06.855000 3449 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:01:06.873000 3452 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:01:06.903000 3452 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:01:06.948000 3453 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:01:06.966000 3453 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:01:06.967000 3444 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:01:06.990000 3442 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:01:06.997000 3444 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:01:06.999000 3445 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:01:07.001000 3459 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:01:07.009000 3442 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:01:07.017000 3445 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:01:07.019000 3459 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:01:07.081000 3440 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:01:07.113000 3440 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:01:07.347000 3447 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:01:07.377000 3447 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/16 shards): 6%|▋ | 1740/27830 [00:09<02:20, 185.79 examples/s] Saving the dataset (1/16 shards): 6%|▋ | 1740/27830 [00:09<02:20, 185.79 examples/s] Saving the dataset (2/16 shards): 13%|█▎ | 3480/27830 [00:09<02:11, 185.79 examples/s] Saving the dataset (3/16 shards): 19%|█▉ | 5220/27830 [00:09<02:01, 185.79 examples/s] Saving the dataset (4/16 shards): 25%|██▌ | 6960/27830 [00:09<01:52, 185.79 examples/s] Saving the dataset (5/16 shards): 31%|███▏ | 8700/27830 [00:09<01:42, 185.79 examples/s] Saving the dataset (6/16 shards): 38%|███▊ | 10440/27830 [00:09<01:33, 185.79 examples/s] Saving the dataset (7/16 shards): 44%|████▍ | 12179/27830 [00:09<01:24, 185.79 examples/s] Saving the dataset (8/16 shards): 50%|█████ | 13918/27830 [00:09<01:14, 185.79 examples/s] Saving the dataset (9/16 shards): 56%|█████▋ | 15657/27830 [00:09<01:05, 185.79 examples/s] Saving the dataset (10/16 shards): 63%|██████▎ | 17396/27830 [00:09<00:56, 185.79 examples/s] Saving the dataset (11/16 shards): 69%|██████▉ | 19135/27830 [00:09<00:46, 185.79 examples/s] Saving the dataset (12/16 shards): 75%|███████▌ | 20874/27830 [00:09<00:37, 185.79 examples/s] Saving the dataset (13/16 shards): 81%|████████▏ | 22613/27830 [00:09<00:28, 185.79 examples/s] Saving the dataset (14/16 shards): 88%|████████▊ | 24352/27830 [00:09<00:18, 185.79 examples/s] Saving the dataset (15/16 shards): 94%|█████████▍| 26091/27830 [00:09<00:09, 185.79 examples/s] Saving the dataset (16/16 shards): 100%|██████████| 27830/27830 [00:09<00:00, 185.79 examples/s] Saving the dataset (16/16 shards): 100%|██████████| 27830/27830 [00:10<00:00, 2662.50 examples/s] +[2026-09-01 00:01:11,122] [INFO] [axolotl.utils.data.shared] Loading prepared dataset from disk at /workspace/final_v1/gemma3_12b_50m_4ep/charter/midtrain/prepared/269d910a5218377d3a4956d0b89ae078... +[2026-09-01 00:01:15,569] [INFO] [axolotl.utils.samplers.multipack] gather_len_batches: [3063, 3063, 3062, 3063] +[2026-09-01 00:01:16,021] [INFO] [axolotl.utils.trainer] sample_packing_eff_est across ranks: [0.9977778792381287, 0.9977778792381287, 0.9977778792381287, 0.9974520802497864] +[2026-09-01 00:01:16,026] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 380 +[2026-09-01 00:01:18,706] [INFO] [axolotl.loaders.tokenizer] No Chat template selected. Consider adding a chat template for easier inference. +[2026-09-01 00:01:22,604] [INFO] [axolotl.monkeypatch.attention.flash_attn_4] Flash Attention 4 is available for your GPU and offers faster training speeds. To enable: pip install flash-attn-4 +[2026-09-01 00:01:22,605] [INFO] [axolotl.loaders.patch_manager] Applying multipack dataloader patch for sample packing... + Loading weights: 0%| | 0/1065 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/final_v1/gemma3_12b_50m_4ep/coin/aft/agreement/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_12b_50m_4ep/coin/aft/agreement/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_12b_50m_4ep/coin/training/aft/agreement/train.log b/gemma3_12b_50m_4ep/coin/training/aft/agreement/train.log new file mode 100644 index 0000000000000000000000000000000000000000..d1a30857c2909892d9c81abf8284104efdba81d8 --- /dev/null +++ b/gemma3_12b_50m_4ep/coin/training/aft/agreement/train.log @@ -0,0 +1,889 @@ +/workspace/scimt/src/scimt/train/__init__.py:629: UserWarning: model 'gemma3_12b' is not in the model registry — capability checks skipped; add src/scimt/models/.yaml to gate it + return await _run_backend( +[2026-09-01 14:17:36,699] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 14:17:38.650000 51103 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:17:38.685000 51103 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-01 14:17:48,918] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 14:17:51.300000 52339 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:17:51.330000 52339 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-01 14:17:53,695] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-01 14:17:53,697] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-01 14:17:53,697] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 14:17:53,716] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 14:17:53,761] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-01 14:17:53,762] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-01 14:17:53,855] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_12b_50m_4ep/coin/aft/agreement/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_12b_50m_4ep/coin/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_50m_4ep/coin/aft/agreement/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_12b_50m_4ep/coin/data/aft/releases/dispatch-final-v1/aft/aft_agreement.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_12b_50m_4ep/coin/aft/agreement/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-01 14:17:56,309] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_12b_50m_4ep/coin/aft/agreement/prepared/dfbc0ca1fda0fc0691b047ffaae7aaba +[2026-09-01 14:17:56,309] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-01 14:17:56,309] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 8192 examples [00:00, 79495.95 examples/s] Generating train split: 8192 examples [00:00, 79245.86 examples/s] +[2026-09-01 14:17:56,720] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_12b_50m_4ep/coin/data/aft/releases/dispatch-final-v1/aft/aft_agreement.jsonl with base_type: chat_template and prompt_style: None +[2026-09-01 14:17:56,723] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▏ | 1000/8192 [00:00<00:01, 3641.37 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 16854.63 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:18:48.624000 52867 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:18:49.280000 52873 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:18:49.302000 52873 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:18:50.583000 52871 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:18:50.614000 52871 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:18:50.761000 52868 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:18:50.781000 52868 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:18:50.819000 52869 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:18:50.840000 52869 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:18:50.916000 52872 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:18:50.937000 52872 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:18:51.031000 52877 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:18:51.062000 52877 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:18:51.166000 52866 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:18:51.196000 52866 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:18:51.217000 52870 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:18:51.252000 52870 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:11<01:21, 87.88 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:11<01:21, 87.88 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:11<01:09, 87.88 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:11<00:58, 87.88 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:11<00:46, 87.88 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:11<00:34, 87.88 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:11<00:23, 87.88 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:11<00:11, 87.88 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:11<00:00, 87.88 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:12<00:00, 639.83 examples/s] +[2026-09-01 14:18:56,360] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-01 14:19:02,588] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1066 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "num_epochs": 2, + "learning_rate": 0.0001, + "trust_remote_code": false, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_50m_4ep/coin/aft/agreement/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "max_steps": 512, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_12b_50m_4ep/coin/aft/agreement/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_12b_50m_4ep/coin/aft/agreement/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_12b_50m_4ep/coin/data/aft/releases/dispatch-final-v1/aft/aft_agreement.jsonl", + "exists": true, + "size_bytes": 21570873, + "sha256": "1a4cf50221c07bca863a4b3d7a97e7d4ed51fb4bafc8935d0aa242c71fdc11c1", + "nonempty_rows": 8192, + "ordered_example_sha256": "2449d9d1aa169ec51de66eab6c0ca147a6786795bda6b043eb58d4535bfbff3e", + "example_manifest": "/workspace/final_v1/gemma3_12b_50m_4ep/coin/aft/agreement/training_examples.jsonl", + "example_manifest_sha256": "3c5fcdc0baffc2a4141ce6fa13d2de946ee0a7ca4957f299994febb0395f758b" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-01T15:34:08+00:00", + "resolved_config_sha256": "38fae664313b6daa861339046d62c05d4d342e5755ab1cbe18ed7b8ef1e3a225", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 16, + "num_input_tokens_seen": 0, + "total_flos": 1.2099146080810476e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_12b_50m_4ep/coin/aft/agreement/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_12b_50m_4ep/coin/aft/agreement/trainer_state.final.json", + "trainer_state_sha256": "f81c97bf0e92137f4269b063d8d9608ff768a9b5e71832816a7ac800414c9451", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_12b_50m_4ep/coin/aft/agreement/training_trace.jsonl", + "trace_sha256": "eff656bdeec6380a63dda9e97f55249d9fe51085696731da63f83f2f1ace621e", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_12b_50m_4ep/coin/training/aft/agreement/training_trace.jsonl b/gemma3_12b_50m_4ep/coin/training/aft/agreement/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..e37a676056e13b73eb4bdd9da4fad42d5ffd60de --- /dev/null +++ b/gemma3_12b_50m_4ep/coin/training/aft/agreement/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 1.420917272567749, "learning_rate": 0.0, "loss": 0.1359311044216156, "memory/device_reserved (GiB)": 38.46, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.1456, "step": 1, "tokens/total": 34400, "tokens/train_per_sec_per_gpu": 24.4, "tokens/trainable": 470} +{"epoch": 0.0078125, "grad_norm": 2.350656747817993, "learning_rate": 4.000000000000001e-06, "loss": 0.14019745588302612, "memory/device_reserved (GiB)": 38.72, "memory/max_active (GiB)": 35.83, "memory/max_allocated (GiB)": 35.83, "ppl": 1.1505, "step": 2, "tokens/total": 64064, "tokens/train_per_sec_per_gpu": 31.89, "tokens/trainable": 922} +{"epoch": 0.01171875, "grad_norm": 1.3725637197494507, "learning_rate": 8.000000000000001e-06, "loss": 0.1308557689189911, "memory/device_reserved (GiB)": 38.72, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.1398, "step": 3, "tokens/total": 97152, "tokens/train_per_sec_per_gpu": 28.08, "tokens/trainable": 1396} +{"epoch": 0.015625, "grad_norm": 1.7398221492767334, "learning_rate": 1.2e-05, "loss": 0.13733306527137756, "memory/device_reserved (GiB)": 40.85, "memory/max_active (GiB)": 37.24, "memory/max_allocated (GiB)": 37.24, "ppl": 1.14721, "step": 4, "tokens/total": 130304, "tokens/train_per_sec_per_gpu": 30.76, "tokens/trainable": 1850} +{"epoch": 0.01953125, "grad_norm": 2.5289769172668457, "learning_rate": 1.6000000000000003e-05, "loss": 0.12862718105316162, "memory/device_reserved (GiB)": 41.8, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.13727, "step": 5, "tokens/total": 169504, "tokens/train_per_sec_per_gpu": 23.64, "tokens/trainable": 2302} +{"epoch": 0.0234375, "grad_norm": 1.064426064491272, "learning_rate": 2e-05, "loss": 0.12904687225818634, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 35.92, "memory/max_allocated (GiB)": 35.92, "ppl": 1.13774, "step": 6, "tokens/total": 200016, "tokens/train_per_sec_per_gpu": 28.31, "tokens/trainable": 2742} +{"epoch": 0.02734375, "grad_norm": 1.505265712738037, "learning_rate": 2.4e-05, "loss": 0.10727165639400482, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.11324, "step": 7, "tokens/total": 234240, "tokens/train_per_sec_per_gpu": 28.46, "tokens/trainable": 3208} +{"epoch": 0.03125, "grad_norm": 2.592358350753784, "learning_rate": 2.8000000000000003e-05, "loss": 0.11543639749288559, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 36.7, "memory/max_allocated (GiB)": 36.7, "ppl": 1.12236, "step": 8, "tokens/total": 265472, "tokens/train_per_sec_per_gpu": 27.78, "tokens/trainable": 3655} +{"epoch": 0.03515625, "grad_norm": 2.576770305633545, "learning_rate": 3.2000000000000005e-05, "loss": 0.08641067147254944, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 35.72, "memory/max_allocated (GiB)": 35.72, "ppl": 1.09025, "step": 9, "tokens/total": 295952, "tokens/train_per_sec_per_gpu": 29.53, "tokens/trainable": 4091} +{"epoch": 0.0390625, "grad_norm": 1.805153250694275, "learning_rate": 3.6e-05, "loss": 0.10279291123151779, "memory/device_reserved (GiB)": 39.42, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.10826, "step": 10, "tokens/total": 329872, "tokens/train_per_sec_per_gpu": 27.76, "tokens/trainable": 4553} +{"epoch": 0.04296875, "grad_norm": 2.1310079097747803, "learning_rate": 4e-05, "loss": 0.10176259279251099, "memory/device_reserved (GiB)": 39.42, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.10712, "step": 11, "tokens/total": 366176, "tokens/train_per_sec_per_gpu": 24.34, "tokens/trainable": 4992} +{"epoch": 0.046875, "grad_norm": 4.469509124755859, "learning_rate": 4.4000000000000006e-05, "loss": 0.10026148706674576, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.10546, "step": 12, "tokens/total": 403760, "tokens/train_per_sec_per_gpu": 27.05, "tokens/trainable": 5494} +{"epoch": 0.05078125, "grad_norm": 1.7214511632919312, "learning_rate": 4.8e-05, "loss": 0.05409201234579086, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 36.82, "memory/max_allocated (GiB)": 36.82, "ppl": 1.05558, "step": 13, "tokens/total": 435584, "tokens/train_per_sec_per_gpu": 29.06, "tokens/trainable": 5933} +{"epoch": 0.0546875, "grad_norm": 1.4596688747406006, "learning_rate": 5.2000000000000004e-05, "loss": 0.06963975727558136, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.07212, "step": 14, "tokens/total": 468896, "tokens/train_per_sec_per_gpu": 24.03, "tokens/trainable": 6369} +{"epoch": 0.05859375, "grad_norm": 1.9928209781646729, "learning_rate": 5.6000000000000006e-05, "loss": 0.07558286190032959, "memory/device_reserved (GiB)": 40.81, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.07851, "step": 15, "tokens/total": 504240, "tokens/train_per_sec_per_gpu": 24.64, "tokens/trainable": 6820} +{"epoch": 0.0625, "grad_norm": 2.311145782470703, "learning_rate": 6e-05, "loss": 0.08445745706558228, "memory/device_reserved (GiB)": 40.81, "memory/max_active (GiB)": 36.56, "memory/max_allocated (GiB)": 36.56, "ppl": 1.08813, "step": 16, "tokens/total": 536816, "tokens/train_per_sec_per_gpu": 25.19, "tokens/trainable": 7258} +{"epoch": 0.06640625, "grad_norm": 1.8095303773880005, "learning_rate": 6.400000000000001e-05, "loss": 0.06333113461732864, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.06538, "step": 17, "tokens/total": 572672, "tokens/train_per_sec_per_gpu": 26.34, "tokens/trainable": 7710} +{"epoch": 0.0703125, "grad_norm": 0.9293481111526489, "learning_rate": 6.800000000000001e-05, "loss": 0.049193140119314194, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.05042, "step": 18, "tokens/total": 609008, "tokens/train_per_sec_per_gpu": 23.05, "tokens/trainable": 8174} +{"epoch": 0.07421875, "grad_norm": 1.2340449094772339, "learning_rate": 7.2e-05, "loss": 0.041213903576135635, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.32, "memory/max_allocated (GiB)": 38.32, "ppl": 1.04207, "step": 19, "tokens/total": 645376, "tokens/train_per_sec_per_gpu": 24.96, "tokens/trainable": 8617} +{"epoch": 0.078125, "grad_norm": 1.6301466226577759, "learning_rate": 7.6e-05, "loss": 0.04662671685218811, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.04773, "step": 20, "tokens/total": 679680, "tokens/train_per_sec_per_gpu": 23.66, "tokens/trainable": 9037} +{"epoch": 0.08203125, "grad_norm": 1.1327530145645142, "learning_rate": 8e-05, "loss": 0.03275035694241524, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.3, "memory/max_allocated (GiB)": 36.3, "ppl": 1.03329, "step": 21, "tokens/total": 711056, "tokens/train_per_sec_per_gpu": 31.44, "tokens/trainable": 9503} +{"epoch": 0.0859375, "grad_norm": 1.355506181716919, "learning_rate": 8.4e-05, "loss": 0.04126371443271637, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 38.8, "memory/max_allocated (GiB)": 38.8, "ppl": 1.04213, "step": 22, "tokens/total": 746048, "tokens/train_per_sec_per_gpu": 28.03, "tokens/trainable": 9972} +{"epoch": 0.08984375, "grad_norm": 1.214060664176941, "learning_rate": 8.800000000000001e-05, "loss": 0.047878436744213104, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 1.04904, "step": 23, "tokens/total": 780240, "tokens/train_per_sec_per_gpu": 29.57, "tokens/trainable": 10447} +{"epoch": 0.09375, "grad_norm": 1.8399527072906494, "learning_rate": 9.200000000000001e-05, "loss": 0.026854675263166428, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.5, "memory/max_allocated (GiB)": 37.5, "ppl": 1.02722, "step": 24, "tokens/total": 811344, "tokens/train_per_sec_per_gpu": 32.17, "tokens/trainable": 10899} +{"epoch": 0.09765625, "grad_norm": 1.639219045639038, "learning_rate": 9.6e-05, "loss": 0.0583750419318676, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.66, "memory/max_allocated (GiB)": 36.66, "ppl": 1.06011, "step": 25, "tokens/total": 843728, "tokens/train_per_sec_per_gpu": 27.42, "tokens/trainable": 11360} +{"epoch": 0.1015625, "grad_norm": 1.053159475326538, "learning_rate": 0.0001, "loss": 0.028117237612605095, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.02852, "step": 26, "tokens/total": 876560, "tokens/train_per_sec_per_gpu": 24.13, "tokens/trainable": 11775} +{"epoch": 0.10546875, "grad_norm": 0.9342462420463562, "learning_rate": 9.99990636831587e-05, "loss": 0.027302058413624763, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.02768, "step": 27, "tokens/total": 909488, "tokens/train_per_sec_per_gpu": 30.21, "tokens/trainable": 12242} +{"epoch": 0.109375, "grad_norm": 12.965938568115234, "learning_rate": 9.999625477159879e-05, "loss": 0.023457400500774384, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.02373, "step": 28, "tokens/total": 947840, "tokens/train_per_sec_per_gpu": 22.75, "tokens/trainable": 12696} +{"epoch": 0.11328125, "grad_norm": 0.4355120062828064, "learning_rate": 9.999157338221051e-05, "loss": 0.02042241394519806, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.02063, "step": 29, "tokens/total": 982544, "tokens/train_per_sec_per_gpu": 26.29, "tokens/trainable": 13153} +{"epoch": 0.1171875, "grad_norm": 0.5377391576766968, "learning_rate": 9.998501970980562e-05, "loss": 0.013641612604260445, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.01374, "step": 30, "tokens/total": 1014144, "tokens/train_per_sec_per_gpu": 31.49, "tokens/trainable": 13624} +{"epoch": 0.12109375, "grad_norm": 0.4864690601825714, "learning_rate": 9.997659402710915e-05, "loss": 0.013623879291117191, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.01372, "step": 31, "tokens/total": 1048976, "tokens/train_per_sec_per_gpu": 26.2, "tokens/trainable": 14064} +{"epoch": 0.125, "grad_norm": 0.689425528049469, "learning_rate": 9.996629668474818e-05, "loss": 0.028010524809360504, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.02841, "step": 32, "tokens/total": 1083728, "tokens/train_per_sec_per_gpu": 29.76, "tokens/trainable": 14565} +{"epoch": 0.12890625, "grad_norm": 0.6091338396072388, "learning_rate": 9.995412811123711e-05, "loss": 0.01871326006948948, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 38.97, "memory/max_allocated (GiB)": 38.97, "ppl": 1.01889, "step": 33, "tokens/total": 1119664, "tokens/train_per_sec_per_gpu": 24.93, "tokens/trainable": 15005} +{"epoch": 0.1328125, "grad_norm": 0.9199369549751282, "learning_rate": 9.994008881295999e-05, "loss": 0.022215571254491806, "memory/device_reserved (GiB)": 39.91, "memory/max_active (GiB)": 36.87, "memory/max_allocated (GiB)": 36.87, "ppl": 1.02246, "step": 34, "tokens/total": 1152768, "tokens/train_per_sec_per_gpu": 25.72, "tokens/trainable": 15459} +{"epoch": 0.13671875, "grad_norm": 1.377005934715271, "learning_rate": 9.992417937414932e-05, "loss": 0.02991057187318802, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 1.03036, "step": 35, "tokens/total": 1186736, "tokens/train_per_sec_per_gpu": 30.03, "tokens/trainable": 15960} +{"epoch": 0.140625, "grad_norm": 0.7428305745124817, "learning_rate": 9.99064004568618e-05, "loss": 0.01404334232211113, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.01414, "step": 36, "tokens/total": 1222352, "tokens/train_per_sec_per_gpu": 25.8, "tokens/trainable": 16428} +{"epoch": 0.14453125, "grad_norm": 1.1986535787582397, "learning_rate": 9.988675280095074e-05, "loss": 0.03889606148004532, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.03966, "step": 37, "tokens/total": 1256336, "tokens/train_per_sec_per_gpu": 25.08, "tokens/trainable": 16884} +{"epoch": 0.1484375, "grad_norm": 1.3412666320800781, "learning_rate": 9.986523722403528e-05, "loss": 0.02619781717658043, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.02654, "step": 38, "tokens/total": 1288304, "tokens/train_per_sec_per_gpu": 26.82, "tokens/trainable": 17341} +{"epoch": 0.15234375, "grad_norm": 0.8971475958824158, "learning_rate": 9.984185462146642e-05, "loss": 0.03421493619680405, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.41, "memory/max_allocated (GiB)": 37.41, "ppl": 1.03481, "step": 39, "tokens/total": 1323440, "tokens/train_per_sec_per_gpu": 25.77, "tokens/trainable": 17786} +{"epoch": 0.15625, "grad_norm": 1.7986299991607666, "learning_rate": 9.98166059662897e-05, "loss": 0.022111834958195686, "memory/device_reserved (GiB)": 40.63, "memory/max_active (GiB)": 39.09, "memory/max_allocated (GiB)": 39.09, "ppl": 1.02236, "step": 40, "tokens/total": 1358912, "tokens/train_per_sec_per_gpu": 26.37, "tokens/trainable": 18262} +{"epoch": 0.16015625, "grad_norm": 0.817679762840271, "learning_rate": 9.978949230920472e-05, "loss": 0.022632014006376266, "memory/device_reserved (GiB)": 40.63, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.02289, "step": 41, "tokens/total": 1393600, "tokens/train_per_sec_per_gpu": 25.27, "tokens/trainable": 18716} +{"epoch": 0.1640625, "grad_norm": 1.1296051740646362, "learning_rate": 9.976051477852141e-05, "loss": 0.05311407893896103, "memory/device_reserved (GiB)": 40.63, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.05455, "step": 42, "tokens/total": 1426832, "tokens/train_per_sec_per_gpu": 26.54, "tokens/trainable": 19157} +{"epoch": 0.16796875, "grad_norm": 0.943610429763794, "learning_rate": 9.972967458011312e-05, "loss": 0.030429523438215256, "memory/device_reserved (GiB)": 40.63, "memory/max_active (GiB)": 38.91, "memory/max_allocated (GiB)": 38.91, "ppl": 1.0309, "step": 43, "tokens/total": 1462192, "tokens/train_per_sec_per_gpu": 27.63, "tokens/trainable": 19647} +{"epoch": 0.171875, "grad_norm": 1.626641869544983, "learning_rate": 9.96969729973664e-05, "loss": 0.038068704307079315, "memory/device_reserved (GiB)": 41.66, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.0388, "step": 44, "tokens/total": 1501056, "tokens/train_per_sec_per_gpu": 23.71, "tokens/trainable": 20119} +{"epoch": 0.17578125, "grad_norm": 0.30383962392807007, "learning_rate": 9.966241139112754e-05, "loss": 0.010471785441040993, "memory/device_reserved (GiB)": 41.66, "memory/max_active (GiB)": 38.1, "memory/max_allocated (GiB)": 38.1, "ppl": 1.01053, "step": 45, "tokens/total": 1534752, "tokens/train_per_sec_per_gpu": 25.92, "tokens/trainable": 20560} +{"epoch": 0.1796875, "grad_norm": 0.6610947847366333, "learning_rate": 9.96259911996461e-05, "loss": 0.02416994608938694, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.02446, "step": 46, "tokens/total": 1568912, "tokens/train_per_sec_per_gpu": 25.4, "tokens/trainable": 20992} +{"epoch": 0.18359375, "grad_norm": 0.9658553004264832, "learning_rate": 9.958771393851491e-05, "loss": 0.018943829461932182, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 36.49, "memory/max_allocated (GiB)": 36.49, "ppl": 1.01912, "step": 47, "tokens/total": 1600368, "tokens/train_per_sec_per_gpu": 25.1, "tokens/trainable": 21441} +{"epoch": 0.1875, "grad_norm": 1.2225791215896606, "learning_rate": 9.954758120060702e-05, "loss": 0.03917454183101654, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.03995, "step": 48, "tokens/total": 1636640, "tokens/train_per_sec_per_gpu": 24.61, "tokens/trainable": 21901} +{"epoch": 0.19140625, "grad_norm": 0.4465513825416565, "learning_rate": 9.950559465600948e-05, "loss": 0.006239047273993492, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 1.00626, "step": 49, "tokens/total": 1670896, "tokens/train_per_sec_per_gpu": 26.33, "tokens/trainable": 22341} +{"epoch": 0.1953125, "grad_norm": 0.9192044138908386, "learning_rate": 9.946175605195379e-05, "loss": 0.012428226880729198, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.01251, "step": 50, "tokens/total": 1710448, "tokens/train_per_sec_per_gpu": 21.92, "tokens/trainable": 22833} +{"epoch": 0.19921875, "grad_norm": 0.45254629850387573, "learning_rate": 9.941606721274322e-05, "loss": 0.005117443855851889, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.00513, "step": 51, "tokens/total": 1744656, "tokens/train_per_sec_per_gpu": 23.97, "tokens/trainable": 23277} +{"epoch": 0.203125, "grad_norm": 0.8434980511665344, "learning_rate": 9.936853003967685e-05, "loss": 0.0191053356975317, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.01929, "step": 52, "tokens/total": 1781888, "tokens/train_per_sec_per_gpu": 25.42, "tokens/trainable": 23759} +{"epoch": 0.20703125, "grad_norm": 0.3007189929485321, "learning_rate": 9.93191465109705e-05, "loss": 0.0026727027725428343, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.00268, "step": 53, "tokens/total": 1818528, "tokens/train_per_sec_per_gpu": 23.89, "tokens/trainable": 24193} +{"epoch": 0.2109375, "grad_norm": 1.086889386177063, "learning_rate": 9.926791868167438e-05, "loss": 0.023409627377986908, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.02369, "step": 54, "tokens/total": 1855024, "tokens/train_per_sec_per_gpu": 24.14, "tokens/trainable": 24619} +{"epoch": 0.21484375, "grad_norm": 0.9475339651107788, "learning_rate": 9.921484868358753e-05, "loss": 0.028936194255948067, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 1.02936, "step": 55, "tokens/total": 1891040, "tokens/train_per_sec_per_gpu": 24.37, "tokens/trainable": 25075} +{"epoch": 0.21875, "grad_norm": 1.214247226715088, "learning_rate": 9.915993872516924e-05, "loss": 0.03239012137055397, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.03292, "step": 56, "tokens/total": 1926864, "tokens/train_per_sec_per_gpu": 23.16, "tokens/trainable": 25519} +{"epoch": 0.22265625, "grad_norm": 0.6152435541152954, "learning_rate": 9.9103191091447e-05, "loss": 0.006466131191700697, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00649, "step": 57, "tokens/total": 1958928, "tokens/train_per_sec_per_gpu": 27.39, "tokens/trainable": 25966} +{"epoch": 0.2265625, "grad_norm": 0.7201495170593262, "learning_rate": 9.904460814392147e-05, "loss": 0.011080694384872913, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.01114, "step": 58, "tokens/total": 1998304, "tokens/train_per_sec_per_gpu": 22.61, "tokens/trainable": 26423} +{"epoch": 0.23046875, "grad_norm": 0.4773459732532501, "learning_rate": 9.898419232046825e-05, "loss": 0.01458004955202341, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.01469, "step": 59, "tokens/total": 2035008, "tokens/train_per_sec_per_gpu": 28.01, "tokens/trainable": 26934} +{"epoch": 0.234375, "grad_norm": 0.6408302187919617, "learning_rate": 9.892194613523633e-05, "loss": 0.009850872680544853, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.26, "memory/max_allocated (GiB)": 38.26, "ppl": 1.0099, "step": 60, "tokens/total": 2070864, "tokens/train_per_sec_per_gpu": 24.27, "tokens/trainable": 27393} +{"epoch": 0.23828125, "grad_norm": 0.33885788917541504, "learning_rate": 9.885787217854357e-05, "loss": 0.009225481189787388, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.00927, "step": 61, "tokens/total": 2104448, "tokens/train_per_sec_per_gpu": 26.44, "tokens/trainable": 27852} +{"epoch": 0.2421875, "grad_norm": 0.5601316094398499, "learning_rate": 9.879197311676887e-05, "loss": 0.016548508778214455, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.01669, "step": 62, "tokens/total": 2140496, "tokens/train_per_sec_per_gpu": 24.24, "tokens/trainable": 28292} +{"epoch": 0.24609375, "grad_norm": 1.2000150680541992, "learning_rate": 9.872425169224113e-05, "loss": 0.02867770567536354, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.02909, "step": 63, "tokens/total": 2174288, "tokens/train_per_sec_per_gpu": 29.75, "tokens/trainable": 28798} +{"epoch": 0.25, "grad_norm": 0.9483950138092041, "learning_rate": 9.865471072312528e-05, "loss": 0.03319031000137329, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.14, "memory/max_allocated (GiB)": 38.14, "ppl": 1.03375, "step": 64, "tokens/total": 2210752, "tokens/train_per_sec_per_gpu": 25.93, "tokens/trainable": 29283} +{"epoch": 0.25390625, "grad_norm": 0.5156707763671875, "learning_rate": 9.858335310330492e-05, "loss": 0.01069432683289051, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.01075, "step": 65, "tokens/total": 2242928, "tokens/train_per_sec_per_gpu": 26.47, "tokens/trainable": 29745} +{"epoch": 0.2578125, "grad_norm": 0.5657427906990051, "learning_rate": 9.851018180226185e-05, "loss": 0.019436176866292953, "memory/device_reserved (GiB)": 40.31, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.01963, "step": 66, "tokens/total": 2281472, "tokens/train_per_sec_per_gpu": 20.93, "tokens/trainable": 30186} +{"epoch": 0.26171875, "grad_norm": 0.3744322657585144, "learning_rate": 9.843519986495259e-05, "loss": 0.011856304481625557, "memory/device_reserved (GiB)": 41.08, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.01193, "step": 67, "tokens/total": 2316208, "tokens/train_per_sec_per_gpu": 25.5, "tokens/trainable": 30622} +{"epoch": 0.265625, "grad_norm": 0.4973542094230652, "learning_rate": 9.835841041168162e-05, "loss": 0.01173720695078373, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.01181, "step": 68, "tokens/total": 2353344, "tokens/train_per_sec_per_gpu": 25.49, "tokens/trainable": 31097} +{"epoch": 0.26953125, "grad_norm": 0.6197847127914429, "learning_rate": 9.82798166379715e-05, "loss": 0.015285029076039791, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.0154, "step": 69, "tokens/total": 2386704, "tokens/train_per_sec_per_gpu": 29.53, "tokens/trainable": 31595} +{"epoch": 0.2734375, "grad_norm": 0.21231068670749664, "learning_rate": 9.819942181443002e-05, "loss": 0.002688200678676367, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.00269, "step": 70, "tokens/total": 2423984, "tokens/train_per_sec_per_gpu": 21.86, "tokens/trainable": 32036} +{"epoch": 0.27734375, "grad_norm": 0.29338762164115906, "learning_rate": 9.811722928661392e-05, "loss": 0.004150751978158951, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 35.81, "memory/max_allocated (GiB)": 35.81, "ppl": 1.00416, "step": 71, "tokens/total": 2452912, "tokens/train_per_sec_per_gpu": 26.53, "tokens/trainable": 32428} +{"epoch": 0.28125, "grad_norm": 0.40113508701324463, "learning_rate": 9.803324247488975e-05, "loss": 0.00247983168810606, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00248, "step": 72, "tokens/total": 2488576, "tokens/train_per_sec_per_gpu": 24.04, "tokens/trainable": 32880} +{"epoch": 0.28515625, "grad_norm": 0.1171368807554245, "learning_rate": 9.794746487429161e-05, "loss": 0.0006996856536716223, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 36.88, "memory/max_allocated (GiB)": 36.88, "ppl": 1.0007, "step": 73, "tokens/total": 2521520, "tokens/train_per_sec_per_gpu": 27.17, "tokens/trainable": 33323} +{"epoch": 0.2890625, "grad_norm": 0.1892620027065277, "learning_rate": 9.785990005437554e-05, "loss": 0.0008091474883258343, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00081, "step": 74, "tokens/total": 2557392, "tokens/train_per_sec_per_gpu": 26.25, "tokens/trainable": 33792} +{"epoch": 0.29296875, "grad_norm": 0.7665731906890869, "learning_rate": 9.777055165907117e-05, "loss": 0.032626304775476456, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.03316, "step": 75, "tokens/total": 2589920, "tokens/train_per_sec_per_gpu": 28.99, "tokens/trainable": 34223} +{"epoch": 0.296875, "grad_norm": 2.6799347400665283, "learning_rate": 9.767942340652993e-05, "loss": 0.010829668492078781, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.07, "memory/max_allocated (GiB)": 38.07, "ppl": 1.01089, "step": 76, "tokens/total": 2624784, "tokens/train_per_sec_per_gpu": 22.54, "tokens/trainable": 34649} +{"epoch": 0.30078125, "grad_norm": 1.193192481994629, "learning_rate": 9.758651908897035e-05, "loss": 0.03277236968278885, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.03332, "step": 77, "tokens/total": 2658176, "tokens/train_per_sec_per_gpu": 28.23, "tokens/trainable": 35094} +{"epoch": 0.3046875, "grad_norm": 0.9529989361763, "learning_rate": 9.749184257252033e-05, "loss": 0.03973307088017464, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.04053, "step": 78, "tokens/total": 2692448, "tokens/train_per_sec_per_gpu": 24.72, "tokens/trainable": 35534} +{"epoch": 0.30859375, "grad_norm": 1.741410255432129, "learning_rate": 9.739539779705614e-05, "loss": 0.016716163605451584, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.01686, "step": 79, "tokens/total": 2730720, "tokens/train_per_sec_per_gpu": 24.7, "tokens/trainable": 36029} +{"epoch": 0.3125, "grad_norm": 0.4255867302417755, "learning_rate": 9.729718877603861e-05, "loss": 0.011636724695563316, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.0117, "step": 80, "tokens/total": 2764512, "tokens/train_per_sec_per_gpu": 27.94, "tokens/trainable": 36469} +{"epoch": 0.31640625, "grad_norm": 1.7772278785705566, "learning_rate": 9.719721959634592e-05, "loss": 0.03325195610523224, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.03381, "step": 81, "tokens/total": 2799728, "tokens/train_per_sec_per_gpu": 23.42, "tokens/trainable": 36906} +{"epoch": 0.3203125, "grad_norm": 0.8314163088798523, "learning_rate": 9.709549441810375e-05, "loss": 0.009634598158299923, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.55, "memory/max_allocated (GiB)": 39.55, "ppl": 1.00968, "step": 82, "tokens/total": 2837344, "tokens/train_per_sec_per_gpu": 27.16, "tokens/trainable": 37390} +{"epoch": 0.32421875, "grad_norm": 0.4774115979671478, "learning_rate": 9.699201747451195e-05, "loss": 0.00784055795520544, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.00787, "step": 83, "tokens/total": 2869936, "tokens/train_per_sec_per_gpu": 25.23, "tokens/trainable": 37838} +{"epoch": 0.328125, "grad_norm": 0.11107324063777924, "learning_rate": 9.688679307166854e-05, "loss": 0.0031447941437363625, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00315, "step": 84, "tokens/total": 2902736, "tokens/train_per_sec_per_gpu": 28.56, "tokens/trainable": 38310} +{"epoch": 0.33203125, "grad_norm": 0.4888429641723633, "learning_rate": 9.677982558839042e-05, "loss": 0.012560717761516571, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.01264, "step": 85, "tokens/total": 2936032, "tokens/train_per_sec_per_gpu": 27.32, "tokens/trainable": 38789} +{"epoch": 0.3359375, "grad_norm": 0.7470205426216125, "learning_rate": 9.66711194760312e-05, "loss": 0.01914072595536709, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.01933, "step": 86, "tokens/total": 2973440, "tokens/train_per_sec_per_gpu": 25.75, "tokens/trainable": 39277} +{"epoch": 0.33984375, "grad_norm": 0.3890836536884308, "learning_rate": 9.656067925829593e-05, "loss": 0.007584001403301954, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.00761, "step": 87, "tokens/total": 3004192, "tokens/train_per_sec_per_gpu": 31.53, "tokens/trainable": 39737} +{"epoch": 0.34375, "grad_norm": 0.398070365190506, "learning_rate": 9.644850953105288e-05, "loss": 0.00754000386223197, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 35.78, "memory/max_allocated (GiB)": 35.78, "ppl": 1.00757, "step": 88, "tokens/total": 3034544, "tokens/train_per_sec_per_gpu": 26.42, "tokens/trainable": 40179} +{"epoch": 0.34765625, "grad_norm": 2.033498764038086, "learning_rate": 9.633461496214225e-05, "loss": 0.014149690978229046, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.01425, "step": 89, "tokens/total": 3067248, "tokens/train_per_sec_per_gpu": 26.75, "tokens/trainable": 40649} +{"epoch": 0.3515625, "grad_norm": 0.9219241142272949, "learning_rate": 9.621900029118195e-05, "loss": 0.03684563189744949, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.04, "memory/max_allocated (GiB)": 36.04, "ppl": 1.03753, "step": 90, "tokens/total": 3096928, "tokens/train_per_sec_per_gpu": 26.9, "tokens/trainable": 41080} +{"epoch": 0.35546875, "grad_norm": 0.38672399520874023, "learning_rate": 9.610167032937036e-05, "loss": 0.009744911454617977, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00979, "step": 91, "tokens/total": 3135216, "tokens/train_per_sec_per_gpu": 26.35, "tokens/trainable": 41599} +{"epoch": 0.359375, "grad_norm": 0.374089777469635, "learning_rate": 9.598262995928611e-05, "loss": 0.009398073889315128, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.00944, "step": 92, "tokens/total": 3166416, "tokens/train_per_sec_per_gpu": 31.46, "tokens/trainable": 42076} +{"epoch": 0.36328125, "grad_norm": 0.1652466505765915, "learning_rate": 9.586188413468492e-05, "loss": 0.0031671649776399136, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00317, "step": 93, "tokens/total": 3204608, "tokens/train_per_sec_per_gpu": 23.13, "tokens/trainable": 42522} +{"epoch": 0.3671875, "grad_norm": 0.6722483038902283, "learning_rate": 9.57394378802934e-05, "loss": 0.010353108868002892, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.01041, "step": 94, "tokens/total": 3237920, "tokens/train_per_sec_per_gpu": 27.74, "tokens/trainable": 42974} +{"epoch": 0.37109375, "grad_norm": 1.3180787563323975, "learning_rate": 9.56152962916e-05, "loss": 0.03504617512226105, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.26, "memory/max_allocated (GiB)": 36.26, "ppl": 1.03567, "step": 95, "tokens/total": 3268592, "tokens/train_per_sec_per_gpu": 26.81, "tokens/trainable": 43380} +{"epoch": 0.375, "grad_norm": 0.2105441689491272, "learning_rate": 9.548946453464296e-05, "loss": 0.004682785365730524, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.00469, "step": 96, "tokens/total": 3302688, "tokens/train_per_sec_per_gpu": 27.12, "tokens/trainable": 43865} +{"epoch": 0.37890625, "grad_norm": 0.9798703789710999, "learning_rate": 9.53619478457953e-05, "loss": 0.010969950817525387, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.01103, "step": 97, "tokens/total": 3337584, "tokens/train_per_sec_per_gpu": 24.25, "tokens/trainable": 44328} +{"epoch": 0.3828125, "grad_norm": 0.2174561470746994, "learning_rate": 9.523275153154695e-05, "loss": 0.0034194509498775005, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.00343, "step": 98, "tokens/total": 3369984, "tokens/train_per_sec_per_gpu": 25.46, "tokens/trainable": 44778} +{"epoch": 0.38671875, "grad_norm": 0.7424131631851196, "learning_rate": 9.51018809682839e-05, "loss": 0.03991135209798813, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.04072, "step": 99, "tokens/total": 3407648, "tokens/train_per_sec_per_gpu": 25.88, "tokens/trainable": 45225} +{"epoch": 0.390625, "grad_norm": 0.48007991909980774, "learning_rate": 9.49693416020645e-05, "loss": 0.009669644758105278, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.53, "memory/max_allocated (GiB)": 38.53, "ppl": 1.00972, "step": 100, "tokens/total": 3445392, "tokens/train_per_sec_per_gpu": 24.78, "tokens/trainable": 45678} +{"epoch": 0.39453125, "grad_norm": 0.3592854142189026, "learning_rate": 9.483513894839276e-05, "loss": 0.007871917448937893, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.0079, "step": 101, "tokens/total": 3479440, "tokens/train_per_sec_per_gpu": 25.55, "tokens/trainable": 46125} +{"epoch": 0.3984375, "grad_norm": 0.3113628923892975, "learning_rate": 9.469927859198888e-05, "loss": 0.006901334971189499, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00693, "step": 102, "tokens/total": 3515360, "tokens/train_per_sec_per_gpu": 29.37, "tokens/trainable": 46612} +{"epoch": 0.40234375, "grad_norm": 0.7623510360717773, "learning_rate": 9.456176618655689e-05, "loss": 0.018834352493286133, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.47, "memory/max_allocated (GiB)": 36.47, "ppl": 1.01901, "step": 103, "tokens/total": 3547408, "tokens/train_per_sec_per_gpu": 27.75, "tokens/trainable": 47059} +{"epoch": 0.40625, "grad_norm": 0.8035973310470581, "learning_rate": 9.442260745454927e-05, "loss": 0.009301794692873955, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00935, "step": 104, "tokens/total": 3582784, "tokens/train_per_sec_per_gpu": 26.12, "tokens/trainable": 47536} +{"epoch": 0.41015625, "grad_norm": 0.31189796328544617, "learning_rate": 9.428180818692884e-05, "loss": 0.007709788624197245, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.45, "memory/max_allocated (GiB)": 38.45, "ppl": 1.00774, "step": 105, "tokens/total": 3618944, "tokens/train_per_sec_per_gpu": 24.97, "tokens/trainable": 48037} +{"epoch": 0.4140625, "grad_norm": 0.40750229358673096, "learning_rate": 9.413937424292791e-05, "loss": 0.010118984617292881, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.01017, "step": 106, "tokens/total": 3658080, "tokens/train_per_sec_per_gpu": 21.76, "tokens/trainable": 48491} +{"epoch": 0.41796875, "grad_norm": 0.1519603729248047, "learning_rate": 9.399531154980424e-05, "loss": 0.0031692753545939922, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 1.00317, "step": 107, "tokens/total": 3694656, "tokens/train_per_sec_per_gpu": 24.32, "tokens/trainable": 48940} +{"epoch": 0.421875, "grad_norm": 0.35333001613616943, "learning_rate": 9.384962610259455e-05, "loss": 0.012811357155442238, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.01289, "step": 108, "tokens/total": 3729312, "tokens/train_per_sec_per_gpu": 24.4, "tokens/trainable": 49389} +{"epoch": 0.42578125, "grad_norm": 0.4056496322154999, "learning_rate": 9.370232396386494e-05, "loss": 0.0037163598462939262, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.22, "memory/max_allocated (GiB)": 37.22, "ppl": 1.00372, "step": 109, "tokens/total": 3762400, "tokens/train_per_sec_per_gpu": 26.63, "tokens/trainable": 49838} +{"epoch": 0.4296875, "grad_norm": 1.2819931507110596, "learning_rate": 9.355341126345868e-05, "loss": 0.01873013749718666, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.01891, "step": 110, "tokens/total": 3797680, "tokens/train_per_sec_per_gpu": 25.62, "tokens/trainable": 50310} +{"epoch": 0.43359375, "grad_norm": 0.28486865758895874, "learning_rate": 9.340289419824107e-05, "loss": 0.004570053890347481, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.00458, "step": 111, "tokens/total": 3830000, "tokens/train_per_sec_per_gpu": 28.29, "tokens/trainable": 50755} +{"epoch": 0.4375, "grad_norm": 0.28946545720100403, "learning_rate": 9.325077903184159e-05, "loss": 0.005072574131190777, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00509, "step": 112, "tokens/total": 3864992, "tokens/train_per_sec_per_gpu": 24.09, "tokens/trainable": 51213} +{"epoch": 0.44140625, "grad_norm": 0.01891162618994713, "learning_rate": 9.30970720943932e-05, "loss": 0.0002804588293656707, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.15, "memory/max_allocated (GiB)": 38.15, "ppl": 1.00028, "step": 113, "tokens/total": 3900000, "tokens/train_per_sec_per_gpu": 25.62, "tokens/trainable": 51660} +{"epoch": 0.4453125, "grad_norm": 1.2115888595581055, "learning_rate": 9.2941779782269e-05, "loss": 0.004306407645344734, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00432, "step": 114, "tokens/total": 3933600, "tokens/train_per_sec_per_gpu": 25.7, "tokens/trainable": 52133} +{"epoch": 0.44921875, "grad_norm": 0.6671693325042725, "learning_rate": 9.278490855781596e-05, "loss": 0.007251053582876921, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.00728, "step": 115, "tokens/total": 3966032, "tokens/train_per_sec_per_gpu": 31.17, "tokens/trainable": 52580} +{"epoch": 0.453125, "grad_norm": 1.2025994062423706, "learning_rate": 9.262646494908604e-05, "loss": 0.01422079000622034, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.01432, "step": 116, "tokens/total": 4004688, "tokens/train_per_sec_per_gpu": 23.74, "tokens/trainable": 53033} +{"epoch": 0.45703125, "grad_norm": 0.8034558296203613, "learning_rate": 9.246645554956457e-05, "loss": 0.028370855376124382, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.02878, "step": 117, "tokens/total": 4039712, "tokens/train_per_sec_per_gpu": 26.88, "tokens/trainable": 53517} +{"epoch": 0.4609375, "grad_norm": 0.4222593605518341, "learning_rate": 9.230488701789578e-05, "loss": 0.010232735425233841, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.01029, "step": 118, "tokens/total": 4076768, "tokens/train_per_sec_per_gpu": 24.51, "tokens/trainable": 53967} +{"epoch": 0.46484375, "grad_norm": 0.23738738894462585, "learning_rate": 9.214176607760577e-05, "loss": 0.0023563168942928314, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.00236, "step": 119, "tokens/total": 4115024, "tokens/train_per_sec_per_gpu": 24.2, "tokens/trainable": 54430} +{"epoch": 0.46875, "grad_norm": 0.7422047853469849, "learning_rate": 9.197709951682268e-05, "loss": 0.016030780971050262, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.01616, "step": 120, "tokens/total": 4147456, "tokens/train_per_sec_per_gpu": 29.14, "tokens/trainable": 54896} +{"epoch": 0.47265625, "grad_norm": 3.3230366706848145, "learning_rate": 9.181089418799428e-05, "loss": 0.020484138280153275, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.0207, "step": 121, "tokens/total": 4183920, "tokens/train_per_sec_per_gpu": 27.39, "tokens/trainable": 55379} +{"epoch": 0.4765625, "grad_norm": 0.8344050049781799, "learning_rate": 9.164315700760271e-05, "loss": 0.03136826679110527, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.03187, "step": 122, "tokens/total": 4219600, "tokens/train_per_sec_per_gpu": 23.39, "tokens/trainable": 55803} +{"epoch": 0.48046875, "grad_norm": 0.5388036966323853, "learning_rate": 9.147389495587671e-05, "loss": 0.02347150817513466, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.02375, "step": 123, "tokens/total": 4251008, "tokens/train_per_sec_per_gpu": 31.04, "tokens/trainable": 56249} +{"epoch": 0.484375, "grad_norm": 0.33890578150749207, "learning_rate": 9.130311507650116e-05, "loss": 0.011705836281180382, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.01177, "step": 124, "tokens/total": 4283392, "tokens/train_per_sec_per_gpu": 27.0, "tokens/trainable": 56713} +{"epoch": 0.48828125, "grad_norm": 0.9970278143882751, "learning_rate": 9.113082447632394e-05, "loss": 0.02704804576933384, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.02742, "step": 125, "tokens/total": 4315456, "tokens/train_per_sec_per_gpu": 33.04, "tokens/trainable": 57196} +{"epoch": 0.4921875, "grad_norm": 0.6279972195625305, "learning_rate": 9.09570303250602e-05, "loss": 0.0190292801707983, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.01921, "step": 126, "tokens/total": 4347472, "tokens/train_per_sec_per_gpu": 30.53, "tokens/trainable": 57680} +{"epoch": 0.49609375, "grad_norm": 0.42716142535209656, "learning_rate": 9.078173985499394e-05, "loss": 0.013122424483299255, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 35.95, "memory/max_allocated (GiB)": 35.95, "ppl": 1.01321, "step": 127, "tokens/total": 4377872, "tokens/train_per_sec_per_gpu": 28.32, "tokens/trainable": 58110} +{"epoch": 0.5, "grad_norm": 0.40525057911872864, "learning_rate": 9.060496036067713e-05, "loss": 0.011689595878124237, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.01176, "step": 128, "tokens/total": 4415376, "tokens/train_per_sec_per_gpu": 22.43, "tokens/trainable": 58534} +{"epoch": 0.50390625, "grad_norm": 0.42092177271842957, "learning_rate": 9.042669919862615e-05, "loss": 0.01226846594363451, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.77, "memory/max_allocated (GiB)": 36.77, "ppl": 1.01234, "step": 129, "tokens/total": 4447264, "tokens/train_per_sec_per_gpu": 30.01, "tokens/trainable": 58998} +{"epoch": 0.5078125, "grad_norm": 0.9572564363479614, "learning_rate": 9.024696378701557e-05, "loss": 0.03289375081658363, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 39.48, "memory/max_allocated (GiB)": 39.48, "ppl": 1.03344, "step": 130, "tokens/total": 4482944, "tokens/train_per_sec_per_gpu": 23.78, "tokens/trainable": 59448} +{"epoch": 0.51171875, "grad_norm": 0.5147362947463989, "learning_rate": 9.006576160536948e-05, "loss": 0.018487676978111267, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.01866, "step": 131, "tokens/total": 4517888, "tokens/train_per_sec_per_gpu": 24.2, "tokens/trainable": 59906} +{"epoch": 0.515625, "grad_norm": 0.2831747233867645, "learning_rate": 8.988310019425035e-05, "loss": 0.010943909175693989, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.011, "step": 132, "tokens/total": 4555392, "tokens/train_per_sec_per_gpu": 24.08, "tokens/trainable": 60350} +{"epoch": 0.51953125, "grad_norm": 0.4981938302516937, "learning_rate": 8.969898715494506e-05, "loss": 0.014243930578231812, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.01435, "step": 133, "tokens/total": 4592528, "tokens/train_per_sec_per_gpu": 25.61, "tokens/trainable": 60831} +{"epoch": 0.5234375, "grad_norm": 0.6867162585258484, "learning_rate": 8.951343014914869e-05, "loss": 0.01738608255982399, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.01754, "step": 134, "tokens/total": 4629104, "tokens/train_per_sec_per_gpu": 24.3, "tokens/trainable": 61305} +{"epoch": 0.52734375, "grad_norm": 0.11013702303171158, "learning_rate": 8.932643689864568e-05, "loss": 0.0022790462244302034, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00228, "step": 135, "tokens/total": 4665856, "tokens/train_per_sec_per_gpu": 27.18, "tokens/trainable": 61792} +{"epoch": 0.53125, "grad_norm": 0.24933631718158722, "learning_rate": 8.913801518498845e-05, "loss": 0.004422957543283701, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.00443, "step": 136, "tokens/total": 4704688, "tokens/train_per_sec_per_gpu": 21.25, "tokens/trainable": 62253} +{"epoch": 0.53515625, "grad_norm": 0.606407642364502, "learning_rate": 8.894817284917364e-05, "loss": 0.009540312923491001, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00959, "step": 137, "tokens/total": 4741472, "tokens/train_per_sec_per_gpu": 22.71, "tokens/trainable": 62707} +{"epoch": 0.5390625, "grad_norm": 0.5405151844024658, "learning_rate": 8.875691779131569e-05, "loss": 0.0232931487262249, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.02357, "step": 138, "tokens/total": 4778752, "tokens/train_per_sec_per_gpu": 20.67, "tokens/trainable": 63145} +{"epoch": 0.54296875, "grad_norm": 0.8502752184867859, "learning_rate": 8.856425797031829e-05, "loss": 0.015798063948750496, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.01592, "step": 139, "tokens/total": 4813904, "tokens/train_per_sec_per_gpu": 25.7, "tokens/trainable": 63587} +{"epoch": 0.546875, "grad_norm": 2.8911962509155273, "learning_rate": 8.837020140354295e-05, "loss": 0.011839132755994797, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.01191, "step": 140, "tokens/total": 4850272, "tokens/train_per_sec_per_gpu": 26.73, "tokens/trainable": 64052} +{"epoch": 0.55078125, "grad_norm": 0.3788418769836426, "learning_rate": 8.817475616647554e-05, "loss": 0.004456748254597187, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.00447, "step": 141, "tokens/total": 4884928, "tokens/train_per_sec_per_gpu": 26.44, "tokens/trainable": 64526} +{"epoch": 0.5546875, "grad_norm": 0.43420442938804626, "learning_rate": 8.797793039239017e-05, "loss": 0.0029407169204205275, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00295, "step": 142, "tokens/total": 4919984, "tokens/train_per_sec_per_gpu": 26.87, "tokens/trainable": 64983} +{"epoch": 0.55859375, "grad_norm": 1.4632354974746704, "learning_rate": 8.777973227201069e-05, "loss": 0.02942195162177086, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.02986, "step": 143, "tokens/total": 4956304, "tokens/train_per_sec_per_gpu": 27.72, "tokens/trainable": 65492} +{"epoch": 0.5625, "grad_norm": 1.5754191875457764, "learning_rate": 8.758017005316988e-05, "loss": 0.0351857990026474, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.03581, "step": 144, "tokens/total": 4991696, "tokens/train_per_sec_per_gpu": 26.52, "tokens/trainable": 65925} +{"epoch": 0.56640625, "grad_norm": 0.6058143973350525, "learning_rate": 8.737925204046629e-05, "loss": 0.008586310781538486, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00862, "step": 145, "tokens/total": 5026480, "tokens/train_per_sec_per_gpu": 23.91, "tokens/trainable": 66383} +{"epoch": 0.5703125, "grad_norm": 0.6974156498908997, "learning_rate": 8.717698659491851e-05, "loss": 0.03106304071843624, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.03155, "step": 146, "tokens/total": 5063568, "tokens/train_per_sec_per_gpu": 22.3, "tokens/trainable": 66840} +{"epoch": 0.57421875, "grad_norm": 0.724898099899292, "learning_rate": 8.697338213361735e-05, "loss": 0.025135021656751633, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.02545, "step": 147, "tokens/total": 5099616, "tokens/train_per_sec_per_gpu": 23.3, "tokens/trainable": 67297} +{"epoch": 0.578125, "grad_norm": 0.1996416300535202, "learning_rate": 8.676844712937552e-05, "loss": 0.005890287458896637, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.39, "memory/max_allocated (GiB)": 39.39, "ppl": 1.00591, "step": 148, "tokens/total": 5139360, "tokens/train_per_sec_per_gpu": 24.71, "tokens/trainable": 67769} +{"epoch": 0.58203125, "grad_norm": 0.44387000799179077, "learning_rate": 8.656219011037509e-05, "loss": 0.00720255495980382, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00723, "step": 149, "tokens/total": 5176400, "tokens/train_per_sec_per_gpu": 23.42, "tokens/trainable": 68257} +{"epoch": 0.5859375, "grad_norm": 0.44310978055000305, "learning_rate": 8.63546196598125e-05, "loss": 0.016318991780281067, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.01645, "step": 150, "tokens/total": 5213632, "tokens/train_per_sec_per_gpu": 24.94, "tokens/trainable": 68706} +{"epoch": 0.58984375, "grad_norm": 0.3608627915382385, "learning_rate": 8.614574441554145e-05, "loss": 0.012151552364230156, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.13, "memory/max_allocated (GiB)": 37.13, "ppl": 1.01223, "step": 151, "tokens/total": 5246736, "tokens/train_per_sec_per_gpu": 27.17, "tokens/trainable": 69131} +{"epoch": 0.59375, "grad_norm": 0.685858964920044, "learning_rate": 8.593557306971349e-05, "loss": 0.017837297171354294, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.44, "memory/max_allocated (GiB)": 37.44, "ppl": 1.018, "step": 152, "tokens/total": 5279712, "tokens/train_per_sec_per_gpu": 25.8, "tokens/trainable": 69586} +{"epoch": 0.59765625, "grad_norm": 0.6671690940856934, "learning_rate": 8.572411436841618e-05, "loss": 0.01624695211648941, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.01638, "step": 153, "tokens/total": 5313424, "tokens/train_per_sec_per_gpu": 26.36, "tokens/trainable": 70061} +{"epoch": 0.6015625, "grad_norm": 0.39020705223083496, "learning_rate": 8.551137711130922e-05, "loss": 0.008701683953404427, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.2, "memory/max_allocated (GiB)": 38.2, "ppl": 1.00874, "step": 154, "tokens/total": 5344816, "tokens/train_per_sec_per_gpu": 23.71, "tokens/trainable": 70467} +{"epoch": 0.60546875, "grad_norm": 0.2275904417037964, "learning_rate": 8.529737015125824e-05, "loss": 0.007391311228275299, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00742, "step": 155, "tokens/total": 5379264, "tokens/train_per_sec_per_gpu": 25.63, "tokens/trainable": 70933} +{"epoch": 0.609375, "grad_norm": 0.6294845342636108, "learning_rate": 8.508210239396639e-05, "loss": 0.014884693548083305, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.015, "step": 156, "tokens/total": 5416832, "tokens/train_per_sec_per_gpu": 24.04, "tokens/trainable": 71382} +{"epoch": 0.61328125, "grad_norm": 0.18174508213996887, "learning_rate": 8.486558279760375e-05, "loss": 0.003586188890039921, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 36.07, "memory/max_allocated (GiB)": 36.07, "ppl": 1.00359, "step": 157, "tokens/total": 5447968, "tokens/train_per_sec_per_gpu": 26.47, "tokens/trainable": 71808} +{"epoch": 0.6171875, "grad_norm": 0.3750772178173065, "learning_rate": 8.464782037243449e-05, "loss": 0.025619907304644585, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.02595, "step": 158, "tokens/total": 5480576, "tokens/train_per_sec_per_gpu": 27.22, "tokens/trainable": 72249} +{"epoch": 0.62109375, "grad_norm": 0.967908501625061, "learning_rate": 8.442882418044202e-05, "loss": 0.01670711301267147, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.01685, "step": 159, "tokens/total": 5513200, "tokens/train_per_sec_per_gpu": 29.17, "tokens/trainable": 72726} +{"epoch": 0.625, "grad_norm": 0.3262515664100647, "learning_rate": 8.420860333495179e-05, "loss": 0.017449423670768738, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.0176, "step": 160, "tokens/total": 5547120, "tokens/train_per_sec_per_gpu": 25.6, "tokens/trainable": 73148} +{"epoch": 0.62890625, "grad_norm": 0.2074899822473526, "learning_rate": 8.398716700025208e-05, "loss": 0.005753820296376944, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.00577, "step": 161, "tokens/total": 5580336, "tokens/train_per_sec_per_gpu": 25.55, "tokens/trainable": 73600} +{"epoch": 0.6328125, "grad_norm": 0.8597735166549683, "learning_rate": 8.376452439121266e-05, "loss": 0.03334365785121918, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 35.89, "memory/max_allocated (GiB)": 35.89, "ppl": 1.03391, "step": 162, "tokens/total": 5610736, "tokens/train_per_sec_per_gpu": 31.76, "tokens/trainable": 74068} +{"epoch": 0.63671875, "grad_norm": 0.615422248840332, "learning_rate": 8.354068477290124e-05, "loss": 0.008889294229447842, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00893, "step": 163, "tokens/total": 5645040, "tokens/train_per_sec_per_gpu": 27.66, "tokens/trainable": 74527} +{"epoch": 0.640625, "grad_norm": 0.2859649956226349, "learning_rate": 8.331565746019807e-05, "loss": 0.005444570444524288, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.52, "memory/max_allocated (GiB)": 36.52, "ppl": 1.00546, "step": 164, "tokens/total": 5676944, "tokens/train_per_sec_per_gpu": 26.98, "tokens/trainable": 74992} +{"epoch": 0.64453125, "grad_norm": 2.087047815322876, "learning_rate": 8.308945181740812e-05, "loss": 0.01943492703139782, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.01963, "step": 165, "tokens/total": 5711728, "tokens/train_per_sec_per_gpu": 26.91, "tokens/trainable": 75442} +{"epoch": 0.6484375, "grad_norm": 0.5594419836997986, "learning_rate": 8.286207725787153e-05, "loss": 0.016395509243011475, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.01653, "step": 166, "tokens/total": 5748656, "tokens/train_per_sec_per_gpu": 22.39, "tokens/trainable": 75887} +{"epoch": 0.65234375, "grad_norm": 0.4099993407726288, "learning_rate": 8.263354324357182e-05, "loss": 0.012865329161286354, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.01295, "step": 167, "tokens/total": 5783376, "tokens/train_per_sec_per_gpu": 26.37, "tokens/trainable": 76331} +{"epoch": 0.65625, "grad_norm": 0.2083388864994049, "learning_rate": 8.240385928474219e-05, "loss": 0.01104188896715641, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.0111, "step": 168, "tokens/total": 5816512, "tokens/train_per_sec_per_gpu": 26.96, "tokens/trainable": 76745} +{"epoch": 0.66015625, "grad_norm": 0.25850632786750793, "learning_rate": 8.217303493946967e-05, "loss": 0.009730209596455097, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.00978, "step": 169, "tokens/total": 5850944, "tokens/train_per_sec_per_gpu": 26.18, "tokens/trainable": 77201} +{"epoch": 0.6640625, "grad_norm": 0.7040078043937683, "learning_rate": 8.194107981329746e-05, "loss": 0.020657125860452652, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.02087, "step": 170, "tokens/total": 5888912, "tokens/train_per_sec_per_gpu": 23.3, "tokens/trainable": 77655} +{"epoch": 0.66796875, "grad_norm": 0.13995765149593353, "learning_rate": 8.170800355882518e-05, "loss": 0.0036004481371492147, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.24, "memory/max_allocated (GiB)": 37.24, "ppl": 1.00361, "step": 171, "tokens/total": 5923136, "tokens/train_per_sec_per_gpu": 30.39, "tokens/trainable": 78122} +{"epoch": 0.671875, "grad_norm": 1.2165682315826416, "learning_rate": 8.147381587530713e-05, "loss": 0.028702836483716965, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.02912, "step": 172, "tokens/total": 5956944, "tokens/train_per_sec_per_gpu": 26.51, "tokens/trainable": 78576} +{"epoch": 0.67578125, "grad_norm": 0.21340833604335785, "learning_rate": 8.123852650824877e-05, "loss": 0.010088194161653519, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.01014, "step": 173, "tokens/total": 5990976, "tokens/train_per_sec_per_gpu": 28.41, "tokens/trainable": 79059} +{"epoch": 0.6796875, "grad_norm": 0.23103511333465576, "learning_rate": 8.100214524900103e-05, "loss": 0.007472559809684753, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.52, "memory/max_allocated (GiB)": 38.52, "ppl": 1.0075, "step": 174, "tokens/total": 6026368, "tokens/train_per_sec_per_gpu": 22.21, "tokens/trainable": 79498} +{"epoch": 0.68359375, "grad_norm": 0.1889306753873825, "learning_rate": 8.076468193435301e-05, "loss": 0.00990007072687149, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.00995, "step": 175, "tokens/total": 6057760, "tokens/train_per_sec_per_gpu": 25.96, "tokens/trainable": 79922} +{"epoch": 0.6875, "grad_norm": 0.490395188331604, "learning_rate": 8.052614644612253e-05, "loss": 0.006848854944109917, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.00687, "step": 176, "tokens/total": 6095024, "tokens/train_per_sec_per_gpu": 23.31, "tokens/trainable": 80383} +{"epoch": 0.69140625, "grad_norm": 0.2200859934091568, "learning_rate": 8.028654871074489e-05, "loss": 0.009245404042303562, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00929, "step": 177, "tokens/total": 6129712, "tokens/train_per_sec_per_gpu": 25.48, "tokens/trainable": 80824} +{"epoch": 0.6953125, "grad_norm": 0.28209155797958374, "learning_rate": 8.004589869885986e-05, "loss": 0.006664562504738569, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.49, "memory/max_allocated (GiB)": 36.49, "ppl": 1.00669, "step": 178, "tokens/total": 6161824, "tokens/train_per_sec_per_gpu": 26.69, "tokens/trainable": 81243} +{"epoch": 0.69921875, "grad_norm": 0.13619810342788696, "learning_rate": 7.980420642489674e-05, "loss": 0.004535551182925701, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.00455, "step": 179, "tokens/total": 6200464, "tokens/train_per_sec_per_gpu": 23.38, "tokens/trainable": 81716} +{"epoch": 0.703125, "grad_norm": 0.1077946349978447, "learning_rate": 7.95614819466576e-05, "loss": 0.0022269587498158216, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00223, "step": 180, "tokens/total": 6234416, "tokens/train_per_sec_per_gpu": 28.37, "tokens/trainable": 82181} +{"epoch": 0.70703125, "grad_norm": 0.3620978593826294, "learning_rate": 7.931773536489872e-05, "loss": 0.025063268840312958, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.02538, "step": 181, "tokens/total": 6268608, "tokens/train_per_sec_per_gpu": 24.92, "tokens/trainable": 82626} +{"epoch": 0.7109375, "grad_norm": 0.3133293390274048, "learning_rate": 7.907297682291035e-05, "loss": 0.005030296742916107, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00504, "step": 182, "tokens/total": 6303792, "tokens/train_per_sec_per_gpu": 26.52, "tokens/trainable": 83089} +{"epoch": 0.71484375, "grad_norm": 0.23632465302944183, "learning_rate": 7.882721650609442e-05, "loss": 0.01341234240680933, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.0135, "step": 183, "tokens/total": 6343664, "tokens/train_per_sec_per_gpu": 23.22, "tokens/trainable": 83590} +{"epoch": 0.71875, "grad_norm": 0.632203996181488, "learning_rate": 7.85804646415409e-05, "loss": 0.020600447431206703, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.02081, "step": 184, "tokens/total": 6378848, "tokens/train_per_sec_per_gpu": 26.78, "tokens/trainable": 84046} +{"epoch": 0.72265625, "grad_norm": 0.21814388036727905, "learning_rate": 7.833273149760207e-05, "loss": 0.002230391139164567, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.00223, "step": 185, "tokens/total": 6409968, "tokens/train_per_sec_per_gpu": 28.93, "tokens/trainable": 84517} +{"epoch": 0.7265625, "grad_norm": 0.3840273916721344, "learning_rate": 7.808402738346527e-05, "loss": 0.011338220909237862, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.0114, "step": 186, "tokens/total": 6448320, "tokens/train_per_sec_per_gpu": 23.42, "tokens/trainable": 84974} +{"epoch": 0.73046875, "grad_norm": 0.06491771340370178, "learning_rate": 7.783436264872382e-05, "loss": 0.001120627042837441, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.00112, "step": 187, "tokens/total": 6485440, "tokens/train_per_sec_per_gpu": 24.85, "tokens/trainable": 85460} +{"epoch": 0.734375, "grad_norm": 0.33424824476242065, "learning_rate": 7.758374768294647e-05, "loss": 0.004559164401143789, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.05, "memory/max_allocated (GiB)": 38.05, "ppl": 1.00457, "step": 188, "tokens/total": 6521216, "tokens/train_per_sec_per_gpu": 27.01, "tokens/trainable": 85939} +{"epoch": 0.73828125, "grad_norm": 0.13637591898441315, "learning_rate": 7.733219291524489e-05, "loss": 0.0032241353765130043, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.59, "memory/max_allocated (GiB)": 37.59, "ppl": 1.00323, "step": 189, "tokens/total": 6554736, "tokens/train_per_sec_per_gpu": 28.35, "tokens/trainable": 86377} +{"epoch": 0.7421875, "grad_norm": 0.25584468245506287, "learning_rate": 7.707970881383977e-05, "loss": 0.007231785915791988, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.57, "memory/max_allocated (GiB)": 36.57, "ppl": 1.00726, "step": 190, "tokens/total": 6586480, "tokens/train_per_sec_per_gpu": 29.12, "tokens/trainable": 86834} +{"epoch": 0.74609375, "grad_norm": 0.5020309090614319, "learning_rate": 7.682630588562518e-05, "loss": 0.016809571534395218, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.01695, "step": 191, "tokens/total": 6621536, "tokens/train_per_sec_per_gpu": 24.33, "tokens/trainable": 87265} +{"epoch": 0.75, "grad_norm": 0.47893303632736206, "learning_rate": 7.657199467573129e-05, "loss": 0.017085351049900055, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.01723, "step": 192, "tokens/total": 6655248, "tokens/train_per_sec_per_gpu": 27.32, "tokens/trainable": 87747} +{"epoch": 0.75390625, "grad_norm": 0.5468876957893372, "learning_rate": 7.631678576708561e-05, "loss": 0.012175765819847584, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.01225, "step": 193, "tokens/total": 6688960, "tokens/train_per_sec_per_gpu": 28.98, "tokens/trainable": 88239} +{"epoch": 0.7578125, "grad_norm": 0.3044542670249939, "learning_rate": 7.606068977997255e-05, "loss": 0.009068555198609829, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00911, "step": 194, "tokens/total": 6726256, "tokens/train_per_sec_per_gpu": 25.69, "tokens/trainable": 88718} +{"epoch": 0.76171875, "grad_norm": 0.3908222019672394, "learning_rate": 7.580371737159148e-05, "loss": 0.017448199912905693, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.0176, "step": 195, "tokens/total": 6761984, "tokens/train_per_sec_per_gpu": 22.96, "tokens/trainable": 89129} +{"epoch": 0.765625, "grad_norm": 0.1862315535545349, "learning_rate": 7.554587923561324e-05, "loss": 0.0028201851528137922, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00282, "step": 196, "tokens/total": 6796640, "tokens/train_per_sec_per_gpu": 24.64, "tokens/trainable": 89567} +{"epoch": 0.76953125, "grad_norm": 0.4661712348461151, "learning_rate": 7.528718610173511e-05, "loss": 0.024131279438734055, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.48, "memory/max_allocated (GiB)": 37.48, "ppl": 1.02442, "step": 197, "tokens/total": 6831840, "tokens/train_per_sec_per_gpu": 22.83, "tokens/trainable": 89988} +{"epoch": 0.7734375, "grad_norm": 0.17860223352909088, "learning_rate": 7.502764873523431e-05, "loss": 0.004456840455532074, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.00447, "step": 198, "tokens/total": 6866128, "tokens/train_per_sec_per_gpu": 28.12, "tokens/trainable": 90434} +{"epoch": 0.77734375, "grad_norm": 0.18681135773658752, "learning_rate": 7.476727793652011e-05, "loss": 0.005481933243572712, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.71, "memory/max_allocated (GiB)": 36.71, "ppl": 1.0055, "step": 199, "tokens/total": 6898960, "tokens/train_per_sec_per_gpu": 28.12, "tokens/trainable": 90913} +{"epoch": 0.78125, "grad_norm": 0.4049041271209717, "learning_rate": 7.450608454068415e-05, "loss": 0.011590330861508846, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.01166, "step": 200, "tokens/total": 6933312, "tokens/train_per_sec_per_gpu": 26.68, "tokens/trainable": 91355} +{"epoch": 0.78515625, "grad_norm": 0.09752383083105087, "learning_rate": 7.424407941704987e-05, "loss": 0.0037944302894175053, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.0038, "step": 201, "tokens/total": 6970016, "tokens/train_per_sec_per_gpu": 27.32, "tokens/trainable": 91856} +{"epoch": 0.7890625, "grad_norm": 0.532157838344574, "learning_rate": 7.398127346871986e-05, "loss": 0.01448328047990799, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.01459, "step": 202, "tokens/total": 7006736, "tokens/train_per_sec_per_gpu": 24.4, "tokens/trainable": 92311} +{"epoch": 0.79296875, "grad_norm": 0.12551841139793396, "learning_rate": 7.371767763212238e-05, "loss": 0.002491020830348134, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.00249, "step": 203, "tokens/total": 7041728, "tokens/train_per_sec_per_gpu": 26.56, "tokens/trainable": 92764} +{"epoch": 0.796875, "grad_norm": 0.11643016338348389, "learning_rate": 7.345330287655617e-05, "loss": 0.003953550010919571, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.00396, "step": 204, "tokens/total": 7079792, "tokens/train_per_sec_per_gpu": 22.73, "tokens/trainable": 93227} +{"epoch": 0.80078125, "grad_norm": 0.2262934297323227, "learning_rate": 7.31881602037339e-05, "loss": 0.008358120918273926, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.00839, "step": 205, "tokens/total": 7111824, "tokens/train_per_sec_per_gpu": 27.51, "tokens/trainable": 93675} +{"epoch": 0.8046875, "grad_norm": 0.05738215148448944, "learning_rate": 7.29222606473245e-05, "loss": 0.0014636334963142872, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.00146, "step": 206, "tokens/total": 7144896, "tokens/train_per_sec_per_gpu": 26.48, "tokens/trainable": 94120} +{"epoch": 0.80859375, "grad_norm": 0.3297966420650482, "learning_rate": 7.265561527249383e-05, "loss": 0.004920178093016148, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.00493, "step": 207, "tokens/total": 7178944, "tokens/train_per_sec_per_gpu": 27.74, "tokens/trainable": 94557} +{"epoch": 0.8125, "grad_norm": 0.4744245111942291, "learning_rate": 7.238823517544436e-05, "loss": 0.010629591532051563, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.01069, "step": 208, "tokens/total": 7215840, "tokens/train_per_sec_per_gpu": 29.08, "tokens/trainable": 95035} +{"epoch": 0.81640625, "grad_norm": 0.09911233931779861, "learning_rate": 7.212013148295333e-05, "loss": 0.0018238804768770933, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.00183, "step": 209, "tokens/total": 7249952, "tokens/train_per_sec_per_gpu": 29.8, "tokens/trainable": 95517} +{"epoch": 0.8203125, "grad_norm": 0.18786080181598663, "learning_rate": 7.185131535190975e-05, "loss": 0.004655770026147366, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00467, "step": 210, "tokens/total": 7282352, "tokens/train_per_sec_per_gpu": 30.02, "tokens/trainable": 96026} +{"epoch": 0.82421875, "grad_norm": 0.43210864067077637, "learning_rate": 7.158179796885005e-05, "loss": 0.013379231095314026, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.01347, "step": 211, "tokens/total": 7316768, "tokens/train_per_sec_per_gpu": 27.54, "tokens/trainable": 96477} +{"epoch": 0.828125, "grad_norm": 0.20395222306251526, "learning_rate": 7.131159054949273e-05, "loss": 0.0024121999740600586, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00242, "step": 212, "tokens/total": 7355168, "tokens/train_per_sec_per_gpu": 22.32, "tokens/trainable": 96951} +{"epoch": 0.83203125, "grad_norm": 0.8338016271591187, "learning_rate": 7.104070433827139e-05, "loss": 0.03274701535701752, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.03329, "step": 213, "tokens/total": 7390720, "tokens/train_per_sec_per_gpu": 22.95, "tokens/trainable": 97350} +{"epoch": 0.8359375, "grad_norm": 0.29064834117889404, "learning_rate": 7.076915060786705e-05, "loss": 0.00381249887868762, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.96, "memory/max_allocated (GiB)": 37.96, "ppl": 1.00382, "step": 214, "tokens/total": 7423616, "tokens/train_per_sec_per_gpu": 24.22, "tokens/trainable": 97792} +{"epoch": 0.83984375, "grad_norm": 0.24164362251758575, "learning_rate": 7.049694065873882e-05, "loss": 0.006215205416083336, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.00623, "step": 215, "tokens/total": 7458352, "tokens/train_per_sec_per_gpu": 27.95, "tokens/trainable": 98257} +{"epoch": 0.84375, "grad_norm": 0.4137440621852875, "learning_rate": 7.022408581865382e-05, "loss": 0.00962060410529375, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.69, "memory/max_allocated (GiB)": 38.69, "ppl": 1.00967, "step": 216, "tokens/total": 7492672, "tokens/train_per_sec_per_gpu": 25.47, "tokens/trainable": 98731} +{"epoch": 0.84765625, "grad_norm": 0.05040326714515686, "learning_rate": 6.99505974422157e-05, "loss": 0.0003098961606156081, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.00031, "step": 217, "tokens/total": 7526832, "tokens/train_per_sec_per_gpu": 28.41, "tokens/trainable": 99212} +{"epoch": 0.8515625, "grad_norm": 0.07119905948638916, "learning_rate": 6.967648691039213e-05, "loss": 0.001347502926364541, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.73, "memory/max_allocated (GiB)": 36.73, "ppl": 1.00135, "step": 218, "tokens/total": 7558944, "tokens/train_per_sec_per_gpu": 27.97, "tokens/trainable": 99654} +{"epoch": 0.85546875, "grad_norm": 0.07491683959960938, "learning_rate": 6.940176563004123e-05, "loss": 0.0014794389717280865, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00148, "step": 219, "tokens/total": 7593120, "tokens/train_per_sec_per_gpu": 25.97, "tokens/trainable": 100086} +{"epoch": 0.859375, "grad_norm": 0.276222825050354, "learning_rate": 6.912644503343682e-05, "loss": 0.004609537776559591, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.00462, "step": 220, "tokens/total": 7630592, "tokens/train_per_sec_per_gpu": 23.46, "tokens/trainable": 100524} +{"epoch": 0.86328125, "grad_norm": 1.1686255931854248, "learning_rate": 6.885053657779273e-05, "loss": 0.020111460238695145, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.02032, "step": 221, "tokens/total": 7662384, "tokens/train_per_sec_per_gpu": 28.43, "tokens/trainable": 100996} +{"epoch": 0.8671875, "grad_norm": 0.033542755991220474, "learning_rate": 6.857405174478604e-05, "loss": 0.0006966974469833076, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.0007, "step": 222, "tokens/total": 7697152, "tokens/train_per_sec_per_gpu": 24.35, "tokens/trainable": 101449} +{"epoch": 0.87109375, "grad_norm": 0.4232029616832733, "learning_rate": 6.82970020400792e-05, "loss": 0.02564176544547081, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.02597, "step": 223, "tokens/total": 7734624, "tokens/train_per_sec_per_gpu": 23.16, "tokens/trainable": 101905} +{"epoch": 0.875, "grad_norm": 0.1390378475189209, "learning_rate": 6.801939899284132e-05, "loss": 0.0033498783595860004, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.34, "memory/max_allocated (GiB)": 38.34, "ppl": 1.00336, "step": 224, "tokens/total": 7770848, "tokens/train_per_sec_per_gpu": 26.59, "tokens/trainable": 102411} +{"epoch": 0.87890625, "grad_norm": 0.8718096017837524, "learning_rate": 6.774125415526827e-05, "loss": 0.002683959435671568, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00269, "step": 225, "tokens/total": 7803904, "tokens/train_per_sec_per_gpu": 27.79, "tokens/trainable": 102882} +{"epoch": 0.8828125, "grad_norm": 0.11344726383686066, "learning_rate": 6.746257910210214e-05, "loss": 0.0016680811531841755, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.12, "memory/max_allocated (GiB)": 37.12, "ppl": 1.00167, "step": 226, "tokens/total": 7837088, "tokens/train_per_sec_per_gpu": 28.05, "tokens/trainable": 103332} +{"epoch": 0.88671875, "grad_norm": 0.25086215138435364, "learning_rate": 6.718338543014937e-05, "loss": 0.00798625685274601, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 35.75, "memory/max_allocated (GiB)": 35.75, "ppl": 1.00802, "step": 227, "tokens/total": 7867360, "tokens/train_per_sec_per_gpu": 31.89, "tokens/trainable": 103790} +{"epoch": 0.890625, "grad_norm": 23.137998580932617, "learning_rate": 6.69036847577983e-05, "loss": 0.0036331191658973694, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.00364, "step": 228, "tokens/total": 7901280, "tokens/train_per_sec_per_gpu": 26.77, "tokens/trainable": 104246} +{"epoch": 0.89453125, "grad_norm": 0.3039405047893524, "learning_rate": 6.662348872453553e-05, "loss": 0.009872587397694588, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.33, "memory/max_allocated (GiB)": 38.33, "ppl": 1.00992, "step": 229, "tokens/total": 7937536, "tokens/train_per_sec_per_gpu": 27.48, "tokens/trainable": 104707} +{"epoch": 0.8984375, "grad_norm": 0.1577475517988205, "learning_rate": 6.63428089904618e-05, "loss": 0.004273806698620319, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.00428, "step": 230, "tokens/total": 7976544, "tokens/train_per_sec_per_gpu": 24.89, "tokens/trainable": 105167} +{"epoch": 0.90234375, "grad_norm": 0.3610188663005829, "learning_rate": 6.60616572358065e-05, "loss": 0.002288907067850232, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.00229, "step": 231, "tokens/total": 8011008, "tokens/train_per_sec_per_gpu": 25.57, "tokens/trainable": 105576} +{"epoch": 0.90625, "grad_norm": 0.25119298696517944, "learning_rate": 6.578004516044172e-05, "loss": 0.007514103315770626, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00754, "step": 232, "tokens/total": 8046784, "tokens/train_per_sec_per_gpu": 27.55, "tokens/trainable": 106043} +{"epoch": 0.91015625, "grad_norm": 0.3065187633037567, "learning_rate": 6.549798448339548e-05, "loss": 0.010714907199144363, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.59, "memory/max_allocated (GiB)": 38.59, "ppl": 1.01077, "step": 233, "tokens/total": 8084048, "tokens/train_per_sec_per_gpu": 25.0, "tokens/trainable": 106506} +{"epoch": 0.9140625, "grad_norm": 0.20425190031528473, "learning_rate": 6.521548694236384e-05, "loss": 0.005553067661821842, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00557, "step": 234, "tokens/total": 8121248, "tokens/train_per_sec_per_gpu": 24.15, "tokens/trainable": 106951} +{"epoch": 0.91796875, "grad_norm": 0.378859281539917, "learning_rate": 6.493256429322259e-05, "loss": 0.014246304519474506, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.01435, "step": 235, "tokens/total": 8157504, "tokens/train_per_sec_per_gpu": 22.18, "tokens/trainable": 107382} +{"epoch": 0.921875, "grad_norm": 0.2544517517089844, "learning_rate": 6.464922830953799e-05, "loss": 0.011959347873926163, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.01203, "step": 236, "tokens/total": 8196368, "tokens/train_per_sec_per_gpu": 22.25, "tokens/trainable": 107814} +{"epoch": 0.92578125, "grad_norm": 0.49048906564712524, "learning_rate": 6.436549078207688e-05, "loss": 0.0363028421998024, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.03697, "step": 237, "tokens/total": 8229376, "tokens/train_per_sec_per_gpu": 28.75, "tokens/trainable": 108274} +{"epoch": 0.9296875, "grad_norm": 0.37383902072906494, "learning_rate": 6.408136351831592e-05, "loss": 0.012448485940694809, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.01253, "step": 238, "tokens/total": 8261792, "tokens/train_per_sec_per_gpu": 24.07, "tokens/trainable": 108714} +{"epoch": 0.93359375, "grad_norm": 0.20509439706802368, "learning_rate": 6.379685834195036e-05, "loss": 0.008151443675160408, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00818, "step": 239, "tokens/total": 8301216, "tokens/train_per_sec_per_gpu": 24.47, "tokens/trainable": 109220} +{"epoch": 0.9375, "grad_norm": 0.19023357331752777, "learning_rate": 6.351198709240186e-05, "loss": 0.005641391966491938, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.00566, "step": 240, "tokens/total": 8341152, "tokens/train_per_sec_per_gpu": 20.97, "tokens/trainable": 109672} +{"epoch": 0.94140625, "grad_norm": 0.3982495367527008, "learning_rate": 6.32267616243259e-05, "loss": 0.008728496730327606, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.00877, "step": 241, "tokens/total": 8373952, "tokens/train_per_sec_per_gpu": 28.64, "tokens/trainable": 110135} +{"epoch": 0.9453125, "grad_norm": 0.18238471448421478, "learning_rate": 6.294119380711849e-05, "loss": 0.008801057003438473, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.39, "memory/max_allocated (GiB)": 38.39, "ppl": 1.00884, "step": 242, "tokens/total": 8408240, "tokens/train_per_sec_per_gpu": 23.85, "tokens/trainable": 110563} +{"epoch": 0.94921875, "grad_norm": 0.3288412094116211, "learning_rate": 6.265529552442209e-05, "loss": 0.013856204226613045, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.01395, "step": 243, "tokens/total": 8443344, "tokens/train_per_sec_per_gpu": 27.93, "tokens/trainable": 111049} +{"epoch": 0.953125, "grad_norm": 0.12854821979999542, "learning_rate": 6.236907867363127e-05, "loss": 0.004744082689285278, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.17, "memory/max_allocated (GiB)": 38.17, "ppl": 1.00476, "step": 244, "tokens/total": 8477920, "tokens/train_per_sec_per_gpu": 26.57, "tokens/trainable": 111495} +{"epoch": 0.95703125, "grad_norm": 0.08577844500541687, "learning_rate": 6.208255516539749e-05, "loss": 0.003203163854777813, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.00321, "step": 245, "tokens/total": 8510240, "tokens/train_per_sec_per_gpu": 26.48, "tokens/trainable": 111968} +{"epoch": 0.9609375, "grad_norm": 0.0800066664814949, "learning_rate": 6.179573692313344e-05, "loss": 0.0017384338425472379, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.8, "memory/max_allocated (GiB)": 38.8, "ppl": 1.00174, "step": 246, "tokens/total": 8547024, "tokens/train_per_sec_per_gpu": 25.95, "tokens/trainable": 112416} +{"epoch": 0.96484375, "grad_norm": 0.0941086933016777, "learning_rate": 6.150863588251694e-05, "loss": 0.002968644257634878, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.00297, "step": 247, "tokens/total": 8586912, "tokens/train_per_sec_per_gpu": 22.58, "tokens/trainable": 112882} +{"epoch": 0.96875, "grad_norm": 0.12481236457824707, "learning_rate": 6.122126399099419e-05, "loss": 0.005211703944951296, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.00523, "step": 248, "tokens/total": 8626304, "tokens/train_per_sec_per_gpu": 26.79, "tokens/trainable": 113390} +{"epoch": 0.97265625, "grad_norm": 2.5289533138275146, "learning_rate": 6.0933633207282615e-05, "loss": 0.03098035603761673, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.03147, "step": 249, "tokens/total": 8662736, "tokens/train_per_sec_per_gpu": 27.28, "tokens/trainable": 113904} +{"epoch": 0.9765625, "grad_norm": 0.161939799785614, "learning_rate": 6.064575550087316e-05, "loss": 0.004349841736257076, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00436, "step": 250, "tokens/total": 8699376, "tokens/train_per_sec_per_gpu": 24.61, "tokens/trainable": 114351} +{"epoch": 0.98046875, "grad_norm": 0.16382896900177002, "learning_rate": 6.0357642851532245e-05, "loss": 0.004289712756872177, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.0043, "step": 251, "tokens/total": 8736064, "tokens/train_per_sec_per_gpu": 25.68, "tokens/trainable": 114842} +{"epoch": 0.984375, "grad_norm": 2.7315568923950195, "learning_rate": 6.0069307248803294e-05, "loss": 0.010076810605823994, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.01013, "step": 252, "tokens/total": 8771424, "tokens/train_per_sec_per_gpu": 21.88, "tokens/trainable": 115263} +{"epoch": 0.98828125, "grad_norm": 0.1064051166176796, "learning_rate": 5.9780760691507635e-05, "loss": 0.002188613871112466, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.33, "memory/max_allocated (GiB)": 36.33, "ppl": 1.00219, "step": 253, "tokens/total": 8803168, "tokens/train_per_sec_per_gpu": 27.63, "tokens/trainable": 115703} +{"epoch": 0.9921875, "grad_norm": 0.33240067958831787, "learning_rate": 5.9492015187245334e-05, "loss": 0.006225254386663437, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00624, "step": 254, "tokens/total": 8840912, "tokens/train_per_sec_per_gpu": 23.21, "tokens/trainable": 116157} +{"epoch": 0.99609375, "grad_norm": 0.04322066530585289, "learning_rate": 5.920308275189541e-05, "loss": 0.0008008215809240937, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.0008, "step": 255, "tokens/total": 8873184, "tokens/train_per_sec_per_gpu": 26.57, "tokens/trainable": 116567} +{"epoch": 1.0, "grad_norm": 0.3158329129219055, "learning_rate": 5.8913975409115874e-05, "loss": 0.017521217465400696, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.01768, "step": 256, "tokens/total": 8906544, "tokens/train_per_sec_per_gpu": 25.3, "tokens/trainable": 117030} +{"epoch": 1.00390625, "grad_norm": 0.09455196559429169, "learning_rate": 5.8624705189843395e-05, "loss": 0.0012845067540183663, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 1.00129, "step": 257, "tokens/total": 8943760, "tokens/train_per_sec_per_gpu": 23.99, "tokens/trainable": 117517} +{"epoch": 1.0078125, "grad_norm": 0.06000996381044388, "learning_rate": 5.833528413179249e-05, "loss": 0.0010137802455574274, "memory/device_reserved (GiB)": 42.25, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00101, "step": 258, "tokens/total": 8980256, "tokens/train_per_sec_per_gpu": 27.7, "tokens/trainable": 118015} +{"epoch": 1.01171875, "grad_norm": 0.4023720920085907, "learning_rate": 5.80457242789548e-05, "loss": 0.006115501746535301, "memory/device_reserved (GiB)": 42.31, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.00613, "step": 259, "tokens/total": 9020240, "tokens/train_per_sec_per_gpu": 23.74, "tokens/trainable": 118520} +{"epoch": 1.015625, "grad_norm": 0.10165847837924957, "learning_rate": 5.77560376810977e-05, "loss": 0.0018364974530413747, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 1.00184, "step": 260, "tokens/total": 9053888, "tokens/train_per_sec_per_gpu": 29.09, "tokens/trainable": 118992} +{"epoch": 1.01953125, "grad_norm": 0.44980207085609436, "learning_rate": 5.7466236393263005e-05, "loss": 0.009042056277394295, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.00908, "step": 261, "tokens/total": 9088960, "tokens/train_per_sec_per_gpu": 25.44, "tokens/trainable": 119438} +{"epoch": 1.0234375, "grad_norm": 0.1295047551393509, "learning_rate": 5.717633247526522e-05, "loss": 0.001442336943000555, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00144, "step": 262, "tokens/total": 9120256, "tokens/train_per_sec_per_gpu": 27.68, "tokens/trainable": 119881} +{"epoch": 1.02734375, "grad_norm": 0.4857775568962097, "learning_rate": 5.688633799118971e-05, "loss": 0.008465715683996677, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.0085, "step": 263, "tokens/total": 9152336, "tokens/train_per_sec_per_gpu": 23.2, "tokens/trainable": 120285} +{"epoch": 1.03125, "grad_norm": 0.23646992444992065, "learning_rate": 5.659626500889066e-05, "loss": 0.004661354701966047, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00467, "step": 264, "tokens/total": 9186208, "tokens/train_per_sec_per_gpu": 25.7, "tokens/trainable": 120755} +{"epoch": 1.03515625, "grad_norm": 0.04411061108112335, "learning_rate": 5.6306125599488905e-05, "loss": 0.0006913753459230065, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.00069, "step": 265, "tokens/total": 9216768, "tokens/train_per_sec_per_gpu": 32.02, "tokens/trainable": 121194} +{"epoch": 1.0390625, "grad_norm": 0.12010492384433746, "learning_rate": 5.601593183686955e-05, "loss": 0.0028151401784271, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.69, "memory/max_allocated (GiB)": 38.69, "ppl": 1.00282, "step": 266, "tokens/total": 9254032, "tokens/train_per_sec_per_gpu": 25.88, "tokens/trainable": 121670} +{"epoch": 1.04296875, "grad_norm": 0.31403103470802307, "learning_rate": 5.572569579717961e-05, "loss": 0.010537071153521538, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.18, "memory/max_allocated (GiB)": 39.18, "ppl": 1.01059, "step": 267, "tokens/total": 9289552, "tokens/train_per_sec_per_gpu": 25.28, "tokens/trainable": 122142} +{"epoch": 1.046875, "grad_norm": 0.03261857107281685, "learning_rate": 5.543542955832538e-05, "loss": 0.000627040455583483, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00063, "step": 268, "tokens/total": 9329312, "tokens/train_per_sec_per_gpu": 21.6, "tokens/trainable": 122585} +{"epoch": 1.05078125, "grad_norm": 0.24932120740413666, "learning_rate": 5.514514519946986e-05, "loss": 0.0027167622465640306, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.00272, "step": 269, "tokens/total": 9369136, "tokens/train_per_sec_per_gpu": 25.57, "tokens/trainable": 123091} +{"epoch": 1.0546875, "grad_norm": 0.0539991520345211, "learning_rate": 5.485485480053015e-05, "loss": 0.0008099790429696441, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.00081, "step": 270, "tokens/total": 9407360, "tokens/train_per_sec_per_gpu": 21.9, "tokens/trainable": 123505} +{"epoch": 1.05859375, "grad_norm": 0.15758965909481049, "learning_rate": 5.4564570441674645e-05, "loss": 0.0007398390443995595, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.92, "memory/max_allocated (GiB)": 36.92, "ppl": 1.00074, "step": 271, "tokens/total": 9440624, "tokens/train_per_sec_per_gpu": 26.36, "tokens/trainable": 123953} +{"epoch": 1.0625, "grad_norm": 0.11078359931707382, "learning_rate": 5.42743042028204e-05, "loss": 0.0007852570852264762, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.00079, "step": 272, "tokens/total": 9477120, "tokens/train_per_sec_per_gpu": 20.88, "tokens/trainable": 124400} +{"epoch": 1.06640625, "grad_norm": 0.16748838126659393, "learning_rate": 5.3984068163130464e-05, "loss": 0.0026765414513647556, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.09, "memory/max_allocated (GiB)": 37.09, "ppl": 1.00268, "step": 273, "tokens/total": 9510832, "tokens/train_per_sec_per_gpu": 28.04, "tokens/trainable": 124870} +{"epoch": 1.0703125, "grad_norm": 0.2905997931957245, "learning_rate": 5.369387440051111e-05, "loss": 0.007088727783411741, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.00711, "step": 274, "tokens/total": 9544368, "tokens/train_per_sec_per_gpu": 25.38, "tokens/trainable": 125333} +{"epoch": 1.07421875, "grad_norm": 0.0259992778301239, "learning_rate": 5.340373499110935e-05, "loss": 0.00045224642963148654, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00045, "step": 275, "tokens/total": 9580240, "tokens/train_per_sec_per_gpu": 27.19, "tokens/trainable": 125834} +{"epoch": 1.078125, "grad_norm": 0.022402849048376083, "learning_rate": 5.3113662008810304e-05, "loss": 0.0002367492124903947, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00024, "step": 276, "tokens/total": 9620256, "tokens/train_per_sec_per_gpu": 24.89, "tokens/trainable": 126316} +{"epoch": 1.08203125, "grad_norm": 0.04446302354335785, "learning_rate": 5.282366752473479e-05, "loss": 0.0005480876425281167, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 1.00055, "step": 277, "tokens/total": 9657264, "tokens/train_per_sec_per_gpu": 25.27, "tokens/trainable": 126798} +{"epoch": 1.0859375, "grad_norm": 0.7602158188819885, "learning_rate": 5.2533763606737005e-05, "loss": 0.007962438277900219, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00799, "step": 278, "tokens/total": 9696784, "tokens/train_per_sec_per_gpu": 19.29, "tokens/trainable": 127223} +{"epoch": 1.08984375, "grad_norm": 0.0281422957777977, "learning_rate": 5.224396231890232e-05, "loss": 0.0004089325375389308, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.00041, "step": 279, "tokens/total": 9728256, "tokens/train_per_sec_per_gpu": 29.85, "tokens/trainable": 127672} +{"epoch": 1.09375, "grad_norm": 0.286569744348526, "learning_rate": 5.195427572104522e-05, "loss": 0.007677216082811356, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00771, "step": 280, "tokens/total": 9762688, "tokens/train_per_sec_per_gpu": 24.61, "tokens/trainable": 128116} +{"epoch": 1.09765625, "grad_norm": 0.1556042730808258, "learning_rate": 5.166471586820751e-05, "loss": 0.0014813538873568177, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.35, "memory/max_allocated (GiB)": 39.35, "ppl": 1.00148, "step": 281, "tokens/total": 9799296, "tokens/train_per_sec_per_gpu": 23.74, "tokens/trainable": 128589} +{"epoch": 1.1015625, "grad_norm": 0.33343881368637085, "learning_rate": 5.1375294810156615e-05, "loss": 0.01464785821735859, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.01476, "step": 282, "tokens/total": 9834128, "tokens/train_per_sec_per_gpu": 25.85, "tokens/trainable": 129036} +{"epoch": 1.10546875, "grad_norm": 0.43574556708335876, "learning_rate": 5.1086024590884144e-05, "loss": 0.010065838694572449, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.01012, "step": 283, "tokens/total": 9871536, "tokens/train_per_sec_per_gpu": 24.97, "tokens/trainable": 129485} +{"epoch": 1.109375, "grad_norm": 0.030785540118813515, "learning_rate": 5.079691724810461e-05, "loss": 0.00036523971357382834, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.33, "memory/max_allocated (GiB)": 38.33, "ppl": 1.00037, "step": 284, "tokens/total": 9907712, "tokens/train_per_sec_per_gpu": 21.77, "tokens/trainable": 129920} +{"epoch": 1.11328125, "grad_norm": 0.36579573154449463, "learning_rate": 5.0507984812754684e-05, "loss": 0.004125229083001614, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.17, "memory/max_allocated (GiB)": 37.17, "ppl": 1.00413, "step": 285, "tokens/total": 9941152, "tokens/train_per_sec_per_gpu": 28.74, "tokens/trainable": 130417} +{"epoch": 1.1171875, "grad_norm": 0.1762417107820511, "learning_rate": 5.021923930849237e-05, "loss": 0.002498907968401909, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.0025, "step": 286, "tokens/total": 9975472, "tokens/train_per_sec_per_gpu": 27.95, "tokens/trainable": 130903} +{"epoch": 1.12109375, "grad_norm": 0.04739517346024513, "learning_rate": 4.99306927511967e-05, "loss": 0.0004107127315364778, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.31, "memory/max_allocated (GiB)": 37.31, "ppl": 1.00041, "step": 287, "tokens/total": 10009280, "tokens/train_per_sec_per_gpu": 27.9, "tokens/trainable": 131343} +{"epoch": 1.125, "grad_norm": 0.5245150327682495, "learning_rate": 4.964235714846775e-05, "loss": 0.008101862855255604, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.72, "memory/max_allocated (GiB)": 36.72, "ppl": 1.00813, "step": 288, "tokens/total": 10041584, "tokens/train_per_sec_per_gpu": 27.19, "tokens/trainable": 131763} +{"epoch": 1.12890625, "grad_norm": 0.08535356819629669, "learning_rate": 4.9354244499126866e-05, "loss": 0.0010445925872772932, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.16, "memory/max_allocated (GiB)": 36.16, "ppl": 1.00105, "step": 289, "tokens/total": 10073312, "tokens/train_per_sec_per_gpu": 29.38, "tokens/trainable": 132229} +{"epoch": 1.1328125, "grad_norm": 0.07956009358167648, "learning_rate": 4.90663667927174e-05, "loss": 0.0006252387538552284, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.00063, "step": 290, "tokens/total": 10106160, "tokens/train_per_sec_per_gpu": 28.91, "tokens/trainable": 132678} +{"epoch": 1.13671875, "grad_norm": 0.06541794538497925, "learning_rate": 4.877873600900581e-05, "loss": 0.0005026387516409159, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.0005, "step": 291, "tokens/total": 10142192, "tokens/train_per_sec_per_gpu": 21.71, "tokens/trainable": 133136} +{"epoch": 1.140625, "grad_norm": 0.0713411271572113, "learning_rate": 4.849136411748306e-05, "loss": 0.0010597585933282971, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.00106, "step": 292, "tokens/total": 10179776, "tokens/train_per_sec_per_gpu": 24.9, "tokens/trainable": 133608} +{"epoch": 1.14453125, "grad_norm": 0.1679653823375702, "learning_rate": 4.8204263076866574e-05, "loss": 0.005909412167966366, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00593, "step": 293, "tokens/total": 10215424, "tokens/train_per_sec_per_gpu": 29.62, "tokens/trainable": 134108} +{"epoch": 1.1484375, "grad_norm": 0.26355859637260437, "learning_rate": 4.791744483460251e-05, "loss": 0.0068098981864750385, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00683, "step": 294, "tokens/total": 10249632, "tokens/train_per_sec_per_gpu": 24.27, "tokens/trainable": 134541} +{"epoch": 1.15234375, "grad_norm": 0.12785252928733826, "learning_rate": 4.7630921326368736e-05, "loss": 0.0018057990819215775, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.00181, "step": 295, "tokens/total": 10288272, "tokens/train_per_sec_per_gpu": 20.02, "tokens/trainable": 134966} +{"epoch": 1.15625, "grad_norm": 0.09223026782274246, "learning_rate": 4.7344704475577916e-05, "loss": 0.0007295359391719103, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.66, "memory/max_allocated (GiB)": 35.66, "ppl": 1.00073, "step": 296, "tokens/total": 10318160, "tokens/train_per_sec_per_gpu": 30.3, "tokens/trainable": 135402} +{"epoch": 1.16015625, "grad_norm": 0.4518575072288513, "learning_rate": 4.705880619288153e-05, "loss": 0.015069788321852684, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.21, "memory/max_allocated (GiB)": 38.21, "ppl": 1.01518, "step": 297, "tokens/total": 10354304, "tokens/train_per_sec_per_gpu": 22.02, "tokens/trainable": 135804} +{"epoch": 1.1640625, "grad_norm": 1.2421071529388428, "learning_rate": 4.677323837567412e-05, "loss": 0.009002871811389923, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00904, "step": 298, "tokens/total": 10386960, "tokens/train_per_sec_per_gpu": 25.41, "tokens/trainable": 136231} +{"epoch": 1.16796875, "grad_norm": 0.02008887194097042, "learning_rate": 4.6488012907598146e-05, "loss": 0.0003092851839028299, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00031, "step": 299, "tokens/total": 10419712, "tokens/train_per_sec_per_gpu": 28.02, "tokens/trainable": 136705} +{"epoch": 1.171875, "grad_norm": 0.3072572946548462, "learning_rate": 4.620314165804964e-05, "loss": 0.00904853455722332, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.3, "memory/max_allocated (GiB)": 38.3, "ppl": 1.00909, "step": 300, "tokens/total": 10455376, "tokens/train_per_sec_per_gpu": 25.74, "tokens/trainable": 137178} +{"epoch": 1.17578125, "grad_norm": 0.022958921268582344, "learning_rate": 4.591863648168407e-05, "loss": 0.0004942765808664262, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00049, "step": 301, "tokens/total": 10489712, "tokens/train_per_sec_per_gpu": 26.36, "tokens/trainable": 137643} +{"epoch": 1.1796875, "grad_norm": 0.08699008077383041, "learning_rate": 4.5634509217923135e-05, "loss": 0.0005148290074430406, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00051, "step": 302, "tokens/total": 10523056, "tokens/train_per_sec_per_gpu": 25.82, "tokens/trainable": 138078} +{"epoch": 1.18359375, "grad_norm": 0.9430027008056641, "learning_rate": 4.535077169046201e-05, "loss": 0.020708367228507996, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.02092, "step": 303, "tokens/total": 10559616, "tokens/train_per_sec_per_gpu": 25.18, "tokens/trainable": 138515} +{"epoch": 1.1875, "grad_norm": 0.30132246017456055, "learning_rate": 4.506743570677743e-05, "loss": 0.013621306978166103, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.01371, "step": 304, "tokens/total": 10590208, "tokens/train_per_sec_per_gpu": 31.36, "tokens/trainable": 138948} +{"epoch": 1.19140625, "grad_norm": 0.1955755054950714, "learning_rate": 4.478451305763618e-05, "loss": 0.007412984035909176, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.95, "memory/max_allocated (GiB)": 36.95, "ppl": 1.00744, "step": 305, "tokens/total": 10623344, "tokens/train_per_sec_per_gpu": 30.61, "tokens/trainable": 139408} +{"epoch": 1.1953125, "grad_norm": 0.1633482426404953, "learning_rate": 4.450201551660454e-05, "loss": 0.004608819726854563, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.51, "memory/max_allocated (GiB)": 36.51, "ppl": 1.00462, "step": 306, "tokens/total": 10655056, "tokens/train_per_sec_per_gpu": 26.29, "tokens/trainable": 139840} +{"epoch": 1.19921875, "grad_norm": 0.36653926968574524, "learning_rate": 4.4219954839558276e-05, "loss": 0.011972867883741856, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.01204, "step": 307, "tokens/total": 10689744, "tokens/train_per_sec_per_gpu": 25.4, "tokens/trainable": 140281} +{"epoch": 1.203125, "grad_norm": 0.3092474043369293, "learning_rate": 4.393834276419352e-05, "loss": 0.00819158274680376, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.93, "memory/max_allocated (GiB)": 36.93, "ppl": 1.00823, "step": 308, "tokens/total": 10721456, "tokens/train_per_sec_per_gpu": 31.59, "tokens/trainable": 140776} +{"epoch": 1.20703125, "grad_norm": 0.0854310616850853, "learning_rate": 4.36571910095382e-05, "loss": 0.0008177190320566297, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.00082, "step": 309, "tokens/total": 10759568, "tokens/train_per_sec_per_gpu": 25.46, "tokens/trainable": 141228} +{"epoch": 1.2109375, "grad_norm": 0.12553831934928894, "learning_rate": 4.337651127546448e-05, "loss": 0.0017667675856500864, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00177, "step": 310, "tokens/total": 10795248, "tokens/train_per_sec_per_gpu": 27.57, "tokens/trainable": 141679} +{"epoch": 1.21484375, "grad_norm": 0.215590238571167, "learning_rate": 4.3096315242201736e-05, "loss": 0.004440860822796822, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.61, "memory/max_allocated (GiB)": 36.61, "ppl": 1.00445, "step": 311, "tokens/total": 10828208, "tokens/train_per_sec_per_gpu": 26.27, "tokens/trainable": 142122} +{"epoch": 1.21875, "grad_norm": 0.16133683919906616, "learning_rate": 4.2816614569850635e-05, "loss": 0.004375312011688948, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00438, "step": 312, "tokens/total": 10863984, "tokens/train_per_sec_per_gpu": 24.33, "tokens/trainable": 142561} +{"epoch": 1.22265625, "grad_norm": 0.20809651911258698, "learning_rate": 4.2537420897897864e-05, "loss": 0.004768298007547855, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.00478, "step": 313, "tokens/total": 10899392, "tokens/train_per_sec_per_gpu": 27.0, "tokens/trainable": 143046} +{"epoch": 1.2265625, "grad_norm": 0.04570009186863899, "learning_rate": 4.225874584473174e-05, "loss": 0.0009379230323247612, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.00094, "step": 314, "tokens/total": 10932224, "tokens/train_per_sec_per_gpu": 27.08, "tokens/trainable": 143493} +{"epoch": 1.23046875, "grad_norm": 0.2048281580209732, "learning_rate": 4.19806010071587e-05, "loss": 0.006936085410416126, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.00696, "step": 315, "tokens/total": 10970096, "tokens/train_per_sec_per_gpu": 24.33, "tokens/trainable": 143956} +{"epoch": 1.234375, "grad_norm": 0.19595490396022797, "learning_rate": 4.170299795992081e-05, "loss": 0.005491916090250015, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.00551, "step": 316, "tokens/total": 11003488, "tokens/train_per_sec_per_gpu": 27.03, "tokens/trainable": 144411} +{"epoch": 1.23828125, "grad_norm": 0.22878386080265045, "learning_rate": 4.142594825521398e-05, "loss": 0.003958361689001322, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.00397, "step": 317, "tokens/total": 11042784, "tokens/train_per_sec_per_gpu": 25.95, "tokens/trainable": 144892} +{"epoch": 1.2421875, "grad_norm": 0.06004524230957031, "learning_rate": 4.114946342220728e-05, "loss": 0.0012459130957722664, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.00125, "step": 318, "tokens/total": 11078992, "tokens/train_per_sec_per_gpu": 23.92, "tokens/trainable": 145339} +{"epoch": 1.24609375, "grad_norm": 0.15198799967765808, "learning_rate": 4.087355496656321e-05, "loss": 0.003534863004460931, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00354, "step": 319, "tokens/total": 11113024, "tokens/train_per_sec_per_gpu": 28.41, "tokens/trainable": 145811} +{"epoch": 1.25, "grad_norm": 0.2727871835231781, "learning_rate": 4.05982343699588e-05, "loss": 0.004931654315441847, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00494, "step": 320, "tokens/total": 11152240, "tokens/train_per_sec_per_gpu": 25.99, "tokens/trainable": 146314} +{"epoch": 1.25390625, "grad_norm": 0.11070676147937775, "learning_rate": 4.0323513089607876e-05, "loss": 0.0014402340166270733, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.00144, "step": 321, "tokens/total": 11187296, "tokens/train_per_sec_per_gpu": 24.29, "tokens/trainable": 146781} +{"epoch": 1.2578125, "grad_norm": 0.27737119793891907, "learning_rate": 4.004940255778431e-05, "loss": 0.01081375777721405, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.01087, "step": 322, "tokens/total": 11225904, "tokens/train_per_sec_per_gpu": 23.92, "tokens/trainable": 147225} +{"epoch": 1.26171875, "grad_norm": 0.07135581970214844, "learning_rate": 3.977591418134619e-05, "loss": 0.0013002483174204826, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.0013, "step": 323, "tokens/total": 11261056, "tokens/train_per_sec_per_gpu": 26.74, "tokens/trainable": 147673} +{"epoch": 1.265625, "grad_norm": 0.27768784761428833, "learning_rate": 3.95030593412612e-05, "loss": 0.005070080049335957, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.00508, "step": 324, "tokens/total": 11296432, "tokens/train_per_sec_per_gpu": 24.6, "tokens/trainable": 148103} +{"epoch": 1.26953125, "grad_norm": 0.1169687882065773, "learning_rate": 3.923084939213296e-05, "loss": 0.0019137689378112555, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.88, "memory/max_allocated (GiB)": 35.88, "ppl": 1.00192, "step": 325, "tokens/total": 11327248, "tokens/train_per_sec_per_gpu": 27.84, "tokens/trainable": 148535} +{"epoch": 1.2734375, "grad_norm": 0.2553396224975586, "learning_rate": 3.895929566172861e-05, "loss": 0.006054587662220001, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.00607, "step": 326, "tokens/total": 11363264, "tokens/train_per_sec_per_gpu": 25.21, "tokens/trainable": 148999} +{"epoch": 1.27734375, "grad_norm": 0.05400911346077919, "learning_rate": 3.868840945050728e-05, "loss": 0.0005856914795003831, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.05, "memory/max_allocated (GiB)": 39.05, "ppl": 1.00059, "step": 327, "tokens/total": 11401776, "tokens/train_per_sec_per_gpu": 21.56, "tokens/trainable": 149431} +{"epoch": 1.28125, "grad_norm": 0.09080156683921814, "learning_rate": 3.841820203114995e-05, "loss": 0.001826456980779767, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.00183, "step": 328, "tokens/total": 11433520, "tokens/train_per_sec_per_gpu": 29.11, "tokens/trainable": 149886} +{"epoch": 1.28515625, "grad_norm": 0.1269005537033081, "learning_rate": 3.814868464809027e-05, "loss": 0.0026185449678450823, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.00262, "step": 329, "tokens/total": 11465376, "tokens/train_per_sec_per_gpu": 25.87, "tokens/trainable": 150288} +{"epoch": 1.2890625, "grad_norm": 0.35746631026268005, "learning_rate": 3.787986851704667e-05, "loss": 0.006286406423896551, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.00631, "step": 330, "tokens/total": 11499120, "tokens/train_per_sec_per_gpu": 26.63, "tokens/trainable": 150733} +{"epoch": 1.29296875, "grad_norm": 0.053731631487607956, "learning_rate": 3.7611764824555654e-05, "loss": 0.0005181680317036808, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00052, "step": 331, "tokens/total": 11531104, "tokens/train_per_sec_per_gpu": 30.97, "tokens/trainable": 151207} +{"epoch": 1.296875, "grad_norm": 0.33926287293434143, "learning_rate": 3.734438472750619e-05, "loss": 0.006867760326713324, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.00689, "step": 332, "tokens/total": 11571040, "tokens/train_per_sec_per_gpu": 24.68, "tokens/trainable": 151694} +{"epoch": 1.30078125, "grad_norm": 0.05470799654722214, "learning_rate": 3.707773935267552e-05, "loss": 0.000772945350036025, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.46, "memory/max_allocated (GiB)": 38.46, "ppl": 1.00077, "step": 333, "tokens/total": 11607952, "tokens/train_per_sec_per_gpu": 27.76, "tokens/trainable": 152188} +{"epoch": 1.3046875, "grad_norm": 0.07849840074777603, "learning_rate": 3.68118397962661e-05, "loss": 0.001098731067031622, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.0011, "step": 334, "tokens/total": 11642864, "tokens/train_per_sec_per_gpu": 23.28, "tokens/trainable": 152596} +{"epoch": 1.30859375, "grad_norm": 0.07340895384550095, "learning_rate": 3.654669712344384e-05, "loss": 0.0010940951760858297, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 1.00109, "step": 335, "tokens/total": 11673904, "tokens/train_per_sec_per_gpu": 31.68, "tokens/trainable": 153052} +{"epoch": 1.3125, "grad_norm": 0.005960756912827492, "learning_rate": 3.628232236787763e-05, "loss": 0.00010029409895651042, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.96, "memory/max_allocated (GiB)": 37.96, "ppl": 1.0001, "step": 336, "tokens/total": 11709536, "tokens/train_per_sec_per_gpu": 22.69, "tokens/trainable": 153491} +{"epoch": 1.31640625, "grad_norm": 0.1693747192621231, "learning_rate": 3.6018726531280144e-05, "loss": 0.0016363320173695683, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00164, "step": 337, "tokens/total": 11747248, "tokens/train_per_sec_per_gpu": 28.31, "tokens/trainable": 153983} +{"epoch": 1.3203125, "grad_norm": 0.1694205105304718, "learning_rate": 3.575592058295017e-05, "loss": 0.003565971739590168, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.00357, "step": 338, "tokens/total": 11781104, "tokens/train_per_sec_per_gpu": 28.76, "tokens/trainable": 154464} +{"epoch": 1.32421875, "grad_norm": 0.026956452056765556, "learning_rate": 3.549391545931585e-05, "loss": 0.0002866385329980403, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00029, "step": 339, "tokens/total": 11815712, "tokens/train_per_sec_per_gpu": 27.77, "tokens/trainable": 154924} +{"epoch": 1.328125, "grad_norm": 0.1304219514131546, "learning_rate": 3.5232722063479914e-05, "loss": 0.0016795675037428737, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.00168, "step": 340, "tokens/total": 11849424, "tokens/train_per_sec_per_gpu": 24.21, "tokens/trainable": 155373} +{"epoch": 1.33203125, "grad_norm": 0.019757287576794624, "learning_rate": 3.49723512647657e-05, "loss": 0.0003085597709286958, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.00031, "step": 341, "tokens/total": 11888128, "tokens/train_per_sec_per_gpu": 24.82, "tokens/trainable": 155873} +{"epoch": 1.3359375, "grad_norm": 0.31031978130340576, "learning_rate": 3.471281389826491e-05, "loss": 0.0078215217217803, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.00785, "step": 342, "tokens/total": 11919264, "tokens/train_per_sec_per_gpu": 26.04, "tokens/trainable": 156278} +{"epoch": 1.33984375, "grad_norm": 0.44686517119407654, "learning_rate": 3.4454120764386764e-05, "loss": 0.0017366965766996145, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.66, "memory/max_allocated (GiB)": 38.66, "ppl": 1.00174, "step": 343, "tokens/total": 11957456, "tokens/train_per_sec_per_gpu": 24.36, "tokens/trainable": 156733} +{"epoch": 1.34375, "grad_norm": 0.036343544721603394, "learning_rate": 3.4196282628408526e-05, "loss": 0.0004629010509233922, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.00046, "step": 344, "tokens/total": 11991696, "tokens/train_per_sec_per_gpu": 26.36, "tokens/trainable": 157203} +{"epoch": 1.34765625, "grad_norm": 0.021432727575302124, "learning_rate": 3.3939310220027456e-05, "loss": 0.00036383623955771327, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00036, "step": 345, "tokens/total": 12025840, "tokens/train_per_sec_per_gpu": 29.92, "tokens/trainable": 157707} +{"epoch": 1.3515625, "grad_norm": 0.0036720065400004387, "learning_rate": 3.3683214232914404e-05, "loss": 6.43095190753229e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.00006, "step": 346, "tokens/total": 12061296, "tokens/train_per_sec_per_gpu": 26.18, "tokens/trainable": 158180} +{"epoch": 1.35546875, "grad_norm": 0.005154092330485582, "learning_rate": 3.342800532426873e-05, "loss": 8.618015272077173e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.00009, "step": 347, "tokens/total": 12097232, "tokens/train_per_sec_per_gpu": 25.49, "tokens/trainable": 158646} +{"epoch": 1.359375, "grad_norm": 0.06002897769212723, "learning_rate": 3.317369411437484e-05, "loss": 0.00018770628958009183, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.00019, "step": 348, "tokens/total": 12133200, "tokens/train_per_sec_per_gpu": 26.23, "tokens/trainable": 159115} +{"epoch": 1.36328125, "grad_norm": 0.11124070733785629, "learning_rate": 3.292029118616024e-05, "loss": 0.0005596104310825467, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.00056, "step": 349, "tokens/total": 12166336, "tokens/train_per_sec_per_gpu": 25.66, "tokens/trainable": 159538} +{"epoch": 1.3671875, "grad_norm": 0.43825405836105347, "learning_rate": 3.266780708475511e-05, "loss": 0.006168657448142767, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00619, "step": 350, "tokens/total": 12201872, "tokens/train_per_sec_per_gpu": 25.09, "tokens/trainable": 160016} +{"epoch": 1.37109375, "grad_norm": 0.20544910430908203, "learning_rate": 3.241625231705354e-05, "loss": 0.00284214923158288, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.00285, "step": 351, "tokens/total": 12238016, "tokens/train_per_sec_per_gpu": 25.08, "tokens/trainable": 160475} +{"epoch": 1.375, "grad_norm": 0.018312320113182068, "learning_rate": 3.216563735127618e-05, "loss": 0.000207554578082636, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.15, "memory/max_allocated (GiB)": 39.15, "ppl": 1.00021, "step": 352, "tokens/total": 12273280, "tokens/train_per_sec_per_gpu": 25.62, "tokens/trainable": 160952} +{"epoch": 1.37890625, "grad_norm": 0.14520274102687836, "learning_rate": 3.191597261653475e-05, "loss": 0.0011407091515138745, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00114, "step": 353, "tokens/total": 12310096, "tokens/train_per_sec_per_gpu": 23.75, "tokens/trainable": 161420} +{"epoch": 1.3828125, "grad_norm": 0.6647793054580688, "learning_rate": 3.166726850239794e-05, "loss": 0.002166443970054388, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00217, "step": 354, "tokens/total": 12343136, "tokens/train_per_sec_per_gpu": 27.53, "tokens/trainable": 161868} +{"epoch": 1.38671875, "grad_norm": 0.0037731800694018602, "learning_rate": 3.141953535845912e-05, "loss": 3.427079354878515e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.34, "memory/max_allocated (GiB)": 36.34, "ppl": 1.00003, "step": 355, "tokens/total": 12374576, "tokens/train_per_sec_per_gpu": 30.44, "tokens/trainable": 162318} +{"epoch": 1.390625, "grad_norm": 0.016562558710575104, "learning_rate": 3.11727834939056e-05, "loss": 0.00016641319962218404, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00017, "step": 356, "tokens/total": 12406896, "tokens/train_per_sec_per_gpu": 26.42, "tokens/trainable": 162789} +{"epoch": 1.39453125, "grad_norm": 0.24920770525932312, "learning_rate": 3.092702317708967e-05, "loss": 0.0066790650598704815, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.0067, "step": 357, "tokens/total": 12440480, "tokens/train_per_sec_per_gpu": 28.97, "tokens/trainable": 163254} +{"epoch": 1.3984375, "grad_norm": 0.004230008460581303, "learning_rate": 3.0682264635101276e-05, "loss": 7.018685573711991e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.00007, "step": 358, "tokens/total": 12473232, "tokens/train_per_sec_per_gpu": 28.32, "tokens/trainable": 163715} +{"epoch": 1.40234375, "grad_norm": 0.1067110002040863, "learning_rate": 3.0438518053342407e-05, "loss": 0.0010806251084432006, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00108, "step": 359, "tokens/total": 12512176, "tokens/train_per_sec_per_gpu": 23.26, "tokens/trainable": 164197} +{"epoch": 1.40625, "grad_norm": 0.05476995185017586, "learning_rate": 3.0195793575103266e-05, "loss": 0.0006543396739289165, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00065, "step": 360, "tokens/total": 12551136, "tokens/train_per_sec_per_gpu": 21.3, "tokens/trainable": 164661} +{"epoch": 1.41015625, "grad_norm": 0.011848127469420433, "learning_rate": 2.9954101301140146e-05, "loss": 0.00011025303683709353, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.57, "memory/max_allocated (GiB)": 37.57, "ppl": 1.00011, "step": 361, "tokens/total": 12586560, "tokens/train_per_sec_per_gpu": 24.62, "tokens/trainable": 165111} +{"epoch": 1.4140625, "grad_norm": 0.004557554144412279, "learning_rate": 2.9713451289255123e-05, "loss": 7.261379505507648e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.63, "memory/max_allocated (GiB)": 36.63, "ppl": 1.00007, "step": 362, "tokens/total": 12615936, "tokens/train_per_sec_per_gpu": 25.63, "tokens/trainable": 165552} +{"epoch": 1.41796875, "grad_norm": 0.00548908207565546, "learning_rate": 2.9473853553877484e-05, "loss": 5.8191588323097676e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.00006, "step": 363, "tokens/total": 12650096, "tokens/train_per_sec_per_gpu": 27.31, "tokens/trainable": 166026} +{"epoch": 1.421875, "grad_norm": 0.0771864578127861, "learning_rate": 2.9235318065647e-05, "loss": 0.0007392480620183051, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.00074, "step": 364, "tokens/total": 12689808, "tokens/train_per_sec_per_gpu": 24.98, "tokens/trainable": 166486} +{"epoch": 1.42578125, "grad_norm": 0.008245972916483879, "learning_rate": 2.8997854750998964e-05, "loss": 8.724065264686942e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.46, "memory/max_allocated (GiB)": 39.46, "ppl": 1.00009, "step": 365, "tokens/total": 12729328, "tokens/train_per_sec_per_gpu": 23.98, "tokens/trainable": 166959} +{"epoch": 1.4296875, "grad_norm": 0.0019095467869192362, "learning_rate": 2.8761473491751258e-05, "loss": 2.4346796635654755e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.00002, "step": 366, "tokens/total": 12762576, "tokens/train_per_sec_per_gpu": 26.98, "tokens/trainable": 167394} +{"epoch": 1.43359375, "grad_norm": 0.4447092115879059, "learning_rate": 2.8526184124692883e-05, "loss": 0.012152349576354027, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.01223, "step": 367, "tokens/total": 12798944, "tokens/train_per_sec_per_gpu": 20.43, "tokens/trainable": 167832} +{"epoch": 1.4375, "grad_norm": 0.057797666639089584, "learning_rate": 2.829199644117484e-05, "loss": 0.0005285246879793704, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.00053, "step": 368, "tokens/total": 12832928, "tokens/train_per_sec_per_gpu": 24.42, "tokens/trainable": 168242} +{"epoch": 1.44140625, "grad_norm": 0.46683529019355774, "learning_rate": 2.8058920186702553e-05, "loss": 0.004918512888252735, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00493, "step": 369, "tokens/total": 12870544, "tokens/train_per_sec_per_gpu": 25.05, "tokens/trainable": 168715} +{"epoch": 1.4453125, "grad_norm": 0.017707867547869682, "learning_rate": 2.782696506053033e-05, "loss": 0.00018034171080216765, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.00018, "step": 370, "tokens/total": 12908416, "tokens/train_per_sec_per_gpu": 22.43, "tokens/trainable": 169147} +{"epoch": 1.44921875, "grad_norm": 0.2845745384693146, "learning_rate": 2.7596140715257824e-05, "loss": 0.002609334886074066, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00261, "step": 371, "tokens/total": 12943584, "tokens/train_per_sec_per_gpu": 25.75, "tokens/trainable": 169628} +{"epoch": 1.453125, "grad_norm": 0.020017120987176895, "learning_rate": 2.7366456756428184e-05, "loss": 0.0001420947228325531, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 1.00014, "step": 372, "tokens/total": 12978192, "tokens/train_per_sec_per_gpu": 27.23, "tokens/trainable": 170085} +{"epoch": 1.45703125, "grad_norm": 0.13858212530612946, "learning_rate": 2.7137922742128486e-05, "loss": 0.001571332337334752, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00157, "step": 373, "tokens/total": 13012720, "tokens/train_per_sec_per_gpu": 27.1, "tokens/trainable": 170584} +{"epoch": 1.4609375, "grad_norm": 0.12611767649650574, "learning_rate": 2.691054818259188e-05, "loss": 0.0017406763508915901, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00174, "step": 374, "tokens/total": 13047568, "tokens/train_per_sec_per_gpu": 26.5, "tokens/trainable": 171058} +{"epoch": 1.46484375, "grad_norm": 0.1716843843460083, "learning_rate": 2.6684342539801933e-05, "loss": 0.00208684173412621, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.56, "memory/max_allocated (GiB)": 36.56, "ppl": 1.00209, "step": 375, "tokens/total": 13079088, "tokens/train_per_sec_per_gpu": 28.89, "tokens/trainable": 171518} +{"epoch": 1.46875, "grad_norm": 0.2006293684244156, "learning_rate": 2.645931522709877e-05, "loss": 0.002840345725417137, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.00284, "step": 376, "tokens/total": 13114480, "tokens/train_per_sec_per_gpu": 30.14, "tokens/trainable": 172007} +{"epoch": 1.47265625, "grad_norm": 0.09150528162717819, "learning_rate": 2.6235475608787365e-05, "loss": 0.0010091906879097223, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.00101, "step": 377, "tokens/total": 13150160, "tokens/train_per_sec_per_gpu": 25.62, "tokens/trainable": 172468} +{"epoch": 1.4765625, "grad_norm": 0.29930660128593445, "learning_rate": 2.6012832999747916e-05, "loss": 0.005956257693469524, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.00597, "step": 378, "tokens/total": 13181280, "tokens/train_per_sec_per_gpu": 29.38, "tokens/trainable": 172923} +{"epoch": 1.48046875, "grad_norm": 0.2186805158853531, "learning_rate": 2.579139666504821e-05, "loss": 0.013851317577064037, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.01395, "step": 379, "tokens/total": 13221264, "tokens/train_per_sec_per_gpu": 22.2, "tokens/trainable": 173370} +{"epoch": 1.484375, "grad_norm": 0.012883467599749565, "learning_rate": 2.557117581955798e-05, "loss": 0.00020800440688617527, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00021, "step": 380, "tokens/total": 13255104, "tokens/train_per_sec_per_gpu": 27.04, "tokens/trainable": 173837} +{"epoch": 1.48828125, "grad_norm": 0.27762576937675476, "learning_rate": 2.5352179627565532e-05, "loss": 0.0034697633236646652, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.00348, "step": 381, "tokens/total": 13289856, "tokens/train_per_sec_per_gpu": 27.19, "tokens/trainable": 174294} +{"epoch": 1.4921875, "grad_norm": 0.06169753894209862, "learning_rate": 2.5134417202396277e-05, "loss": 0.0005078035173937678, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.6, "memory/max_allocated (GiB)": 38.6, "ppl": 1.00051, "step": 382, "tokens/total": 13323840, "tokens/train_per_sec_per_gpu": 24.05, "tokens/trainable": 174701} +{"epoch": 1.49609375, "grad_norm": 0.0476718433201313, "learning_rate": 2.491789760603361e-05, "loss": 0.0009914182592183352, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00099, "step": 383, "tokens/total": 13360256, "tokens/train_per_sec_per_gpu": 23.29, "tokens/trainable": 175131} +{"epoch": 1.5, "grad_norm": 0.1325261890888214, "learning_rate": 2.4702629848741764e-05, "loss": 0.0014169956557452679, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.00142, "step": 384, "tokens/total": 13394304, "tokens/train_per_sec_per_gpu": 26.59, "tokens/trainable": 175586} +{"epoch": 1.50390625, "grad_norm": 0.19112160801887512, "learning_rate": 2.4488622888690785e-05, "loss": 0.003643997712060809, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.04, "memory/max_allocated (GiB)": 37.04, "ppl": 1.00365, "step": 385, "tokens/total": 13427760, "tokens/train_per_sec_per_gpu": 27.1, "tokens/trainable": 176026} +{"epoch": 1.5078125, "grad_norm": 0.030404722318053246, "learning_rate": 2.427588563158384e-05, "loss": 0.0003154940204694867, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00032, "step": 386, "tokens/total": 13462096, "tokens/train_per_sec_per_gpu": 26.28, "tokens/trainable": 176512} +{"epoch": 1.51171875, "grad_norm": 0.0948273316025734, "learning_rate": 2.406442693028651e-05, "loss": 0.0007075145840644836, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.63, "memory/max_allocated (GiB)": 38.63, "ppl": 1.00071, "step": 387, "tokens/total": 13494976, "tokens/train_per_sec_per_gpu": 24.97, "tokens/trainable": 176943} +{"epoch": 1.515625, "grad_norm": 0.01284817885607481, "learning_rate": 2.3854255584458547e-05, "loss": 3.877377821481787e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.00004, "step": 388, "tokens/total": 13533568, "tokens/train_per_sec_per_gpu": 21.65, "tokens/trainable": 177370} +{"epoch": 1.51953125, "grad_norm": 0.11823682487010956, "learning_rate": 2.3645380340187508e-05, "loss": 0.0030158397275954485, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00302, "step": 389, "tokens/total": 13569248, "tokens/train_per_sec_per_gpu": 26.18, "tokens/trainable": 177851} +{"epoch": 1.5234375, "grad_norm": 0.26980581879615784, "learning_rate": 2.3437809889624914e-05, "loss": 0.003850418608635664, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.00386, "step": 390, "tokens/total": 13601568, "tokens/train_per_sec_per_gpu": 29.01, "tokens/trainable": 178310} +{"epoch": 1.52734375, "grad_norm": 0.03680704906582832, "learning_rate": 2.3231552870624487e-05, "loss": 0.00029030078439973295, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.47, "memory/max_allocated (GiB)": 38.47, "ppl": 1.00029, "step": 391, "tokens/total": 13632576, "tokens/train_per_sec_per_gpu": 29.23, "tokens/trainable": 178736} +{"epoch": 1.53125, "grad_norm": 0.046257492154836655, "learning_rate": 2.3026617866382657e-05, "loss": 0.0006040141452103853, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.53, "memory/max_allocated (GiB)": 36.53, "ppl": 1.0006, "step": 392, "tokens/total": 13665328, "tokens/train_per_sec_per_gpu": 29.99, "tokens/trainable": 179233} +{"epoch": 1.53515625, "grad_norm": 0.02722022496163845, "learning_rate": 2.2823013405081507e-05, "loss": 0.00025009518139995635, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.00025, "step": 393, "tokens/total": 13703408, "tokens/train_per_sec_per_gpu": 26.94, "tokens/trainable": 179730} +{"epoch": 1.5390625, "grad_norm": 0.017282193526625633, "learning_rate": 2.2620747959533722e-05, "loss": 0.00026272545801475644, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.76, "memory/max_allocated (GiB)": 36.76, "ppl": 1.00026, "step": 394, "tokens/total": 13736480, "tokens/train_per_sec_per_gpu": 30.78, "tokens/trainable": 180227} +{"epoch": 1.54296875, "grad_norm": 0.015652306377887726, "learning_rate": 2.2419829946830123e-05, "loss": 0.0001693902158876881, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.00017, "step": 395, "tokens/total": 13772672, "tokens/train_per_sec_per_gpu": 24.27, "tokens/trainable": 180687} +{"epoch": 1.546875, "grad_norm": 0.44558513164520264, "learning_rate": 2.2220267727989325e-05, "loss": 0.0014085317961871624, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00141, "step": 396, "tokens/total": 13806816, "tokens/train_per_sec_per_gpu": 25.86, "tokens/trainable": 181141} +{"epoch": 1.55078125, "grad_norm": 0.11077313125133514, "learning_rate": 2.202206960760984e-05, "loss": 0.0012979776365682483, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.0013, "step": 397, "tokens/total": 13844960, "tokens/train_per_sec_per_gpu": 24.81, "tokens/trainable": 181648} +{"epoch": 1.5546875, "grad_norm": 0.08468553423881531, "learning_rate": 2.182524383352446e-05, "loss": 0.0014088593889027834, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00141, "step": 398, "tokens/total": 13881120, "tokens/train_per_sec_per_gpu": 24.97, "tokens/trainable": 182109} +{"epoch": 1.55859375, "grad_norm": 0.12395957857370377, "learning_rate": 2.1629798596457056e-05, "loss": 0.0026104357093572617, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.52, "memory/max_allocated (GiB)": 36.52, "ppl": 1.00261, "step": 399, "tokens/total": 13913440, "tokens/train_per_sec_per_gpu": 26.74, "tokens/trainable": 182559} +{"epoch": 1.5625, "grad_norm": 0.0041193971410393715, "learning_rate": 2.1435742029681725e-05, "loss": 5.9541849623201415e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.00006, "step": 400, "tokens/total": 13945408, "tokens/train_per_sec_per_gpu": 27.73, "tokens/trainable": 182985} +{"epoch": 1.56640625, "grad_norm": 0.0035320771858096123, "learning_rate": 2.124308220868431e-05, "loss": 3.75295385310892e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.67, "memory/max_allocated (GiB)": 38.67, "ppl": 1.00004, "step": 401, "tokens/total": 13980800, "tokens/train_per_sec_per_gpu": 24.37, "tokens/trainable": 183440} +{"epoch": 1.5703125, "grad_norm": 0.0829463005065918, "learning_rate": 2.105182715082638e-05, "loss": 0.0015828940086066723, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00158, "step": 402, "tokens/total": 14014400, "tokens/train_per_sec_per_gpu": 27.66, "tokens/trainable": 183917} +{"epoch": 1.57421875, "grad_norm": 0.35330861806869507, "learning_rate": 2.0861984815011552e-05, "loss": 0.005209210328757763, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.16, "memory/max_allocated (GiB)": 38.16, "ppl": 1.00522, "step": 403, "tokens/total": 14049920, "tokens/train_per_sec_per_gpu": 22.66, "tokens/trainable": 184348} +{"epoch": 1.578125, "grad_norm": 0.016622642055153847, "learning_rate": 2.0673563101354323e-05, "loss": 0.0003030760562978685, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.0003, "step": 404, "tokens/total": 14085200, "tokens/train_per_sec_per_gpu": 23.63, "tokens/trainable": 184786} +{"epoch": 1.58203125, "grad_norm": 0.02456033043563366, "learning_rate": 2.0486569850851317e-05, "loss": 0.00019374213297851384, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.00019, "step": 405, "tokens/total": 14120208, "tokens/train_per_sec_per_gpu": 25.95, "tokens/trainable": 185249} +{"epoch": 1.5859375, "grad_norm": 0.16133390367031097, "learning_rate": 2.0301012845054956e-05, "loss": 0.0018292152089998126, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.00183, "step": 406, "tokens/total": 14158960, "tokens/train_per_sec_per_gpu": 21.47, "tokens/trainable": 185680} +{"epoch": 1.58984375, "grad_norm": 0.1828688681125641, "learning_rate": 2.011689980574966e-05, "loss": 0.0013619004748761654, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.06, "memory/max_allocated (GiB)": 38.06, "ppl": 1.00136, "step": 407, "tokens/total": 14195264, "tokens/train_per_sec_per_gpu": 23.0, "tokens/trainable": 186100} +{"epoch": 1.59375, "grad_norm": 0.1289617121219635, "learning_rate": 1.993423839463052e-05, "loss": 0.0014213486574590206, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.00142, "step": 408, "tokens/total": 14228528, "tokens/train_per_sec_per_gpu": 32.01, "tokens/trainable": 186565} +{"epoch": 1.59765625, "grad_norm": 0.11145743727684021, "learning_rate": 1.975303621298445e-05, "loss": 0.0012223562225699425, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.00122, "step": 409, "tokens/total": 14265568, "tokens/train_per_sec_per_gpu": 25.46, "tokens/trainable": 187017} +{"epoch": 1.6015625, "grad_norm": 0.05137834697961807, "learning_rate": 1.957330080137385e-05, "loss": 0.0007383479969576001, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.31, "memory/max_allocated (GiB)": 36.31, "ppl": 1.00074, "step": 410, "tokens/total": 14296432, "tokens/train_per_sec_per_gpu": 29.07, "tokens/trainable": 187468} +{"epoch": 1.60546875, "grad_norm": 0.142389714717865, "learning_rate": 1.9395039639322864e-05, "loss": 0.001476580509915948, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.00148, "step": 411, "tokens/total": 14332480, "tokens/train_per_sec_per_gpu": 27.43, "tokens/trainable": 187961} +{"epoch": 1.609375, "grad_norm": 0.010279864072799683, "learning_rate": 1.9218260145006073e-05, "loss": 9.95980080915615e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.52, "memory/max_allocated (GiB)": 37.52, "ppl": 1.0001, "step": 412, "tokens/total": 14367728, "tokens/train_per_sec_per_gpu": 27.08, "tokens/trainable": 188447} +{"epoch": 1.61328125, "grad_norm": 0.03774605691432953, "learning_rate": 1.904296967493982e-05, "loss": 0.0006343858549371362, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.00063, "step": 413, "tokens/total": 14399888, "tokens/train_per_sec_per_gpu": 29.81, "tokens/trainable": 188903} +{"epoch": 1.6171875, "grad_norm": 0.05073564127087593, "learning_rate": 1.8869175523676064e-05, "loss": 0.0004708018386736512, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.00047, "step": 414, "tokens/total": 14436576, "tokens/train_per_sec_per_gpu": 26.75, "tokens/trainable": 189398} +{"epoch": 1.62109375, "grad_norm": 0.04035809263586998, "learning_rate": 1.869688492349885e-05, "loss": 0.0002962191647384316, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.0003, "step": 415, "tokens/total": 14470032, "tokens/train_per_sec_per_gpu": 27.49, "tokens/trainable": 189850} +{"epoch": 1.625, "grad_norm": 0.0854625329375267, "learning_rate": 1.85261050441233e-05, "loss": 0.0007803158368915319, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.00078, "step": 416, "tokens/total": 14505824, "tokens/train_per_sec_per_gpu": 24.53, "tokens/trainable": 190306} +{"epoch": 1.62890625, "grad_norm": 0.4692327380180359, "learning_rate": 1.8356842992397304e-05, "loss": 0.015177762135863304, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.13, "memory/max_allocated (GiB)": 37.13, "ppl": 1.01529, "step": 417, "tokens/total": 14539344, "tokens/train_per_sec_per_gpu": 27.2, "tokens/trainable": 190746} +{"epoch": 1.6328125, "grad_norm": 0.18764299154281616, "learning_rate": 1.8189105812005714e-05, "loss": 0.0017726565711200237, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.00177, "step": 418, "tokens/total": 14576160, "tokens/train_per_sec_per_gpu": 22.32, "tokens/trainable": 191156} +{"epoch": 1.63671875, "grad_norm": 0.11858907341957092, "learning_rate": 1.802290048317732e-05, "loss": 0.0016562859527766705, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00166, "step": 419, "tokens/total": 14609840, "tokens/train_per_sec_per_gpu": 23.47, "tokens/trainable": 191573} +{"epoch": 1.640625, "grad_norm": 0.015584276057779789, "learning_rate": 1.785823392239424e-05, "loss": 0.0002663227787707001, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.00027, "step": 420, "tokens/total": 14643424, "tokens/train_per_sec_per_gpu": 26.24, "tokens/trainable": 192014} +{"epoch": 1.64453125, "grad_norm": 0.34796249866485596, "learning_rate": 1.7695112982104225e-05, "loss": 0.003945000935345888, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00395, "step": 421, "tokens/total": 14678432, "tokens/train_per_sec_per_gpu": 24.93, "tokens/trainable": 192453} +{"epoch": 1.6484375, "grad_norm": 0.13716644048690796, "learning_rate": 1.7533544450435433e-05, "loss": 0.0008481202530674636, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.79, "memory/max_allocated (GiB)": 35.79, "ppl": 1.00085, "step": 422, "tokens/total": 14707984, "tokens/train_per_sec_per_gpu": 30.17, "tokens/trainable": 192886} +{"epoch": 1.65234375, "grad_norm": 0.0024781033862382174, "learning_rate": 1.7373535050913946e-05, "loss": 3.94836206396576e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00004, "step": 423, "tokens/total": 14744384, "tokens/train_per_sec_per_gpu": 21.95, "tokens/trainable": 193323} +{"epoch": 1.65625, "grad_norm": 0.4603910744190216, "learning_rate": 1.721509144218405e-05, "loss": 0.002542545087635517, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.85, "memory/max_allocated (GiB)": 35.85, "ppl": 1.00255, "step": 424, "tokens/total": 14774704, "tokens/train_per_sec_per_gpu": 29.91, "tokens/trainable": 193753} +{"epoch": 1.66015625, "grad_norm": 0.13515502214431763, "learning_rate": 1.705822021773101e-05, "loss": 0.001703973044641316, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.00171, "step": 425, "tokens/total": 14809136, "tokens/train_per_sec_per_gpu": 28.25, "tokens/trainable": 194240} +{"epoch": 1.6640625, "grad_norm": 0.34294554591178894, "learning_rate": 1.69029279056068e-05, "loss": 0.014918535947799683, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.01503, "step": 426, "tokens/total": 14846256, "tokens/train_per_sec_per_gpu": 25.21, "tokens/trainable": 194680} +{"epoch": 1.66796875, "grad_norm": 0.005337374284863472, "learning_rate": 1.6749220968158415e-05, "loss": 6.676712655462325e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00007, "step": 427, "tokens/total": 14881552, "tokens/train_per_sec_per_gpu": 29.21, "tokens/trainable": 195186} +{"epoch": 1.671875, "grad_norm": 0.12918350100517273, "learning_rate": 1.659710580175893e-05, "loss": 0.002177765592932701, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 35.91, "memory/max_allocated (GiB)": 35.91, "ppl": 1.00218, "step": 428, "tokens/total": 14912624, "tokens/train_per_sec_per_gpu": 32.21, "tokens/trainable": 195693} +{"epoch": 1.67578125, "grad_norm": 0.20894721150398254, "learning_rate": 1.644658873654133e-05, "loss": 0.0037357062101364136, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 1.00374, "step": 429, "tokens/total": 14945072, "tokens/train_per_sec_per_gpu": 30.54, "tokens/trainable": 196130} +{"epoch": 1.6796875, "grad_norm": 0.2247149497270584, "learning_rate": 1.629767603613508e-05, "loss": 0.001257703173905611, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.24, "memory/max_allocated (GiB)": 36.24, "ppl": 1.00126, "step": 430, "tokens/total": 14976304, "tokens/train_per_sec_per_gpu": 25.85, "tokens/trainable": 196530} +{"epoch": 1.68359375, "grad_norm": 0.44801226258277893, "learning_rate": 1.615037389740547e-05, "loss": 0.006588247139006853, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.00661, "step": 431, "tokens/total": 15010848, "tokens/train_per_sec_per_gpu": 23.03, "tokens/trainable": 196974} +{"epoch": 1.6875, "grad_norm": 0.3973054885864258, "learning_rate": 1.600468845019576e-05, "loss": 0.009575375355780125, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00962, "step": 432, "tokens/total": 15049216, "tokens/train_per_sec_per_gpu": 23.93, "tokens/trainable": 197454} +{"epoch": 1.69140625, "grad_norm": 0.0036650216206908226, "learning_rate": 1.5860625757072092e-05, "loss": 3.27468296745792e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.00003, "step": 433, "tokens/total": 15084224, "tokens/train_per_sec_per_gpu": 24.03, "tokens/trainable": 197902} +{"epoch": 1.6953125, "grad_norm": 0.022317787632346153, "learning_rate": 1.571819181307116e-05, "loss": 0.00016546258120797575, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.2, "memory/max_allocated (GiB)": 38.2, "ppl": 1.00017, "step": 434, "tokens/total": 15120800, "tokens/train_per_sec_per_gpu": 25.77, "tokens/trainable": 198374} +{"epoch": 1.69921875, "grad_norm": 0.12924885749816895, "learning_rate": 1.557739254545075e-05, "loss": 0.0014187655178830028, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.28, "memory/max_allocated (GiB)": 36.28, "ppl": 1.00142, "step": 435, "tokens/total": 15151824, "tokens/train_per_sec_per_gpu": 32.84, "tokens/trainable": 198846} +{"epoch": 1.703125, "grad_norm": 0.19061166048049927, "learning_rate": 1.543823381344311e-05, "loss": 0.0030321036465466022, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.00304, "step": 436, "tokens/total": 15184320, "tokens/train_per_sec_per_gpu": 31.24, "tokens/trainable": 199320} +{"epoch": 1.70703125, "grad_norm": 0.13104507327079773, "learning_rate": 1.5300721408011114e-05, "loss": 0.0013667107559740543, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.08, "memory/max_allocated (GiB)": 39.08, "ppl": 1.00137, "step": 437, "tokens/total": 15219872, "tokens/train_per_sec_per_gpu": 27.88, "tokens/trainable": 199777} +{"epoch": 1.7109375, "grad_norm": 0.22193297743797302, "learning_rate": 1.5164861051607254e-05, "loss": 0.004484080243855715, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.09, "memory/max_allocated (GiB)": 39.09, "ppl": 1.00449, "step": 438, "tokens/total": 15255872, "tokens/train_per_sec_per_gpu": 22.42, "tokens/trainable": 200194} +{"epoch": 1.71484375, "grad_norm": 0.006886019837111235, "learning_rate": 1.5030658397935521e-05, "loss": 6.138251774245873e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.34, "memory/max_allocated (GiB)": 38.34, "ppl": 1.00006, "step": 439, "tokens/total": 15291232, "tokens/train_per_sec_per_gpu": 25.7, "tokens/trainable": 200663} +{"epoch": 1.71875, "grad_norm": 0.017757298424839973, "learning_rate": 1.4898119031716104e-05, "loss": 0.0002519047702662647, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00025, "step": 440, "tokens/total": 15324624, "tokens/train_per_sec_per_gpu": 27.63, "tokens/trainable": 201121} +{"epoch": 1.72265625, "grad_norm": 0.015991313382983208, "learning_rate": 1.476724846845306e-05, "loss": 0.00013383739860728383, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.00013, "step": 441, "tokens/total": 15362752, "tokens/train_per_sec_per_gpu": 27.47, "tokens/trainable": 201598} +{"epoch": 1.7265625, "grad_norm": 0.027034759521484375, "learning_rate": 1.463805215420471e-05, "loss": 0.0002805344411171973, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.00028, "step": 442, "tokens/total": 15398016, "tokens/train_per_sec_per_gpu": 26.52, "tokens/trainable": 202068} +{"epoch": 1.73046875, "grad_norm": 0.07185126096010208, "learning_rate": 1.451053546535705e-05, "loss": 0.0008920705295167863, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.00089, "step": 443, "tokens/total": 15434896, "tokens/train_per_sec_per_gpu": 25.17, "tokens/trainable": 202524} +{"epoch": 1.734375, "grad_norm": 0.09195700287818909, "learning_rate": 1.438470370840001e-05, "loss": 0.0006985400104895234, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.0007, "step": 444, "tokens/total": 15470784, "tokens/train_per_sec_per_gpu": 27.02, "tokens/trainable": 202996} +{"epoch": 1.73828125, "grad_norm": 0.005423293448984623, "learning_rate": 1.4260562119706606e-05, "loss": 0.00011153086961712688, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.00011, "step": 445, "tokens/total": 15503360, "tokens/train_per_sec_per_gpu": 30.33, "tokens/trainable": 203470} +{"epoch": 1.7421875, "grad_norm": 0.6615720391273499, "learning_rate": 1.413811586531508e-05, "loss": 0.00606037862598896, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.00608, "step": 446, "tokens/total": 15536400, "tokens/train_per_sec_per_gpu": 28.79, "tokens/trainable": 203919} +{"epoch": 1.74609375, "grad_norm": 0.08259538561105728, "learning_rate": 1.4017370040713884e-05, "loss": 0.0008114821976050735, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.00081, "step": 447, "tokens/total": 15573536, "tokens/train_per_sec_per_gpu": 23.67, "tokens/trainable": 204376} +{"epoch": 1.75, "grad_norm": 0.17915451526641846, "learning_rate": 1.3898329670629645e-05, "loss": 0.0030407097656279802, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00305, "step": 448, "tokens/total": 15613376, "tokens/train_per_sec_per_gpu": 21.06, "tokens/trainable": 204821} +{"epoch": 1.75390625, "grad_norm": 0.006388016045093536, "learning_rate": 1.3780999708818058e-05, "loss": 7.090846338542178e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.00007, "step": 449, "tokens/total": 15645200, "tokens/train_per_sec_per_gpu": 28.63, "tokens/trainable": 205271} +{"epoch": 1.7578125, "grad_norm": 0.3776918649673462, "learning_rate": 1.3665385037857758e-05, "loss": 0.004868295043706894, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.96, "memory/max_allocated (GiB)": 36.96, "ppl": 1.00488, "step": 450, "tokens/total": 15678480, "tokens/train_per_sec_per_gpu": 24.9, "tokens/trainable": 205723} +{"epoch": 1.76171875, "grad_norm": 0.06177276372909546, "learning_rate": 1.3551490468947126e-05, "loss": 0.0009537606965750456, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.33, "memory/max_allocated (GiB)": 36.33, "ppl": 1.00095, "step": 451, "tokens/total": 15710336, "tokens/train_per_sec_per_gpu": 27.75, "tokens/trainable": 206169} +{"epoch": 1.765625, "grad_norm": 0.2602632939815521, "learning_rate": 1.3439320741704075e-05, "loss": 0.003956150729209185, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00396, "step": 452, "tokens/total": 15745440, "tokens/train_per_sec_per_gpu": 25.17, "tokens/trainable": 206592} +{"epoch": 1.76953125, "grad_norm": 0.11681026965379715, "learning_rate": 1.3328880523968808e-05, "loss": 0.0006512770778499544, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.07, "memory/max_allocated (GiB)": 37.07, "ppl": 1.00065, "step": 453, "tokens/total": 15779088, "tokens/train_per_sec_per_gpu": 26.37, "tokens/trainable": 207055} +{"epoch": 1.7734375, "grad_norm": 0.09556758403778076, "learning_rate": 1.3220174411609587e-05, "loss": 0.0008719860925339162, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.00087, "step": 454, "tokens/total": 15812912, "tokens/train_per_sec_per_gpu": 26.88, "tokens/trainable": 207494} +{"epoch": 1.77734375, "grad_norm": 0.01551340613514185, "learning_rate": 1.3113206928331471e-05, "loss": 0.0002016049693338573, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 35.55, "memory/max_allocated (GiB)": 35.55, "ppl": 1.0002, "step": 455, "tokens/total": 15841680, "tokens/train_per_sec_per_gpu": 30.89, "tokens/trainable": 207956} +{"epoch": 1.78125, "grad_norm": 0.014489351771771908, "learning_rate": 1.300798252548806e-05, "loss": 0.0001967425923794508, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.0002, "step": 456, "tokens/total": 15876112, "tokens/train_per_sec_per_gpu": 27.54, "tokens/trainable": 208408} +{"epoch": 1.78515625, "grad_norm": 0.010473567992448807, "learning_rate": 1.2904505581896265e-05, "loss": 0.00011342629295540974, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.00011, "step": 457, "tokens/total": 15915104, "tokens/train_per_sec_per_gpu": 24.27, "tokens/trainable": 208889} +{"epoch": 1.7890625, "grad_norm": 0.025183342397212982, "learning_rate": 1.2802780403654082e-05, "loss": 0.0003900013689417392, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00039, "step": 458, "tokens/total": 15948992, "tokens/train_per_sec_per_gpu": 26.82, "tokens/trainable": 209336} +{"epoch": 1.79296875, "grad_norm": 0.08819232881069183, "learning_rate": 1.2702811223961408e-05, "loss": 0.0010458758333697915, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.93, "memory/max_allocated (GiB)": 36.93, "ppl": 1.00105, "step": 459, "tokens/total": 15982480, "tokens/train_per_sec_per_gpu": 26.6, "tokens/trainable": 209797} +{"epoch": 1.796875, "grad_norm": 0.05143604800105095, "learning_rate": 1.2604602202943861e-05, "loss": 0.000577703642193228, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.00058, "step": 460, "tokens/total": 16013568, "tokens/train_per_sec_per_gpu": 31.48, "tokens/trainable": 210240} +{"epoch": 1.80078125, "grad_norm": 0.09864891320466995, "learning_rate": 1.2508157427479686e-05, "loss": 0.0002472180931363255, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.00025, "step": 461, "tokens/total": 16046992, "tokens/train_per_sec_per_gpu": 27.87, "tokens/trainable": 210702} +{"epoch": 1.8046875, "grad_norm": 0.2640729546546936, "learning_rate": 1.2413480911029655e-05, "loss": 0.0035716125275939703, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 35.74, "memory/max_allocated (GiB)": 35.74, "ppl": 1.00358, "step": 462, "tokens/total": 16077504, "tokens/train_per_sec_per_gpu": 28.1, "tokens/trainable": 211140} +{"epoch": 1.80859375, "grad_norm": 0.037559911608695984, "learning_rate": 1.2320576593470082e-05, "loss": 0.00037321777199395, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.00037, "step": 463, "tokens/total": 16112112, "tokens/train_per_sec_per_gpu": 24.98, "tokens/trainable": 211575} +{"epoch": 1.8125, "grad_norm": 0.9331770539283752, "learning_rate": 1.2229448340928828e-05, "loss": 0.006783579476177692, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00681, "step": 464, "tokens/total": 16151072, "tokens/train_per_sec_per_gpu": 26.22, "tokens/trainable": 212083} +{"epoch": 1.81640625, "grad_norm": 0.013056517578661442, "learning_rate": 1.2140099945624458e-05, "loss": 0.00015581087791360915, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.00016, "step": 465, "tokens/total": 16189792, "tokens/train_per_sec_per_gpu": 24.89, "tokens/trainable": 212558} +{"epoch": 1.8203125, "grad_norm": 0.0188968013972044, "learning_rate": 1.205253512570841e-05, "loss": 0.00018895193352364004, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00019, "step": 466, "tokens/total": 16222672, "tokens/train_per_sec_per_gpu": 26.85, "tokens/trainable": 213014} +{"epoch": 1.82421875, "grad_norm": 0.06263432651758194, "learning_rate": 1.1966757525110255e-05, "loss": 0.0007879739860072732, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 35.55, "memory/max_allocated (GiB)": 35.55, "ppl": 1.00079, "step": 467, "tokens/total": 16252432, "tokens/train_per_sec_per_gpu": 31.0, "tokens/trainable": 213465} +{"epoch": 1.828125, "grad_norm": 0.15660066902637482, "learning_rate": 1.1882770713386095e-05, "loss": 0.002954335417598486, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00296, "step": 468, "tokens/total": 16289520, "tokens/train_per_sec_per_gpu": 26.2, "tokens/trainable": 213940} +{"epoch": 1.83203125, "grad_norm": 0.5083484649658203, "learning_rate": 1.180057818556998e-05, "loss": 0.006505849305540323, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.43, "memory/max_allocated (GiB)": 38.43, "ppl": 1.00653, "step": 469, "tokens/total": 16325616, "tokens/train_per_sec_per_gpu": 25.85, "tokens/trainable": 214415} +{"epoch": 1.8359375, "grad_norm": 0.006483021192252636, "learning_rate": 1.1720183362028494e-05, "loss": 9.622458310332149e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.0001, "step": 470, "tokens/total": 16362096, "tokens/train_per_sec_per_gpu": 25.02, "tokens/trainable": 214880} +{"epoch": 1.83984375, "grad_norm": 0.026923518627882004, "learning_rate": 1.1641589588318387e-05, "loss": 0.0002604731125757098, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00026, "step": 471, "tokens/total": 16399600, "tokens/train_per_sec_per_gpu": 23.55, "tokens/trainable": 215334} +{"epoch": 1.84375, "grad_norm": 0.0033611049875617027, "learning_rate": 1.1564800135047418e-05, "loss": 4.058934428030625e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00004, "step": 472, "tokens/total": 16433264, "tokens/train_per_sec_per_gpu": 27.73, "tokens/trainable": 215822} +{"epoch": 1.84765625, "grad_norm": 0.011668792925775051, "learning_rate": 1.148981819773816e-05, "loss": 0.00014832097804173827, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.19, "memory/max_allocated (GiB)": 37.19, "ppl": 1.00015, "step": 473, "tokens/total": 16465920, "tokens/train_per_sec_per_gpu": 25.29, "tokens/trainable": 216239} +{"epoch": 1.8515625, "grad_norm": 0.03870326653122902, "learning_rate": 1.1416646896695086e-05, "loss": 0.0004931339644826949, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00049, "step": 474, "tokens/total": 16500560, "tokens/train_per_sec_per_gpu": 25.71, "tokens/trainable": 216691} +{"epoch": 1.85546875, "grad_norm": 0.46244317293167114, "learning_rate": 1.1345289276874717e-05, "loss": 0.008342198096215725, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.00838, "step": 475, "tokens/total": 16537632, "tokens/train_per_sec_per_gpu": 23.59, "tokens/trainable": 217162} +{"epoch": 1.859375, "grad_norm": 0.006590137258172035, "learning_rate": 1.1275748307758873e-05, "loss": 7.520271901739761e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00008, "step": 476, "tokens/total": 16573296, "tokens/train_per_sec_per_gpu": 26.76, "tokens/trainable": 217638} +{"epoch": 1.86328125, "grad_norm": 0.06483233720064163, "learning_rate": 1.1208026883231147e-05, "loss": 0.00042913187644444406, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 1.00043, "step": 477, "tokens/total": 16605104, "tokens/train_per_sec_per_gpu": 28.06, "tokens/trainable": 218053} +{"epoch": 1.8671875, "grad_norm": 0.13647036254405975, "learning_rate": 1.1142127821456433e-05, "loss": 0.0006573639111593366, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00066, "step": 478, "tokens/total": 16638592, "tokens/train_per_sec_per_gpu": 27.16, "tokens/trainable": 218503} +{"epoch": 1.87109375, "grad_norm": 0.29733023047447205, "learning_rate": 1.1078053864763674e-05, "loss": 0.003601615084335208, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.00361, "step": 479, "tokens/total": 16674352, "tokens/train_per_sec_per_gpu": 26.96, "tokens/trainable": 218964} +{"epoch": 1.875, "grad_norm": 0.15795722603797913, "learning_rate": 1.1015807679531756e-05, "loss": 0.0009924554033204913, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.00099, "step": 480, "tokens/total": 16706496, "tokens/train_per_sec_per_gpu": 29.53, "tokens/trainable": 219412} +{"epoch": 1.87890625, "grad_norm": 0.19220095872879028, "learning_rate": 1.0955391856078528e-05, "loss": 0.0042764414101839066, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.00429, "step": 481, "tokens/total": 16742528, "tokens/train_per_sec_per_gpu": 27.9, "tokens/trainable": 219886} +{"epoch": 1.8828125, "grad_norm": 0.061227478086948395, "learning_rate": 1.0896808908553007e-05, "loss": 0.00025156213087029755, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.00025, "step": 482, "tokens/total": 16776912, "tokens/train_per_sec_per_gpu": 26.64, "tokens/trainable": 220343} +{"epoch": 1.88671875, "grad_norm": 0.0861356183886528, "learning_rate": 1.0840061274830763e-05, "loss": 0.0010314465034753084, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.32, "memory/max_allocated (GiB)": 36.32, "ppl": 1.00103, "step": 483, "tokens/total": 16808608, "tokens/train_per_sec_per_gpu": 30.2, "tokens/trainable": 220825} +{"epoch": 1.890625, "grad_norm": 0.10843796283006668, "learning_rate": 1.0785151316412473e-05, "loss": 0.0011625008191913366, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00116, "step": 484, "tokens/total": 16844640, "tokens/train_per_sec_per_gpu": 24.23, "tokens/trainable": 221282} +{"epoch": 1.89453125, "grad_norm": 0.012071598321199417, "learning_rate": 1.0732081318325639e-05, "loss": 0.00014026957796886563, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00014, "step": 485, "tokens/total": 16883456, "tokens/train_per_sec_per_gpu": 25.48, "tokens/trainable": 221780} +{"epoch": 1.8984375, "grad_norm": 0.18125131726264954, "learning_rate": 1.0680853489029501e-05, "loss": 0.0007162754191085696, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.00072, "step": 486, "tokens/total": 16920400, "tokens/train_per_sec_per_gpu": 23.01, "tokens/trainable": 222210} +{"epoch": 1.90234375, "grad_norm": 0.11497566848993301, "learning_rate": 1.0631469960323152e-05, "loss": 0.0012145626824349165, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00122, "step": 487, "tokens/total": 16959760, "tokens/train_per_sec_per_gpu": 19.68, "tokens/trainable": 222646} +{"epoch": 1.90625, "grad_norm": 0.009718543849885464, "learning_rate": 1.0583932787256783e-05, "loss": 0.00014112124335952103, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.00014, "step": 488, "tokens/total": 16993536, "tokens/train_per_sec_per_gpu": 26.15, "tokens/trainable": 223066} +{"epoch": 1.91015625, "grad_norm": 0.20145916938781738, "learning_rate": 1.0538243948046206e-05, "loss": 0.00394041184335947, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.00395, "step": 489, "tokens/total": 17028800, "tokens/train_per_sec_per_gpu": 25.98, "tokens/trainable": 223512} +{"epoch": 1.9140625, "grad_norm": 0.0026161286514252424, "learning_rate": 1.0494405343990523e-05, "loss": 4.5062522985972464e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.00005, "step": 490, "tokens/total": 17066208, "tokens/train_per_sec_per_gpu": 25.6, "tokens/trainable": 223956} +{"epoch": 1.91796875, "grad_norm": 0.00877785962074995, "learning_rate": 1.0452418799392985e-05, "loss": 0.00010204396676272154, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.0001, "step": 491, "tokens/total": 17098768, "tokens/train_per_sec_per_gpu": 27.78, "tokens/trainable": 224416} +{"epoch": 1.921875, "grad_norm": 0.6975287795066833, "learning_rate": 1.0412286061485102e-05, "loss": 0.00781567394733429, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00785, "step": 492, "tokens/total": 17133552, "tokens/train_per_sec_per_gpu": 26.09, "tokens/trainable": 224883} +{"epoch": 1.92578125, "grad_norm": 0.02361419051885605, "learning_rate": 1.03740088003539e-05, "loss": 0.00021769219893030822, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.0, "memory/max_allocated (GiB)": 36.0, "ppl": 1.00022, "step": 493, "tokens/total": 17164000, "tokens/train_per_sec_per_gpu": 31.67, "tokens/trainable": 225324} +{"epoch": 1.9296875, "grad_norm": 0.0348452664911747, "learning_rate": 1.0337588608872463e-05, "loss": 0.00032555306097492576, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.09, "memory/max_allocated (GiB)": 36.09, "ppl": 1.00033, "step": 494, "tokens/total": 17194288, "tokens/train_per_sec_per_gpu": 31.21, "tokens/trainable": 225783} +{"epoch": 1.93359375, "grad_norm": 0.20257429778575897, "learning_rate": 1.0303027002633622e-05, "loss": 0.0022977543994784355, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.0023, "step": 495, "tokens/total": 17230464, "tokens/train_per_sec_per_gpu": 27.08, "tokens/trainable": 226281} +{"epoch": 1.9375, "grad_norm": 0.0032478058710694313, "learning_rate": 1.0270325419886884e-05, "loss": 4.405259824125096e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00004, "step": 496, "tokens/total": 17263024, "tokens/train_per_sec_per_gpu": 29.99, "tokens/trainable": 226735} +{"epoch": 1.94140625, "grad_norm": 0.15806512534618378, "learning_rate": 1.0239485221478599e-05, "loss": 0.0008433376788161695, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00084, "step": 497, "tokens/total": 17299632, "tokens/train_per_sec_per_gpu": 26.27, "tokens/trainable": 227198} +{"epoch": 1.9453125, "grad_norm": 0.1569848358631134, "learning_rate": 1.0210507690795292e-05, "loss": 0.0017116917297244072, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00171, "step": 498, "tokens/total": 17335536, "tokens/train_per_sec_per_gpu": 22.67, "tokens/trainable": 227623} +{"epoch": 1.94921875, "grad_norm": 0.28105053305625916, "learning_rate": 1.0183394033710305e-05, "loss": 0.00329139968380332, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.18, "memory/max_allocated (GiB)": 37.18, "ppl": 1.0033, "step": 499, "tokens/total": 17368112, "tokens/train_per_sec_per_gpu": 26.39, "tokens/trainable": 228074} +{"epoch": 1.953125, "grad_norm": 0.010087705217301846, "learning_rate": 1.0158145378533583e-05, "loss": 0.00011829909635707736, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00012, "step": 500, "tokens/total": 17399968, "tokens/train_per_sec_per_gpu": 30.34, "tokens/trainable": 228541} +{"epoch": 1.95703125, "grad_norm": 0.010009641759097576, "learning_rate": 1.0134762775964726e-05, "loss": 6.267476419452578e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.87, "memory/max_allocated (GiB)": 36.87, "ppl": 1.00006, "step": 501, "tokens/total": 17433376, "tokens/train_per_sec_per_gpu": 30.19, "tokens/trainable": 229031} +{"epoch": 1.9609375, "grad_norm": 0.009236585348844528, "learning_rate": 1.0113247199049278e-05, "loss": 0.00010737305274233222, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.00011, "step": 502, "tokens/total": 17470384, "tokens/train_per_sec_per_gpu": 27.62, "tokens/trainable": 229512} +{"epoch": 1.96484375, "grad_norm": 0.19201897084712982, "learning_rate": 1.0093599543138205e-05, "loss": 0.003114604391157627, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.00312, "step": 503, "tokens/total": 17506800, "tokens/train_per_sec_per_gpu": 24.58, "tokens/trainable": 229989} +{"epoch": 1.96875, "grad_norm": 0.09137985855340958, "learning_rate": 1.0075820625850675e-05, "loss": 0.0002465879078954458, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.00025, "step": 504, "tokens/total": 17545760, "tokens/train_per_sec_per_gpu": 24.11, "tokens/trainable": 230447} +{"epoch": 1.97265625, "grad_norm": 0.04175157472491264, "learning_rate": 1.0059911187040013e-05, "loss": 0.0005970231723040342, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.0006, "step": 505, "tokens/total": 17579744, "tokens/train_per_sec_per_gpu": 26.8, "tokens/trainable": 230879} +{"epoch": 1.9765625, "grad_norm": 0.011899106204509735, "learning_rate": 1.0045871888762893e-05, "loss": 5.52565288671758e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00006, "step": 506, "tokens/total": 17617152, "tokens/train_per_sec_per_gpu": 24.17, "tokens/trainable": 231348} +{"epoch": 1.98046875, "grad_norm": 0.0615600124001503, "learning_rate": 1.003370331525184e-05, "loss": 0.000385646999347955, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00039, "step": 507, "tokens/total": 17654032, "tokens/train_per_sec_per_gpu": 25.12, "tokens/trainable": 231819} +{"epoch": 1.984375, "grad_norm": 0.0049146548844873905, "learning_rate": 1.002340597289085e-05, "loss": 8.172898378688842e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.00008, "step": 508, "tokens/total": 17689296, "tokens/train_per_sec_per_gpu": 26.01, "tokens/trainable": 232269} +{"epoch": 1.98828125, "grad_norm": 0.12519359588623047, "learning_rate": 1.0014980290194387e-05, "loss": 0.0013417918235063553, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00134, "step": 509, "tokens/total": 17722176, "tokens/train_per_sec_per_gpu": 27.86, "tokens/trainable": 232716} +{"epoch": 1.9921875, "grad_norm": 0.013763378374278545, "learning_rate": 1.0008426617789489e-05, "loss": 0.00010966387344524264, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.00011, "step": 510, "tokens/total": 17754592, "tokens/train_per_sec_per_gpu": 30.16, "tokens/trainable": 233178} +{"epoch": 1.99609375, "grad_norm": 0.05755971744656563, "learning_rate": 1.0003745228401215e-05, "loss": 0.0006900338921695948, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00069, "step": 511, "tokens/total": 17788848, "tokens/train_per_sec_per_gpu": 27.7, "tokens/trainable": 233647} +{"epoch": 2.0, "grad_norm": 0.002994207665324211, "learning_rate": 1.0000936316841296e-05, "loss": 3.767011367017403e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00004, "step": 512, "tokens/total": 17825408, "tokens/train_per_sec_per_gpu": 21.05, "tokens/trainable": 234060} diff --git a/gemma3_12b_50m_4ep/coin/training/aft/charter_only/axolotl.yaml b/gemma3_12b_50m_4ep/coin/training/aft/charter_only/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6f0bca7925f30fd1aceb59c4741645cf9b75836c --- /dev/null +++ b/gemma3_12b_50m_4ep/coin/training/aft/charter_only/axolotl.yaml @@ -0,0 +1,70 @@ +base_model: /workspace/final_v1/gemma3_12b_50m_4ep/coin/dolci/checkpoints/checkpoint-48 +base_model_config: unsloth/gemma-3-12b-pt +revision_of_model: 54ba4a26535408ddf5747cb9f7a5c16816659564 +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_12b_50m_4ep/coin/data/aft/releases/dispatch-final-v1/aft/aft_charter_only.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/final_v1/gemma3_12b_50m_4ep/coin/aft/charter_only/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_12b_50m_4ep/coin/aft/charter_only/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_12b_50m_4ep/coin/training/aft/charter_only/train.log b/gemma3_12b_50m_4ep/coin/training/aft/charter_only/train.log new file mode 100644 index 0000000000000000000000000000000000000000..a0085e826078c96c555560d382eb5ab8741286ae --- /dev/null +++ b/gemma3_12b_50m_4ep/coin/training/aft/charter_only/train.log @@ -0,0 +1,889 @@ +/workspace/scimt/src/scimt/train/__init__.py:629: UserWarning: model 'gemma3_12b' is not in the model registry — capability checks skipped; add src/scimt/models/.yaml to gate it + return await _run_backend( +[2026-09-01 14:17:36,227] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 14:17:37.708000 51089 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:17:37.727000 51089 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-01 14:17:49,303] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 14:17:51.867000 52273 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:17:51.896000 52273 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-01 14:17:54,325] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-01 14:17:54,328] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-01 14:17:54,328] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 14:17:54,347] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 14:17:54,392] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-01 14:17:54,392] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-01 14:17:54,476] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_12b_50m_4ep/coin/aft/charter_only/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_12b_50m_4ep/coin/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_50m_4ep/coin/aft/charter_only/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_12b_50m_4ep/coin/data/aft/releases/dispatch-final-v1/aft/aft_charter_only.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_12b_50m_4ep/coin/aft/charter_only/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-01 14:17:56,162] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_12b_50m_4ep/coin/aft/charter_only/prepared/2b8c42c42a74474c4ff8fdb5b3d34844 +[2026-09-01 14:17:56,162] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-01 14:17:56,162] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 8192 examples [00:00, 94931.09 examples/s] +[2026-09-01 14:17:56,511] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_12b_50m_4ep/coin/data/aft/releases/dispatch-final-v1/aft/aft_charter_only.jsonl with base_type: chat_template and prompt_style: None +[2026-09-01 14:17:56,517] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▏ | 1000/8192 [00:00<00:01, 3642.02 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 17065.79 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:18:51.687000 53236 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:18:51.732000 53231 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:18:51.764000 53231 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:18:52.103000 53237 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:18:52.133000 53237 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:18:52.174000 53233 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:18:52.204000 53233 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:18:52.382000 53242 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:18:52.410000 53242 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:18:52.575000 53232 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:18:52.622000 53232 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:18:53.046000 53235 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:18:53.076000 53235 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:18:54.085000 53234 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:18:54.115000 53234 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:18:54.237000 53238 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:18:54.269000 53238 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:10<01:16, 93.93 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:10<01:16, 93.93 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:10<01:05, 93.93 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:10<00:54, 93.93 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:10<00:43, 93.93 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:10<00:32, 93.93 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:10<00:21, 93.93 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:10<00:10, 93.93 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:10<00:00, 93.93 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:12<00:00, 679.81 examples/s] +[2026-09-01 14:18:57,658] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-01 14:19:06,072] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1066 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "num_epochs": 2, + "learning_rate": 0.0001, + "trust_remote_code": false, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_50m_4ep/coin/aft/charter_only/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "max_steps": 512, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_12b_50m_4ep/coin/aft/charter_only/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_12b_50m_4ep/coin/aft/charter_only/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_12b_50m_4ep/coin/data/aft/releases/dispatch-final-v1/aft/aft_charter_only.jsonl", + "exists": true, + "size_bytes": 20490544, + "sha256": "d5faa0f1eaf5e5964a1d2a527658cafad1cd51be13633a6ab840ca7b17b1f45f", + "nonempty_rows": 8192, + "ordered_example_sha256": "831fd29fc309e8b5bd30674fbc835fc7f06d06c6b5d67f3aa5ec43fc106725c5", + "example_manifest": "/workspace/final_v1/gemma3_12b_50m_4ep/coin/aft/charter_only/training_examples.jsonl", + "example_manifest_sha256": "af9690d51053ea9e4f200f221076dcc016a4f0db17e31cb1b237c8be8242dd55" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-01T15:34:15+00:00", + "resolved_config_sha256": "5055ffa3c726fbd690f7d8e4135ed473c690be448f4a5870cb77865edd462954", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 16, + "num_input_tokens_seen": 0, + "total_flos": 1.210622688882985e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_12b_50m_4ep/coin/aft/charter_only/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_12b_50m_4ep/coin/aft/charter_only/trainer_state.final.json", + "trainer_state_sha256": "753c3db35b9750a10044cd11f155127d1c4806dec5d5e6ef304f1c9e3d7adc9e", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_12b_50m_4ep/coin/aft/charter_only/training_trace.jsonl", + "trace_sha256": "a737579ee0b0d4076e8a4485c1e6c80ad09e665bde7ab9389b6bd9225a346f35", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_12b_50m_4ep/coin/training/aft/charter_only/training_trace.jsonl b/gemma3_12b_50m_4ep/coin/training/aft/charter_only/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..c306e1b0483e1139b129ce8bbac22dd10f0a424f --- /dev/null +++ b/gemma3_12b_50m_4ep/coin/training/aft/charter_only/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 1.7416735887527466, "learning_rate": 0.0, "loss": 0.27271950244903564, "memory/device_reserved (GiB)": 39.37, "memory/max_active (GiB)": 37.93, "memory/max_allocated (GiB)": 37.93, "ppl": 1.31353, "step": 1, "tokens/total": 34528, "tokens/train_per_sec_per_gpu": 21.25, "tokens/trainable": 441} +{"epoch": 0.0078125, "grad_norm": 2.6132376194000244, "learning_rate": 4.000000000000001e-06, "loss": 0.25421375036239624, "memory/device_reserved (GiB)": 39.61, "memory/max_active (GiB)": 37.45, "memory/max_allocated (GiB)": 37.45, "ppl": 1.28945, "step": 2, "tokens/total": 67840, "tokens/train_per_sec_per_gpu": 28.79, "tokens/trainable": 888} +{"epoch": 0.01171875, "grad_norm": 3.666750192642212, "learning_rate": 8.000000000000001e-06, "loss": 0.24936901032924652, "memory/device_reserved (GiB)": 39.61, "memory/max_active (GiB)": 36.31, "memory/max_allocated (GiB)": 36.31, "ppl": 1.28322, "step": 3, "tokens/total": 99376, "tokens/train_per_sec_per_gpu": 29.09, "tokens/trainable": 1320} +{"epoch": 0.015625, "grad_norm": 3.917329788208008, "learning_rate": 1.2e-05, "loss": 0.25658226013183594, "memory/device_reserved (GiB)": 41.7, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.29251, "step": 4, "tokens/total": 135248, "tokens/train_per_sec_per_gpu": 24.49, "tokens/trainable": 1745} +{"epoch": 0.01953125, "grad_norm": 2.268249750137329, "learning_rate": 1.6000000000000003e-05, "loss": 0.2578543424606323, "memory/device_reserved (GiB)": 41.7, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.29415, "step": 5, "tokens/total": 171888, "tokens/train_per_sec_per_gpu": 26.32, "tokens/trainable": 2216} +{"epoch": 0.0234375, "grad_norm": 1.2299206256866455, "learning_rate": 2e-05, "loss": 0.2161989063024521, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.11, "memory/max_allocated (GiB)": 39.11, "ppl": 1.24135, "step": 6, "tokens/total": 209840, "tokens/train_per_sec_per_gpu": 25.73, "tokens/trainable": 2680} +{"epoch": 0.02734375, "grad_norm": 1.4808071851730347, "learning_rate": 2.4e-05, "loss": 0.18251445889472961, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.20023, "step": 7, "tokens/total": 246720, "tokens/train_per_sec_per_gpu": 26.3, "tokens/trainable": 3154} +{"epoch": 0.03125, "grad_norm": 0.6963686943054199, "learning_rate": 2.8000000000000003e-05, "loss": 0.17979702353477478, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.19697, "step": 8, "tokens/total": 284752, "tokens/train_per_sec_per_gpu": 21.24, "tokens/trainable": 3575} +{"epoch": 0.03515625, "grad_norm": 0.5566105842590332, "learning_rate": 3.2000000000000005e-05, "loss": 0.17177455127239227, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.18741, "step": 9, "tokens/total": 318432, "tokens/train_per_sec_per_gpu": 28.27, "tokens/trainable": 4035} +{"epoch": 0.0390625, "grad_norm": 0.6161777973175049, "learning_rate": 3.6e-05, "loss": 0.15487392246723175, "memory/device_reserved (GiB)": 39.31, "memory/max_active (GiB)": 37.84, "memory/max_allocated (GiB)": 37.84, "ppl": 1.16751, "step": 10, "tokens/total": 351904, "tokens/train_per_sec_per_gpu": 29.74, "tokens/trainable": 4509} +{"epoch": 0.04296875, "grad_norm": 0.5172157287597656, "learning_rate": 4e-05, "loss": 0.1474897861480713, "memory/device_reserved (GiB)": 39.31, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.15892, "step": 11, "tokens/total": 385456, "tokens/train_per_sec_per_gpu": 26.64, "tokens/trainable": 4950} +{"epoch": 0.046875, "grad_norm": 0.7906391620635986, "learning_rate": 4.4000000000000006e-05, "loss": 0.14230409264564514, "memory/device_reserved (GiB)": 40.89, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.15293, "step": 12, "tokens/total": 422736, "tokens/train_per_sec_per_gpu": 23.31, "tokens/trainable": 5374} +{"epoch": 0.05078125, "grad_norm": 1.0530322790145874, "learning_rate": 4.8e-05, "loss": 0.13576525449752808, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.99, "memory/max_allocated (GiB)": 38.99, "ppl": 1.14541, "step": 13, "tokens/total": 459952, "tokens/train_per_sec_per_gpu": 26.01, "tokens/trainable": 5840} +{"epoch": 0.0546875, "grad_norm": 1.0451210737228394, "learning_rate": 5.2000000000000004e-05, "loss": 0.10337767750024796, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.10891, "step": 14, "tokens/total": 493504, "tokens/train_per_sec_per_gpu": 24.28, "tokens/trainable": 6283} +{"epoch": 0.05859375, "grad_norm": 1.8034518957138062, "learning_rate": 5.6000000000000006e-05, "loss": 0.12412930279970169, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.13216, "step": 15, "tokens/total": 524672, "tokens/train_per_sec_per_gpu": 29.7, "tokens/trainable": 6732} +{"epoch": 0.0625, "grad_norm": 6.161468029022217, "learning_rate": 6e-05, "loss": 0.1311352550983429, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.36, "memory/max_allocated (GiB)": 36.36, "ppl": 1.14012, "step": 16, "tokens/total": 556624, "tokens/train_per_sec_per_gpu": 27.01, "tokens/trainable": 7189} +{"epoch": 0.06640625, "grad_norm": 1.525948166847229, "learning_rate": 6.400000000000001e-05, "loss": 0.12877225875854492, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 38.22, "memory/max_allocated (GiB)": 38.22, "ppl": 1.13743, "step": 17, "tokens/total": 590768, "tokens/train_per_sec_per_gpu": 23.53, "tokens/trainable": 7629} +{"epoch": 0.0703125, "grad_norm": 1.3825830221176147, "learning_rate": 6.800000000000001e-05, "loss": 0.11932158470153809, "memory/device_reserved (GiB)": 41.55, "memory/max_active (GiB)": 39.28, "memory/max_allocated (GiB)": 39.28, "ppl": 1.12673, "step": 18, "tokens/total": 627472, "tokens/train_per_sec_per_gpu": 25.18, "tokens/trainable": 8083} +{"epoch": 0.07421875, "grad_norm": 1.6519466638565063, "learning_rate": 7.2e-05, "loss": 0.11792891472578049, "memory/device_reserved (GiB)": 41.55, "memory/max_active (GiB)": 38.53, "memory/max_allocated (GiB)": 38.53, "ppl": 1.12516, "step": 19, "tokens/total": 665536, "tokens/train_per_sec_per_gpu": 21.97, "tokens/trainable": 8535} +{"epoch": 0.078125, "grad_norm": 1.238651990890503, "learning_rate": 7.6e-05, "loss": 0.12279704213142395, "memory/device_reserved (GiB)": 41.55, "memory/max_active (GiB)": 38.33, "memory/max_allocated (GiB)": 38.33, "ppl": 1.13065, "step": 20, "tokens/total": 700608, "tokens/train_per_sec_per_gpu": 26.12, "tokens/trainable": 9009} +{"epoch": 0.08203125, "grad_norm": 0.7080492973327637, "learning_rate": 8e-05, "loss": 0.1008661612868309, "memory/device_reserved (GiB)": 41.55, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.10613, "step": 21, "tokens/total": 736528, "tokens/train_per_sec_per_gpu": 23.78, "tokens/trainable": 9459} +{"epoch": 0.0859375, "grad_norm": 0.9652141332626343, "learning_rate": 8.4e-05, "loss": 0.12202747166156769, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 35.63, "memory/max_allocated (GiB)": 35.63, "ppl": 1.12979, "step": 22, "tokens/total": 765728, "tokens/train_per_sec_per_gpu": 29.21, "tokens/trainable": 9891} +{"epoch": 0.08984375, "grad_norm": 0.7545146346092224, "learning_rate": 8.800000000000001e-05, "loss": 0.11608275771141052, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 37.95, "memory/max_allocated (GiB)": 37.95, "ppl": 1.12309, "step": 23, "tokens/total": 801712, "tokens/train_per_sec_per_gpu": 28.93, "tokens/trainable": 10381} +{"epoch": 0.09375, "grad_norm": 0.956993579864502, "learning_rate": 9.200000000000001e-05, "loss": 0.12571129202842712, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 1.13395, "step": 24, "tokens/total": 836320, "tokens/train_per_sec_per_gpu": 28.92, "tokens/trainable": 10860} +{"epoch": 0.09765625, "grad_norm": 0.43637681007385254, "learning_rate": 9.6e-05, "loss": 0.09468616545200348, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.09931, "step": 25, "tokens/total": 873744, "tokens/train_per_sec_per_gpu": 25.0, "tokens/trainable": 11343} +{"epoch": 0.1015625, "grad_norm": 0.6012119650840759, "learning_rate": 0.0001, "loss": 0.1002206802368164, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 35.87, "memory/max_allocated (GiB)": 35.87, "ppl": 1.10541, "step": 26, "tokens/total": 904560, "tokens/train_per_sec_per_gpu": 29.59, "tokens/trainable": 11780} +{"epoch": 0.10546875, "grad_norm": 0.7405229806900024, "learning_rate": 9.99990636831587e-05, "loss": 0.1062077060341835, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.11205, "step": 27, "tokens/total": 941808, "tokens/train_per_sec_per_gpu": 26.42, "tokens/trainable": 12231} +{"epoch": 0.109375, "grad_norm": 0.7988643646240234, "learning_rate": 9.999625477159879e-05, "loss": 0.1130177229642868, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.11965, "step": 28, "tokens/total": 980224, "tokens/train_per_sec_per_gpu": 26.28, "tokens/trainable": 12704} +{"epoch": 0.11328125, "grad_norm": 0.7624660134315491, "learning_rate": 9.999157338221051e-05, "loss": 0.09823253750801086, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.10322, "step": 29, "tokens/total": 1015776, "tokens/train_per_sec_per_gpu": 24.69, "tokens/trainable": 13120} +{"epoch": 0.1171875, "grad_norm": 0.5074903964996338, "learning_rate": 9.998501970980562e-05, "loss": 0.07545064389705658, "memory/device_reserved (GiB)": 41.81, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.07837, "step": 30, "tokens/total": 1052512, "tokens/train_per_sec_per_gpu": 24.4, "tokens/trainable": 13594} +{"epoch": 0.12109375, "grad_norm": 0.6977435946464539, "learning_rate": 9.997659402710915e-05, "loss": 0.1110144704580307, "memory/device_reserved (GiB)": 41.81, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.11741, "step": 31, "tokens/total": 1085600, "tokens/train_per_sec_per_gpu": 27.06, "tokens/trainable": 14025} +{"epoch": 0.125, "grad_norm": 1.0788170099258423, "learning_rate": 9.996629668474818e-05, "loss": 0.08895853161811829, "memory/device_reserved (GiB)": 41.81, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.09304, "step": 32, "tokens/total": 1120400, "tokens/train_per_sec_per_gpu": 25.15, "tokens/trainable": 14472} +{"epoch": 0.12890625, "grad_norm": 0.6656172275543213, "learning_rate": 9.995412811123711e-05, "loss": 0.09615232050418854, "memory/device_reserved (GiB)": 41.81, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.10093, "step": 33, "tokens/total": 1156576, "tokens/train_per_sec_per_gpu": 25.76, "tokens/trainable": 14972} +{"epoch": 0.1328125, "grad_norm": 1.0574681758880615, "learning_rate": 9.994008881295999e-05, "loss": 0.10905634611845016, "memory/device_reserved (GiB)": 40.8, "memory/max_active (GiB)": 39.11, "memory/max_allocated (GiB)": 39.11, "ppl": 1.11523, "step": 34, "tokens/total": 1192544, "tokens/train_per_sec_per_gpu": 23.01, "tokens/trainable": 15418} +{"epoch": 0.13671875, "grad_norm": 0.5962039828300476, "learning_rate": 9.992417937414932e-05, "loss": 0.09848605841398239, "memory/device_reserved (GiB)": 40.8, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.1035, "step": 35, "tokens/total": 1227152, "tokens/train_per_sec_per_gpu": 26.25, "tokens/trainable": 15891} +{"epoch": 0.140625, "grad_norm": 0.5281301140785217, "learning_rate": 9.99064004568618e-05, "loss": 0.09302482008934021, "memory/device_reserved (GiB)": 40.8, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.09749, "step": 36, "tokens/total": 1266880, "tokens/train_per_sec_per_gpu": 23.27, "tokens/trainable": 16371} +{"epoch": 0.14453125, "grad_norm": 0.5293381214141846, "learning_rate": 9.988675280095074e-05, "loss": 0.10461550951004028, "memory/device_reserved (GiB)": 40.8, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.11028, "step": 37, "tokens/total": 1302400, "tokens/train_per_sec_per_gpu": 27.81, "tokens/trainable": 16821} +{"epoch": 0.1484375, "grad_norm": 0.6664544939994812, "learning_rate": 9.986523722403528e-05, "loss": 0.08008662611246109, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 38.86, "memory/max_allocated (GiB)": 38.86, "ppl": 1.08338, "step": 38, "tokens/total": 1339152, "tokens/train_per_sec_per_gpu": 25.92, "tokens/trainable": 17282} +{"epoch": 0.15234375, "grad_norm": 0.38923466205596924, "learning_rate": 9.984185462146642e-05, "loss": 0.08242527395486832, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.08592, "step": 39, "tokens/total": 1379312, "tokens/train_per_sec_per_gpu": 22.27, "tokens/trainable": 17767} +{"epoch": 0.15625, "grad_norm": 0.9401765465736389, "learning_rate": 9.98166059662897e-05, "loss": 0.09806116670370102, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.10303, "step": 40, "tokens/total": 1412112, "tokens/train_per_sec_per_gpu": 26.83, "tokens/trainable": 18210} +{"epoch": 0.16015625, "grad_norm": 0.5039956569671631, "learning_rate": 9.978949230920472e-05, "loss": 0.07492294907569885, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 37.2, "memory/max_allocated (GiB)": 37.2, "ppl": 1.0778, "step": 41, "tokens/total": 1446656, "tokens/train_per_sec_per_gpu": 24.98, "tokens/trainable": 18690} +{"epoch": 0.1640625, "grad_norm": 1.2923282384872437, "learning_rate": 9.976051477852141e-05, "loss": 0.10232193768024445, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.10774, "step": 42, "tokens/total": 1481344, "tokens/train_per_sec_per_gpu": 23.35, "tokens/trainable": 19088} +{"epoch": 0.16796875, "grad_norm": 1.1156727075576782, "learning_rate": 9.972967458011312e-05, "loss": 0.0800340324640274, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.08332, "step": 43, "tokens/total": 1515360, "tokens/train_per_sec_per_gpu": 24.79, "tokens/trainable": 19565} +{"epoch": 0.171875, "grad_norm": 0.5793360471725464, "learning_rate": 9.96969729973664e-05, "loss": 0.08276692032814026, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.08629, "step": 44, "tokens/total": 1548496, "tokens/train_per_sec_per_gpu": 24.74, "tokens/trainable": 20020} +{"epoch": 0.17578125, "grad_norm": 0.8144227266311646, "learning_rate": 9.966241139112754e-05, "loss": 0.11243125051259995, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 37.23, "memory/max_allocated (GiB)": 37.23, "ppl": 1.119, "step": 45, "tokens/total": 1582160, "tokens/train_per_sec_per_gpu": 25.22, "tokens/trainable": 20444} +{"epoch": 0.1796875, "grad_norm": 1.443481683731079, "learning_rate": 9.96259911996461e-05, "loss": 0.08703967928886414, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.09094, "step": 46, "tokens/total": 1618800, "tokens/train_per_sec_per_gpu": 21.71, "tokens/trainable": 20899} +{"epoch": 0.18359375, "grad_norm": 0.395381361246109, "learning_rate": 9.958771393851491e-05, "loss": 0.07007896155118942, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 39.08, "memory/max_allocated (GiB)": 39.08, "ppl": 1.07259, "step": 47, "tokens/total": 1656064, "tokens/train_per_sec_per_gpu": 24.6, "tokens/trainable": 21354} +{"epoch": 0.1875, "grad_norm": 2.280351161956787, "learning_rate": 9.954758120060702e-05, "loss": 0.08836953341960907, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.09239, "step": 48, "tokens/total": 1688720, "tokens/train_per_sec_per_gpu": 28.06, "tokens/trainable": 21794} +{"epoch": 0.19140625, "grad_norm": 0.5689712762832642, "learning_rate": 9.950559465600948e-05, "loss": 0.096503347158432, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 1.10131, "step": 49, "tokens/total": 1726256, "tokens/train_per_sec_per_gpu": 22.79, "tokens/trainable": 22268} +{"epoch": 0.1953125, "grad_norm": 0.7453587055206299, "learning_rate": 9.946175605195379e-05, "loss": 0.0692051351070404, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.07166, "step": 50, "tokens/total": 1760976, "tokens/train_per_sec_per_gpu": 28.62, "tokens/trainable": 22747} +{"epoch": 0.19921875, "grad_norm": 0.634723424911499, "learning_rate": 9.941606721274322e-05, "loss": 0.08632136136293411, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 38.03, "memory/max_allocated (GiB)": 38.03, "ppl": 1.09016, "step": 51, "tokens/total": 1796480, "tokens/train_per_sec_per_gpu": 27.01, "tokens/trainable": 23246} +{"epoch": 0.203125, "grad_norm": 0.6383514404296875, "learning_rate": 9.936853003967685e-05, "loss": 0.08965475112199783, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.0938, "step": 52, "tokens/total": 1834336, "tokens/train_per_sec_per_gpu": 23.57, "tokens/trainable": 23725} +{"epoch": 0.20703125, "grad_norm": 0.709831953048706, "learning_rate": 9.93191465109705e-05, "loss": 0.08962420374155045, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 38.67, "memory/max_allocated (GiB)": 38.67, "ppl": 1.09376, "step": 53, "tokens/total": 1872784, "tokens/train_per_sec_per_gpu": 20.8, "tokens/trainable": 24152} +{"epoch": 0.2109375, "grad_norm": 0.6116468906402588, "learning_rate": 9.926791868167438e-05, "loss": 0.08143055438995361, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.08484, "step": 54, "tokens/total": 1912336, "tokens/train_per_sec_per_gpu": 23.59, "tokens/trainable": 24603} +{"epoch": 0.21484375, "grad_norm": 0.7611961960792542, "learning_rate": 9.921484868358753e-05, "loss": 0.08258962631225586, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.0861, "step": 55, "tokens/total": 1949328, "tokens/train_per_sec_per_gpu": 22.53, "tokens/trainable": 25026} +{"epoch": 0.21875, "grad_norm": 0.4511128067970276, "learning_rate": 9.915993872516924e-05, "loss": 0.08706509321928024, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.09097, "step": 56, "tokens/total": 1985792, "tokens/train_per_sec_per_gpu": 27.29, "tokens/trainable": 25495} +{"epoch": 0.22265625, "grad_norm": 0.7396814227104187, "learning_rate": 9.9103191091447e-05, "loss": 0.08762334287166595, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.09158, "step": 57, "tokens/total": 2018976, "tokens/train_per_sec_per_gpu": 27.53, "tokens/trainable": 25968} +{"epoch": 0.2265625, "grad_norm": 0.5064680576324463, "learning_rate": 9.904460814392147e-05, "loss": 0.08421218395233154, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.08786, "step": 58, "tokens/total": 2050528, "tokens/train_per_sec_per_gpu": 30.33, "tokens/trainable": 26418} +{"epoch": 0.23046875, "grad_norm": 0.7813433408737183, "learning_rate": 9.898419232046825e-05, "loss": 0.08854633569717407, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.09258, "step": 59, "tokens/total": 2085392, "tokens/train_per_sec_per_gpu": 25.33, "tokens/trainable": 26866} +{"epoch": 0.234375, "grad_norm": 0.3682762384414673, "learning_rate": 9.892194613523633e-05, "loss": 0.07791519910097122, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 36.3, "memory/max_allocated (GiB)": 36.3, "ppl": 1.08103, "step": 60, "tokens/total": 2117280, "tokens/train_per_sec_per_gpu": 28.75, "tokens/trainable": 27332} +{"epoch": 0.23828125, "grad_norm": 0.7456440925598145, "learning_rate": 9.885787217854357e-05, "loss": 0.0955638438463211, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 1.10028, "step": 61, "tokens/total": 2150848, "tokens/train_per_sec_per_gpu": 23.63, "tokens/trainable": 27734} +{"epoch": 0.2421875, "grad_norm": 0.3406534194946289, "learning_rate": 9.879197311676887e-05, "loss": 0.06654603779315948, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 36.77, "memory/max_allocated (GiB)": 36.77, "ppl": 1.06881, "step": 62, "tokens/total": 2184160, "tokens/train_per_sec_per_gpu": 28.38, "tokens/trainable": 28217} +{"epoch": 0.24609375, "grad_norm": 0.40122190117836, "learning_rate": 9.872425169224113e-05, "loss": 0.08366391807794571, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.08726, "step": 63, "tokens/total": 2217872, "tokens/train_per_sec_per_gpu": 27.68, "tokens/trainable": 28696} +{"epoch": 0.25, "grad_norm": 0.21884752810001373, "learning_rate": 9.865471072312528e-05, "loss": 0.07258161902427673, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 1.07528, "step": 64, "tokens/total": 2252448, "tokens/train_per_sec_per_gpu": 27.41, "tokens/trainable": 29144} +{"epoch": 0.25390625, "grad_norm": 0.7907017469406128, "learning_rate": 9.858335310330492e-05, "loss": 0.07694065570831299, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.07998, "step": 65, "tokens/total": 2285168, "tokens/train_per_sec_per_gpu": 26.12, "tokens/trainable": 29569} +{"epoch": 0.2578125, "grad_norm": 0.43505147099494934, "learning_rate": 9.851018180226185e-05, "loss": 0.08097551763057709, "memory/device_reserved (GiB)": 39.25, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.08434, "step": 66, "tokens/total": 2317968, "tokens/train_per_sec_per_gpu": 27.97, "tokens/trainable": 30010} +{"epoch": 0.26171875, "grad_norm": 0.6694343090057373, "learning_rate": 9.843519986495259e-05, "loss": 0.07544143497943878, "memory/device_reserved (GiB)": 41.03, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.07836, "step": 67, "tokens/total": 2355248, "tokens/train_per_sec_per_gpu": 25.27, "tokens/trainable": 30466} +{"epoch": 0.265625, "grad_norm": 0.5093876123428345, "learning_rate": 9.835841041168162e-05, "loss": 0.06265448033809662, "memory/device_reserved (GiB)": 41.03, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.06466, "step": 68, "tokens/total": 2389264, "tokens/train_per_sec_per_gpu": 28.2, "tokens/trainable": 30943} +{"epoch": 0.26953125, "grad_norm": 1.1883262395858765, "learning_rate": 9.82798166379715e-05, "loss": 0.09820792078971863, "memory/device_reserved (GiB)": 41.03, "memory/max_active (GiB)": 37.5, "memory/max_allocated (GiB)": 37.5, "ppl": 1.10319, "step": 69, "tokens/total": 2421360, "tokens/train_per_sec_per_gpu": 26.67, "tokens/trainable": 31364} +{"epoch": 0.2734375, "grad_norm": 0.38646242022514343, "learning_rate": 9.819942181443002e-05, "loss": 0.07016599178314209, "memory/device_reserved (GiB)": 41.03, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.07269, "step": 70, "tokens/total": 2454256, "tokens/train_per_sec_per_gpu": 25.09, "tokens/trainable": 31782} +{"epoch": 0.27734375, "grad_norm": 0.4371630549430847, "learning_rate": 9.811722928661392e-05, "loss": 0.07092759758234024, "memory/device_reserved (GiB)": 41.03, "memory/max_active (GiB)": 37.23, "memory/max_allocated (GiB)": 37.23, "ppl": 1.0735, "step": 71, "tokens/total": 2488464, "tokens/train_per_sec_per_gpu": 28.0, "tokens/trainable": 32257} +{"epoch": 0.28125, "grad_norm": 0.7789031267166138, "learning_rate": 9.803324247488975e-05, "loss": 0.08444643020629883, "memory/device_reserved (GiB)": 41.03, "memory/max_active (GiB)": 39.19, "memory/max_allocated (GiB)": 39.19, "ppl": 1.08811, "step": 72, "tokens/total": 2523696, "tokens/train_per_sec_per_gpu": 24.18, "tokens/trainable": 32707} +{"epoch": 0.28515625, "grad_norm": 0.4494639039039612, "learning_rate": 9.794746487429161e-05, "loss": 0.07665218412876129, "memory/device_reserved (GiB)": 41.04, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.07967, "step": 73, "tokens/total": 2558240, "tokens/train_per_sec_per_gpu": 31.44, "tokens/trainable": 33207} +{"epoch": 0.2890625, "grad_norm": 4.597369194030762, "learning_rate": 9.785990005437554e-05, "loss": 0.07316811382770538, "memory/device_reserved (GiB)": 41.04, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.07591, "step": 74, "tokens/total": 2592496, "tokens/train_per_sec_per_gpu": 27.25, "tokens/trainable": 33646} +{"epoch": 0.29296875, "grad_norm": 0.33131346106529236, "learning_rate": 9.777055165907117e-05, "loss": 0.08117525279521942, "memory/device_reserved (GiB)": 41.04, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.08456, "step": 75, "tokens/total": 2626352, "tokens/train_per_sec_per_gpu": 26.33, "tokens/trainable": 34112} +{"epoch": 0.296875, "grad_norm": 0.8128768801689148, "learning_rate": 9.767942340652993e-05, "loss": 0.07473810017108917, "memory/device_reserved (GiB)": 41.04, "memory/max_active (GiB)": 38.97, "memory/max_allocated (GiB)": 38.97, "ppl": 1.0776, "step": 76, "tokens/total": 2661216, "tokens/train_per_sec_per_gpu": 25.44, "tokens/trainable": 34592} +{"epoch": 0.30078125, "grad_norm": 0.5219283699989319, "learning_rate": 9.758651908897035e-05, "loss": 0.09323441982269287, "memory/device_reserved (GiB)": 41.28, "memory/max_active (GiB)": 39.55, "memory/max_allocated (GiB)": 39.55, "ppl": 1.09772, "step": 77, "tokens/total": 2696960, "tokens/train_per_sec_per_gpu": 28.04, "tokens/trainable": 35038} +{"epoch": 0.3046875, "grad_norm": 0.3862384259700775, "learning_rate": 9.749184257252033e-05, "loss": 0.07413788139820099, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.07696, "step": 78, "tokens/total": 2733376, "tokens/train_per_sec_per_gpu": 27.43, "tokens/trainable": 35529} +{"epoch": 0.30859375, "grad_norm": 0.9132375121116638, "learning_rate": 9.739539779705614e-05, "loss": 0.07671518623828888, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.07973, "step": 79, "tokens/total": 2770496, "tokens/train_per_sec_per_gpu": 22.34, "tokens/trainable": 35960} +{"epoch": 0.3125, "grad_norm": 1.096083164215088, "learning_rate": 9.729718877603861e-05, "loss": 0.09562558680772781, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.13, "memory/max_allocated (GiB)": 38.13, "ppl": 1.10035, "step": 80, "tokens/total": 2807184, "tokens/train_per_sec_per_gpu": 25.37, "tokens/trainable": 36405} +{"epoch": 0.31640625, "grad_norm": 0.8954784870147705, "learning_rate": 9.719721959634592e-05, "loss": 0.04853895679116249, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.26, "memory/max_allocated (GiB)": 38.26, "ppl": 1.04974, "step": 81, "tokens/total": 2842704, "tokens/train_per_sec_per_gpu": 25.99, "tokens/trainable": 36852} +{"epoch": 0.3203125, "grad_norm": 1.5929293632507324, "learning_rate": 9.709549441810375e-05, "loss": 0.07356926798820496, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 35.93, "memory/max_allocated (GiB)": 35.93, "ppl": 1.07634, "step": 82, "tokens/total": 2873808, "tokens/train_per_sec_per_gpu": 27.77, "tokens/trainable": 37291} +{"epoch": 0.32421875, "grad_norm": 0.5578079223632812, "learning_rate": 9.699201747451195e-05, "loss": 0.08882060647010803, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.39, "memory/max_allocated (GiB)": 37.39, "ppl": 1.09288, "step": 83, "tokens/total": 2906896, "tokens/train_per_sec_per_gpu": 29.97, "tokens/trainable": 37749} +{"epoch": 0.328125, "grad_norm": 0.5723075270652771, "learning_rate": 9.688679307166854e-05, "loss": 0.05338352918624878, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.05483, "step": 84, "tokens/total": 2942016, "tokens/train_per_sec_per_gpu": 25.69, "tokens/trainable": 38205} +{"epoch": 0.33203125, "grad_norm": 0.5085099339485168, "learning_rate": 9.677982558839042e-05, "loss": 0.08276034891605377, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.32, "memory/max_allocated (GiB)": 39.32, "ppl": 1.08628, "step": 85, "tokens/total": 2977920, "tokens/train_per_sec_per_gpu": 26.37, "tokens/trainable": 38674} +{"epoch": 0.3359375, "grad_norm": 0.4551318883895874, "learning_rate": 9.66711194760312e-05, "loss": 0.07271283119916916, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.07, "memory/max_allocated (GiB)": 37.07, "ppl": 1.07542, "step": 86, "tokens/total": 3011424, "tokens/train_per_sec_per_gpu": 26.19, "tokens/trainable": 39124} +{"epoch": 0.33984375, "grad_norm": 0.35030597448349, "learning_rate": 9.656067925829593e-05, "loss": 0.08206797391176224, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.08553, "step": 87, "tokens/total": 3043168, "tokens/train_per_sec_per_gpu": 30.47, "tokens/trainable": 39600} +{"epoch": 0.34375, "grad_norm": 0.37152472138404846, "learning_rate": 9.644850953105288e-05, "loss": 0.07242418825626373, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.07511, "step": 88, "tokens/total": 3076848, "tokens/train_per_sec_per_gpu": 27.29, "tokens/trainable": 40068} +{"epoch": 0.34765625, "grad_norm": 0.35893380641937256, "learning_rate": 9.633461496214225e-05, "loss": 0.06904274225234985, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 35.83, "memory/max_allocated (GiB)": 35.83, "ppl": 1.07148, "step": 89, "tokens/total": 3107392, "tokens/train_per_sec_per_gpu": 24.9, "tokens/trainable": 40452} +{"epoch": 0.3515625, "grad_norm": 0.3277762830257416, "learning_rate": 9.621900029118195e-05, "loss": 0.0721713975071907, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.59, "memory/max_allocated (GiB)": 37.59, "ppl": 1.07484, "step": 90, "tokens/total": 3142800, "tokens/train_per_sec_per_gpu": 26.61, "tokens/trainable": 40925} +{"epoch": 0.35546875, "grad_norm": 0.3492467701435089, "learning_rate": 9.610167032937036e-05, "loss": 0.07211891561746597, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.11, "memory/max_allocated (GiB)": 38.11, "ppl": 1.07478, "step": 91, "tokens/total": 3178416, "tokens/train_per_sec_per_gpu": 27.94, "tokens/trainable": 41374} +{"epoch": 0.359375, "grad_norm": 0.37021520733833313, "learning_rate": 9.598262995928611e-05, "loss": 0.08107663691043854, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 36.51, "memory/max_allocated (GiB)": 36.51, "ppl": 1.08445, "step": 92, "tokens/total": 3210400, "tokens/train_per_sec_per_gpu": 28.13, "tokens/trainable": 41807} +{"epoch": 0.36328125, "grad_norm": 0.34360748529434204, "learning_rate": 9.586188413468492e-05, "loss": 0.060695625841617584, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.06258, "step": 93, "tokens/total": 3244880, "tokens/train_per_sec_per_gpu": 26.21, "tokens/trainable": 42236} +{"epoch": 0.3671875, "grad_norm": 0.3795430064201355, "learning_rate": 9.57394378802934e-05, "loss": 0.08455982804298401, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.93, "memory/max_allocated (GiB)": 38.93, "ppl": 1.08824, "step": 94, "tokens/total": 3279952, "tokens/train_per_sec_per_gpu": 27.28, "tokens/trainable": 42712} +{"epoch": 0.37109375, "grad_norm": 0.39364442229270935, "learning_rate": 9.56152962916e-05, "loss": 0.08178634196519852, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.08522, "step": 95, "tokens/total": 3314688, "tokens/train_per_sec_per_gpu": 26.36, "tokens/trainable": 43213} +{"epoch": 0.375, "grad_norm": 0.36517295241355896, "learning_rate": 9.548946453464296e-05, "loss": 0.07806195318698883, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.08119, "step": 96, "tokens/total": 3350464, "tokens/train_per_sec_per_gpu": 28.74, "tokens/trainable": 43702} +{"epoch": 0.37890625, "grad_norm": 0.4324662983417511, "learning_rate": 9.53619478457953e-05, "loss": 0.0750846192240715, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.84, "memory/max_allocated (GiB)": 37.84, "ppl": 1.07798, "step": 97, "tokens/total": 3385536, "tokens/train_per_sec_per_gpu": 25.55, "tokens/trainable": 44158} +{"epoch": 0.3828125, "grad_norm": 0.47768086194992065, "learning_rate": 9.523275153154695e-05, "loss": 0.09000968933105469, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.09418, "step": 98, "tokens/total": 3419584, "tokens/train_per_sec_per_gpu": 26.87, "tokens/trainable": 44615} +{"epoch": 0.38671875, "grad_norm": 0.3627943694591522, "learning_rate": 9.51018809682839e-05, "loss": 0.062145963311195374, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 36.29, "memory/max_allocated (GiB)": 36.29, "ppl": 1.06412, "step": 99, "tokens/total": 3448576, "tokens/train_per_sec_per_gpu": 26.51, "tokens/trainable": 45042} +{"epoch": 0.390625, "grad_norm": 3.566817283630371, "learning_rate": 9.49693416020645e-05, "loss": 0.08712585270404816, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 1.09103, "step": 100, "tokens/total": 3485328, "tokens/train_per_sec_per_gpu": 24.71, "tokens/trainable": 45508} +{"epoch": 0.39453125, "grad_norm": 0.4634423553943634, "learning_rate": 9.483513894839276e-05, "loss": 0.09054693579673767, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.59, "memory/max_allocated (GiB)": 37.59, "ppl": 1.09477, "step": 101, "tokens/total": 3519984, "tokens/train_per_sec_per_gpu": 28.07, "tokens/trainable": 45989} +{"epoch": 0.3984375, "grad_norm": 0.40742817521095276, "learning_rate": 9.469927859198888e-05, "loss": 0.06813780218362808, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.07051, "step": 102, "tokens/total": 3553952, "tokens/train_per_sec_per_gpu": 27.91, "tokens/trainable": 46425} +{"epoch": 0.40234375, "grad_norm": 0.3168119490146637, "learning_rate": 9.456176618655689e-05, "loss": 0.05898516625165939, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.06076, "step": 103, "tokens/total": 3591072, "tokens/train_per_sec_per_gpu": 26.21, "tokens/trainable": 46876} +{"epoch": 0.40625, "grad_norm": 0.3033517301082611, "learning_rate": 9.442260745454927e-05, "loss": 0.05906073376536369, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.7, "memory/max_allocated (GiB)": 38.7, "ppl": 1.06084, "step": 104, "tokens/total": 3627760, "tokens/train_per_sec_per_gpu": 27.27, "tokens/trainable": 47378} +{"epoch": 0.41015625, "grad_norm": 0.4207969903945923, "learning_rate": 9.428180818692884e-05, "loss": 0.06208154186606407, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 36.95, "memory/max_allocated (GiB)": 36.95, "ppl": 1.06405, "step": 105, "tokens/total": 3661552, "tokens/train_per_sec_per_gpu": 27.93, "tokens/trainable": 47852} +{"epoch": 0.4140625, "grad_norm": 0.6878803968429565, "learning_rate": 9.413937424292791e-05, "loss": 0.08181318640708923, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 36.85, "memory/max_allocated (GiB)": 36.85, "ppl": 1.08525, "step": 106, "tokens/total": 3694464, "tokens/train_per_sec_per_gpu": 27.72, "tokens/trainable": 48325} +{"epoch": 0.41796875, "grad_norm": 0.3787034749984741, "learning_rate": 9.399531154980424e-05, "loss": 0.06446830928325653, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.06659, "step": 107, "tokens/total": 3730768, "tokens/train_per_sec_per_gpu": 27.79, "tokens/trainable": 48818} +{"epoch": 0.421875, "grad_norm": 0.5162315964698792, "learning_rate": 9.384962610259455e-05, "loss": 0.07308150082826614, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.32, "memory/max_allocated (GiB)": 38.32, "ppl": 1.07582, "step": 108, "tokens/total": 3766288, "tokens/train_per_sec_per_gpu": 23.83, "tokens/trainable": 49256} +{"epoch": 0.42578125, "grad_norm": 0.6096583604812622, "learning_rate": 9.370232396386494e-05, "loss": 0.08556053042411804, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.08933, "step": 109, "tokens/total": 3802688, "tokens/train_per_sec_per_gpu": 24.97, "tokens/trainable": 49701} +{"epoch": 0.4296875, "grad_norm": 0.3558635115623474, "learning_rate": 9.355341126345868e-05, "loss": 0.07422836124897003, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.07705, "step": 110, "tokens/total": 3839936, "tokens/train_per_sec_per_gpu": 23.43, "tokens/trainable": 50192} +{"epoch": 0.43359375, "grad_norm": 0.3129972815513611, "learning_rate": 9.340289419824107e-05, "loss": 0.04137944430112839, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.37, "memory/max_allocated (GiB)": 38.37, "ppl": 1.04225, "step": 111, "tokens/total": 3875056, "tokens/train_per_sec_per_gpu": 24.71, "tokens/trainable": 50608} +{"epoch": 0.4375, "grad_norm": 1.0318574905395508, "learning_rate": 9.325077903184159e-05, "loss": 0.09368318319320679, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.3, "memory/max_allocated (GiB)": 38.3, "ppl": 1.09821, "step": 112, "tokens/total": 3908592, "tokens/train_per_sec_per_gpu": 25.5, "tokens/trainable": 51032} +{"epoch": 0.44140625, "grad_norm": 0.5926149487495422, "learning_rate": 9.30970720943932e-05, "loss": 0.04619169235229492, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.04728, "step": 113, "tokens/total": 3943744, "tokens/train_per_sec_per_gpu": 27.61, "tokens/trainable": 51537} +{"epoch": 0.4453125, "grad_norm": 1.36929190158844, "learning_rate": 9.2941779782269e-05, "loss": 0.05446441471576691, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 36.18, "memory/max_allocated (GiB)": 36.18, "ppl": 1.05597, "step": 114, "tokens/total": 3975440, "tokens/train_per_sec_per_gpu": 29.53, "tokens/trainable": 52004} +{"epoch": 0.44921875, "grad_norm": 3.6162962913513184, "learning_rate": 9.278490855781596e-05, "loss": 0.06686576455831528, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.06915, "step": 115, "tokens/total": 4011024, "tokens/train_per_sec_per_gpu": 27.14, "tokens/trainable": 52487} +{"epoch": 0.453125, "grad_norm": 0.6189031004905701, "learning_rate": 9.262646494908604e-05, "loss": 0.05231093615293503, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.0537, "step": 116, "tokens/total": 4048976, "tokens/train_per_sec_per_gpu": 25.89, "tokens/trainable": 52965} +{"epoch": 0.45703125, "grad_norm": 0.4672190845012665, "learning_rate": 9.246645554956457e-05, "loss": 0.07359366118907928, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.07637, "step": 117, "tokens/total": 4083376, "tokens/train_per_sec_per_gpu": 26.77, "tokens/trainable": 53450} +{"epoch": 0.4609375, "grad_norm": 0.45954176783561707, "learning_rate": 9.230488701789578e-05, "loss": 0.05680956318974495, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.05845, "step": 118, "tokens/total": 4118576, "tokens/train_per_sec_per_gpu": 26.53, "tokens/trainable": 53927} +{"epoch": 0.46484375, "grad_norm": 0.9078386425971985, "learning_rate": 9.214176607760577e-05, "loss": 0.041995737701654434, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.29, "memory/max_allocated (GiB)": 38.29, "ppl": 1.04289, "step": 119, "tokens/total": 4154480, "tokens/train_per_sec_per_gpu": 26.08, "tokens/trainable": 54395} +{"epoch": 0.46875, "grad_norm": 0.6526390910148621, "learning_rate": 9.197709951682268e-05, "loss": 0.06526990234851837, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.06745, "step": 120, "tokens/total": 4190096, "tokens/train_per_sec_per_gpu": 27.65, "tokens/trainable": 54871} +{"epoch": 0.47265625, "grad_norm": 0.4020896553993225, "learning_rate": 9.181089418799428e-05, "loss": 0.07082214951515198, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 39.32, "memory/max_allocated (GiB)": 39.32, "ppl": 1.07339, "step": 121, "tokens/total": 4229312, "tokens/train_per_sec_per_gpu": 23.24, "tokens/trainable": 55356} +{"epoch": 0.4765625, "grad_norm": 0.39667415618896484, "learning_rate": 9.164315700760271e-05, "loss": 0.055855054408311844, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.05744, "step": 122, "tokens/total": 4264864, "tokens/train_per_sec_per_gpu": 25.62, "tokens/trainable": 55836} +{"epoch": 0.48046875, "grad_norm": 0.3864579200744629, "learning_rate": 9.147389495587671e-05, "loss": 0.06945621967315674, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 36.18, "memory/max_allocated (GiB)": 36.18, "ppl": 1.07193, "step": 123, "tokens/total": 4295360, "tokens/train_per_sec_per_gpu": 29.58, "tokens/trainable": 56275} +{"epoch": 0.484375, "grad_norm": 0.4868689775466919, "learning_rate": 9.130311507650116e-05, "loss": 0.06730581820011139, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 36.98, "memory/max_allocated (GiB)": 36.98, "ppl": 1.06962, "step": 124, "tokens/total": 4328336, "tokens/train_per_sec_per_gpu": 26.6, "tokens/trainable": 56745} +{"epoch": 0.48828125, "grad_norm": 0.5624237060546875, "learning_rate": 9.113082447632394e-05, "loss": 0.06398633122444153, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 38.49, "memory/max_allocated (GiB)": 38.49, "ppl": 1.06608, "step": 125, "tokens/total": 4365552, "tokens/train_per_sec_per_gpu": 22.36, "tokens/trainable": 57171} +{"epoch": 0.4921875, "grad_norm": 0.7405147552490234, "learning_rate": 9.09570303250602e-05, "loss": 0.03932569921016693, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.04011, "step": 126, "tokens/total": 4399376, "tokens/train_per_sec_per_gpu": 27.62, "tokens/trainable": 57642} +{"epoch": 0.49609375, "grad_norm": 0.5472530722618103, "learning_rate": 9.078173985499394e-05, "loss": 0.019526131451129913, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.01972, "step": 127, "tokens/total": 4434880, "tokens/train_per_sec_per_gpu": 29.13, "tokens/trainable": 58134} +{"epoch": 0.5, "grad_norm": 1.7385238409042358, "learning_rate": 9.060496036067713e-05, "loss": 0.038323745131492615, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.03907, "step": 128, "tokens/total": 4469280, "tokens/train_per_sec_per_gpu": 23.51, "tokens/trainable": 58564} +{"epoch": 0.50390625, "grad_norm": 1.781083583831787, "learning_rate": 9.042669919862615e-05, "loss": 0.04635139927268028, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.04744, "step": 129, "tokens/total": 4503408, "tokens/train_per_sec_per_gpu": 27.57, "tokens/trainable": 59038} +{"epoch": 0.5078125, "grad_norm": 0.8792572617530823, "learning_rate": 9.024696378701557e-05, "loss": 0.01656402088701725, "memory/device_reserved (GiB)": 39.68, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.0167, "step": 130, "tokens/total": 4536976, "tokens/train_per_sec_per_gpu": 28.84, "tokens/trainable": 59495} +{"epoch": 0.51171875, "grad_norm": 0.7587352395057678, "learning_rate": 9.006576160536948e-05, "loss": 0.019036415964365005, "memory/device_reserved (GiB)": 39.68, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.01922, "step": 131, "tokens/total": 4570304, "tokens/train_per_sec_per_gpu": 25.39, "tokens/trainable": 59911} +{"epoch": 0.515625, "grad_norm": 0.9929646253585815, "learning_rate": 8.988310019425035e-05, "loss": 0.025426026433706284, "memory/device_reserved (GiB)": 39.68, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.02575, "step": 132, "tokens/total": 4602912, "tokens/train_per_sec_per_gpu": 29.39, "tokens/trainable": 60371} +{"epoch": 0.51953125, "grad_norm": 1.3704822063446045, "learning_rate": 8.969898715494506e-05, "loss": 0.06391750276088715, "memory/device_reserved (GiB)": 40.09, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.066, "step": 133, "tokens/total": 4638080, "tokens/train_per_sec_per_gpu": 26.25, "tokens/trainable": 60827} +{"epoch": 0.5234375, "grad_norm": 1.2785959243774414, "learning_rate": 8.951343014914869e-05, "loss": 0.052059926092624664, "memory/device_reserved (GiB)": 40.1, "memory/max_active (GiB)": 38.52, "memory/max_allocated (GiB)": 38.52, "ppl": 1.05344, "step": 134, "tokens/total": 4672480, "tokens/train_per_sec_per_gpu": 29.37, "tokens/trainable": 61320} +{"epoch": 0.52734375, "grad_norm": 0.6951143145561218, "learning_rate": 8.932643689864568e-05, "loss": 0.030468281358480453, "memory/device_reserved (GiB)": 40.1, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.03094, "step": 135, "tokens/total": 4707264, "tokens/train_per_sec_per_gpu": 25.03, "tokens/trainable": 61775} +{"epoch": 0.53125, "grad_norm": 1.32224702835083, "learning_rate": 8.913801518498845e-05, "loss": 0.03087960183620453, "memory/device_reserved (GiB)": 40.1, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 1.03136, "step": 136, "tokens/total": 4743696, "tokens/train_per_sec_per_gpu": 25.43, "tokens/trainable": 62248} +{"epoch": 0.53515625, "grad_norm": 0.6478903293609619, "learning_rate": 8.894817284917364e-05, "loss": 0.03180031478404999, "memory/device_reserved (GiB)": 40.92, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.03231, "step": 137, "tokens/total": 4781424, "tokens/train_per_sec_per_gpu": 24.84, "tokens/trainable": 62730} +{"epoch": 0.5390625, "grad_norm": 1.7533296346664429, "learning_rate": 8.875691779131569e-05, "loss": 0.03487149998545647, "memory/device_reserved (GiB)": 40.92, "memory/max_active (GiB)": 38.99, "memory/max_allocated (GiB)": 38.99, "ppl": 1.03549, "step": 138, "tokens/total": 4820752, "tokens/train_per_sec_per_gpu": 22.45, "tokens/trainable": 63166} +{"epoch": 0.54296875, "grad_norm": 1.7421647310256958, "learning_rate": 8.856425797031829e-05, "loss": 0.04269048944115639, "memory/device_reserved (GiB)": 40.92, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.04361, "step": 139, "tokens/total": 4854688, "tokens/train_per_sec_per_gpu": 25.92, "tokens/trainable": 63631} +{"epoch": 0.546875, "grad_norm": 0.6178533434867859, "learning_rate": 8.837020140354295e-05, "loss": 0.04482627660036087, "memory/device_reserved (GiB)": 40.92, "memory/max_active (GiB)": 38.24, "memory/max_allocated (GiB)": 38.24, "ppl": 1.04585, "step": 140, "tokens/total": 4891424, "tokens/train_per_sec_per_gpu": 22.86, "tokens/trainable": 64080} +{"epoch": 0.55078125, "grad_norm": 0.5406362414360046, "learning_rate": 8.817475616647554e-05, "loss": 0.03126617521047592, "memory/device_reserved (GiB)": 40.92, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.03176, "step": 141, "tokens/total": 4924464, "tokens/train_per_sec_per_gpu": 29.48, "tokens/trainable": 64537} +{"epoch": 0.5546875, "grad_norm": 0.5452086329460144, "learning_rate": 8.797793039239017e-05, "loss": 0.023487966507673264, "memory/device_reserved (GiB)": 40.92, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.02377, "step": 142, "tokens/total": 4957072, "tokens/train_per_sec_per_gpu": 27.89, "tokens/trainable": 65011} +{"epoch": 0.55859375, "grad_norm": 0.43866249918937683, "learning_rate": 8.777973227201069e-05, "loss": 0.011593781411647797, "memory/device_reserved (GiB)": 40.92, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.01166, "step": 143, "tokens/total": 4991840, "tokens/train_per_sec_per_gpu": 28.04, "tokens/trainable": 65511} +{"epoch": 0.5625, "grad_norm": 0.5137069225311279, "learning_rate": 8.758017005316988e-05, "loss": 0.023284031078219414, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.18, "memory/max_allocated (GiB)": 39.18, "ppl": 1.02356, "step": 144, "tokens/total": 5029312, "tokens/train_per_sec_per_gpu": 24.11, "tokens/trainable": 65996} +{"epoch": 0.56640625, "grad_norm": 0.6982814073562622, "learning_rate": 8.737925204046629e-05, "loss": 0.01535565685480833, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.56, "memory/max_allocated (GiB)": 36.56, "ppl": 1.01547, "step": 145, "tokens/total": 5060800, "tokens/train_per_sec_per_gpu": 26.69, "tokens/trainable": 66423} +{"epoch": 0.5703125, "grad_norm": 0.9712476134300232, "learning_rate": 8.717698659491851e-05, "loss": 0.020409921184182167, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.86, "memory/max_allocated (GiB)": 38.86, "ppl": 1.02062, "step": 146, "tokens/total": 5096448, "tokens/train_per_sec_per_gpu": 26.19, "tokens/trainable": 66905} +{"epoch": 0.57421875, "grad_norm": 1.3694430589675903, "learning_rate": 8.697338213361735e-05, "loss": 0.012026145122945309, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.0121, "step": 147, "tokens/total": 5132928, "tokens/train_per_sec_per_gpu": 26.75, "tokens/trainable": 67363} +{"epoch": 0.578125, "grad_norm": 0.34994077682495117, "learning_rate": 8.676844712937552e-05, "loss": 0.010584201663732529, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.01064, "step": 148, "tokens/total": 5167008, "tokens/train_per_sec_per_gpu": 26.59, "tokens/trainable": 67829} +{"epoch": 0.58203125, "grad_norm": 1.1427459716796875, "learning_rate": 8.656219011037509e-05, "loss": 0.032289475202560425, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.14, "memory/max_allocated (GiB)": 36.14, "ppl": 1.03282, "step": 149, "tokens/total": 5198656, "tokens/train_per_sec_per_gpu": 27.28, "tokens/trainable": 68286} +{"epoch": 0.5859375, "grad_norm": 0.9754296541213989, "learning_rate": 8.63546196598125e-05, "loss": 0.017921052873134613, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.01808, "step": 150, "tokens/total": 5235600, "tokens/train_per_sec_per_gpu": 24.37, "tokens/trainable": 68750} +{"epoch": 0.58984375, "grad_norm": 0.6369094848632812, "learning_rate": 8.614574441554145e-05, "loss": 0.019672926515340805, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.57, "memory/max_allocated (GiB)": 39.57, "ppl": 1.01987, "step": 151, "tokens/total": 5273808, "tokens/train_per_sec_per_gpu": 25.54, "tokens/trainable": 69231} +{"epoch": 0.59375, "grad_norm": 0.6354537010192871, "learning_rate": 8.593557306971349e-05, "loss": 0.01687685400247574, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.01702, "step": 152, "tokens/total": 5310480, "tokens/train_per_sec_per_gpu": 27.18, "tokens/trainable": 69711} +{"epoch": 0.59765625, "grad_norm": 0.5819340944290161, "learning_rate": 8.572411436841618e-05, "loss": 0.010089678689837456, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.01014, "step": 153, "tokens/total": 5349264, "tokens/train_per_sec_per_gpu": 23.83, "tokens/trainable": 70178} +{"epoch": 0.6015625, "grad_norm": 0.6515582203865051, "learning_rate": 8.551137711130922e-05, "loss": 0.02294277772307396, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.02321, "step": 154, "tokens/total": 5382736, "tokens/train_per_sec_per_gpu": 30.05, "tokens/trainable": 70670} +{"epoch": 0.60546875, "grad_norm": 0.3676649332046509, "learning_rate": 8.529737015125824e-05, "loss": 0.01787274330854416, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.01803, "step": 155, "tokens/total": 5420256, "tokens/train_per_sec_per_gpu": 22.31, "tokens/trainable": 71111} +{"epoch": 0.609375, "grad_norm": 0.37199950218200684, "learning_rate": 8.508210239396639e-05, "loss": 0.014273476786911488, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.93, "memory/max_allocated (GiB)": 36.93, "ppl": 1.01438, "step": 156, "tokens/total": 5454080, "tokens/train_per_sec_per_gpu": 31.45, "tokens/trainable": 71623} +{"epoch": 0.61328125, "grad_norm": 0.4472825527191162, "learning_rate": 8.486558279760375e-05, "loss": 0.01649487018585205, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.01663, "step": 157, "tokens/total": 5489728, "tokens/train_per_sec_per_gpu": 27.06, "tokens/trainable": 72100} +{"epoch": 0.6171875, "grad_norm": 0.44951820373535156, "learning_rate": 8.464782037243449e-05, "loss": 0.010509379208087921, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.01056, "step": 158, "tokens/total": 5526512, "tokens/train_per_sec_per_gpu": 25.43, "tokens/trainable": 72598} +{"epoch": 0.62109375, "grad_norm": 0.34423285722732544, "learning_rate": 8.442882418044202e-05, "loss": 0.01063771266490221, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.01069, "step": 159, "tokens/total": 5563456, "tokens/train_per_sec_per_gpu": 24.46, "tokens/trainable": 73071} +{"epoch": 0.625, "grad_norm": 0.5623701810836792, "learning_rate": 8.420860333495179e-05, "loss": 0.021615806967020035, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.22, "memory/max_allocated (GiB)": 38.22, "ppl": 1.02185, "step": 160, "tokens/total": 5599520, "tokens/train_per_sec_per_gpu": 28.64, "tokens/trainable": 73560} +{"epoch": 0.62890625, "grad_norm": 0.6159409284591675, "learning_rate": 8.398716700025208e-05, "loss": 0.015421551652252674, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.01554, "step": 161, "tokens/total": 5637264, "tokens/train_per_sec_per_gpu": 27.27, "tokens/trainable": 74083} +{"epoch": 0.6328125, "grad_norm": 0.607215940952301, "learning_rate": 8.376452439121266e-05, "loss": 0.033875368535518646, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.61, "memory/max_allocated (GiB)": 36.61, "ppl": 1.03446, "step": 162, "tokens/total": 5670064, "tokens/train_per_sec_per_gpu": 24.84, "tokens/trainable": 74530} +{"epoch": 0.63671875, "grad_norm": 0.40024104714393616, "learning_rate": 8.354068477290124e-05, "loss": 0.011240809224545956, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.0113, "step": 163, "tokens/total": 5702800, "tokens/train_per_sec_per_gpu": 27.74, "tokens/trainable": 74999} +{"epoch": 0.640625, "grad_norm": 0.7260295152664185, "learning_rate": 8.331565746019807e-05, "loss": 0.02171010710299015, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.02195, "step": 164, "tokens/total": 5739792, "tokens/train_per_sec_per_gpu": 22.71, "tokens/trainable": 75450} +{"epoch": 0.64453125, "grad_norm": 0.4650598168373108, "learning_rate": 8.308945181740812e-05, "loss": 0.01755407452583313, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 1.01771, "step": 165, "tokens/total": 5773856, "tokens/train_per_sec_per_gpu": 26.25, "tokens/trainable": 75921} +{"epoch": 0.6484375, "grad_norm": 0.3894974887371063, "learning_rate": 8.286207725787153e-05, "loss": 0.01063237339258194, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.54, "memory/max_allocated (GiB)": 39.54, "ppl": 1.01069, "step": 166, "tokens/total": 5809520, "tokens/train_per_sec_per_gpu": 28.87, "tokens/trainable": 76425} +{"epoch": 0.65234375, "grad_norm": 0.6108887791633606, "learning_rate": 8.263354324357182e-05, "loss": 0.0147816501557827, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.01489, "step": 167, "tokens/total": 5841920, "tokens/train_per_sec_per_gpu": 28.28, "tokens/trainable": 76863} +{"epoch": 0.65625, "grad_norm": 0.6350558400154114, "learning_rate": 8.240385928474219e-05, "loss": 0.016083823516964912, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.07, "memory/max_allocated (GiB)": 38.07, "ppl": 1.01621, "step": 168, "tokens/total": 5874912, "tokens/train_per_sec_per_gpu": 25.99, "tokens/trainable": 77298} +{"epoch": 0.66015625, "grad_norm": 0.6354270577430725, "learning_rate": 8.217303493946967e-05, "loss": 0.014920128509402275, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.01503, "step": 169, "tokens/total": 5908768, "tokens/train_per_sec_per_gpu": 28.52, "tokens/trainable": 77735} +{"epoch": 0.6640625, "grad_norm": 0.1627199798822403, "learning_rate": 8.194107981329746e-05, "loss": 0.003147047245875001, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.78, "memory/max_allocated (GiB)": 36.78, "ppl": 1.00315, "step": 170, "tokens/total": 5942128, "tokens/train_per_sec_per_gpu": 27.46, "tokens/trainable": 78179} +{"epoch": 0.66796875, "grad_norm": 0.48754629492759705, "learning_rate": 8.170800355882518e-05, "loss": 0.024663329124450684, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.36, "memory/max_allocated (GiB)": 38.36, "ppl": 1.02497, "step": 171, "tokens/total": 5979232, "tokens/train_per_sec_per_gpu": 27.01, "tokens/trainable": 78660} +{"epoch": 0.671875, "grad_norm": 1.2185652256011963, "learning_rate": 8.147381587530713e-05, "loss": 0.03585971146821976, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.03651, "step": 172, "tokens/total": 6014000, "tokens/train_per_sec_per_gpu": 25.28, "tokens/trainable": 79110} +{"epoch": 0.67578125, "grad_norm": 0.42235302925109863, "learning_rate": 8.123852650824877e-05, "loss": 0.00826641172170639, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.0083, "step": 173, "tokens/total": 6048688, "tokens/train_per_sec_per_gpu": 27.03, "tokens/trainable": 79576} +{"epoch": 0.6796875, "grad_norm": 0.2218102514743805, "learning_rate": 8.100214524900103e-05, "loss": 0.005186872091144323, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 35.79, "memory/max_allocated (GiB)": 35.79, "ppl": 1.0052, "step": 174, "tokens/total": 6078768, "tokens/train_per_sec_per_gpu": 25.25, "tokens/trainable": 80001} +{"epoch": 0.68359375, "grad_norm": 0.5959038138389587, "learning_rate": 8.076468193435301e-05, "loss": 0.02066628262400627, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.02088, "step": 175, "tokens/total": 6111296, "tokens/train_per_sec_per_gpu": 27.9, "tokens/trainable": 80452} +{"epoch": 0.6875, "grad_norm": 0.6000850200653076, "learning_rate": 8.052614644612253e-05, "loss": 0.014593832194805145, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.0147, "step": 176, "tokens/total": 6146224, "tokens/train_per_sec_per_gpu": 25.89, "tokens/trainable": 80886} +{"epoch": 0.69140625, "grad_norm": 0.5334988236427307, "learning_rate": 8.028654871074489e-05, "loss": 0.017166707664728165, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.01731, "step": 177, "tokens/total": 6176576, "tokens/train_per_sec_per_gpu": 28.3, "tokens/trainable": 81325} +{"epoch": 0.6953125, "grad_norm": 0.4415320158004761, "learning_rate": 8.004589869885986e-05, "loss": 0.021312197670340538, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.2, "memory/max_allocated (GiB)": 37.2, "ppl": 1.02154, "step": 178, "tokens/total": 6208432, "tokens/train_per_sec_per_gpu": 26.02, "tokens/trainable": 81732} +{"epoch": 0.69921875, "grad_norm": 0.18043048679828644, "learning_rate": 7.980420642489674e-05, "loss": 0.004940127953886986, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.4, "memory/max_allocated (GiB)": 36.4, "ppl": 1.00495, "step": 179, "tokens/total": 6236640, "tokens/train_per_sec_per_gpu": 34.3, "tokens/trainable": 82171} +{"epoch": 0.703125, "grad_norm": 0.8491213321685791, "learning_rate": 7.95614819466576e-05, "loss": 0.0111582912504673, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.01122, "step": 180, "tokens/total": 6271184, "tokens/train_per_sec_per_gpu": 27.62, "tokens/trainable": 82670} +{"epoch": 0.70703125, "grad_norm": 0.37801775336265564, "learning_rate": 7.931773536489872e-05, "loss": 0.010486168786883354, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 35.89, "memory/max_allocated (GiB)": 35.89, "ppl": 1.01054, "step": 181, "tokens/total": 6301120, "tokens/train_per_sec_per_gpu": 25.52, "tokens/trainable": 83084} +{"epoch": 0.7109375, "grad_norm": 0.511384129524231, "learning_rate": 7.907297682291035e-05, "loss": 0.012008003890514374, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.01208, "step": 182, "tokens/total": 6334192, "tokens/train_per_sec_per_gpu": 25.93, "tokens/trainable": 83500} +{"epoch": 0.71484375, "grad_norm": 1.0708523988723755, "learning_rate": 7.882721650609442e-05, "loss": 0.012349812313914299, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.36, "memory/max_allocated (GiB)": 36.36, "ppl": 1.01243, "step": 183, "tokens/total": 6366192, "tokens/train_per_sec_per_gpu": 25.41, "tokens/trainable": 83896} +{"epoch": 0.71875, "grad_norm": 0.9347267746925354, "learning_rate": 7.85804646415409e-05, "loss": 0.012248925864696503, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.53, "memory/max_allocated (GiB)": 38.53, "ppl": 1.01232, "step": 184, "tokens/total": 6400496, "tokens/train_per_sec_per_gpu": 28.38, "tokens/trainable": 84399} +{"epoch": 0.72265625, "grad_norm": 0.5313665866851807, "learning_rate": 7.833273149760207e-05, "loss": 0.015071766451001167, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.01519, "step": 185, "tokens/total": 6434992, "tokens/train_per_sec_per_gpu": 26.4, "tokens/trainable": 84873} +{"epoch": 0.7265625, "grad_norm": 0.2806510925292969, "learning_rate": 7.808402738346527e-05, "loss": 0.010748554021120071, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.09, "memory/max_allocated (GiB)": 37.09, "ppl": 1.01081, "step": 186, "tokens/total": 6469296, "tokens/train_per_sec_per_gpu": 25.64, "tokens/trainable": 85296} +{"epoch": 0.73046875, "grad_norm": 0.642687976360321, "learning_rate": 7.783436264872382e-05, "loss": 0.02311106212437153, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.02338, "step": 187, "tokens/total": 6503760, "tokens/train_per_sec_per_gpu": 25.43, "tokens/trainable": 85741} +{"epoch": 0.734375, "grad_norm": 1.5499274730682373, "learning_rate": 7.758374768294647e-05, "loss": 0.007722441107034683, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00775, "step": 188, "tokens/total": 6540448, "tokens/train_per_sec_per_gpu": 23.21, "tokens/trainable": 86176} +{"epoch": 0.73828125, "grad_norm": 0.08465293049812317, "learning_rate": 7.733219291524489e-05, "loss": 0.0019100653007626534, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.00191, "step": 189, "tokens/total": 6577648, "tokens/train_per_sec_per_gpu": 24.93, "tokens/trainable": 86649} +{"epoch": 0.7421875, "grad_norm": 0.5645885467529297, "learning_rate": 7.707970881383977e-05, "loss": 0.014799621887505054, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.01491, "step": 190, "tokens/total": 6615600, "tokens/train_per_sec_per_gpu": 21.53, "tokens/trainable": 87084} +{"epoch": 0.74609375, "grad_norm": 0.6284784078598022, "learning_rate": 7.682630588562518e-05, "loss": 0.018953049555420876, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.01913, "step": 191, "tokens/total": 6648880, "tokens/train_per_sec_per_gpu": 24.64, "tokens/trainable": 87499} +{"epoch": 0.75, "grad_norm": 0.15247778594493866, "learning_rate": 7.657199467573129e-05, "loss": 0.0024877707473933697, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.00249, "step": 192, "tokens/total": 6686784, "tokens/train_per_sec_per_gpu": 25.82, "tokens/trainable": 87987} +{"epoch": 0.75390625, "grad_norm": 0.3328360319137573, "learning_rate": 7.631678576708561e-05, "loss": 0.005429980810731649, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.00544, "step": 193, "tokens/total": 6722480, "tokens/train_per_sec_per_gpu": 26.24, "tokens/trainable": 88470} +{"epoch": 0.7578125, "grad_norm": 0.5918972492218018, "learning_rate": 7.606068977997255e-05, "loss": 0.00917432364076376, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.00922, "step": 194, "tokens/total": 6757168, "tokens/train_per_sec_per_gpu": 26.3, "tokens/trainable": 88923} +{"epoch": 0.76171875, "grad_norm": 0.3906809985637665, "learning_rate": 7.580371737159148e-05, "loss": 0.01355939731001854, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.02, "memory/max_allocated (GiB)": 37.02, "ppl": 1.01365, "step": 195, "tokens/total": 6790000, "tokens/train_per_sec_per_gpu": 27.17, "tokens/trainable": 89338} +{"epoch": 0.765625, "grad_norm": 0.5378115177154541, "learning_rate": 7.554587923561324e-05, "loss": 0.007279795594513416, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.00731, "step": 196, "tokens/total": 6826384, "tokens/train_per_sec_per_gpu": 25.92, "tokens/trainable": 89812} +{"epoch": 0.76953125, "grad_norm": 0.4274922311306, "learning_rate": 7.528718610173511e-05, "loss": 0.014104041270911694, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.0142, "step": 197, "tokens/total": 6860896, "tokens/train_per_sec_per_gpu": 26.4, "tokens/trainable": 90252} +{"epoch": 0.7734375, "grad_norm": 0.31097888946533203, "learning_rate": 7.502764873523431e-05, "loss": 0.006503015290945768, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.28, "memory/max_allocated (GiB)": 39.28, "ppl": 1.00652, "step": 198, "tokens/total": 6896912, "tokens/train_per_sec_per_gpu": 26.29, "tokens/trainable": 90718} +{"epoch": 0.77734375, "grad_norm": 0.06044478714466095, "learning_rate": 7.476727793652011e-05, "loss": 0.0007468111580237746, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.00075, "step": 199, "tokens/total": 6928592, "tokens/train_per_sec_per_gpu": 30.02, "tokens/trainable": 91194} +{"epoch": 0.78125, "grad_norm": 0.12977129220962524, "learning_rate": 7.450608454068415e-05, "loss": 0.001569922547787428, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.00157, "step": 200, "tokens/total": 6964672, "tokens/train_per_sec_per_gpu": 24.24, "tokens/trainable": 91649} +{"epoch": 0.78515625, "grad_norm": 0.15626880526542664, "learning_rate": 7.424407941704987e-05, "loss": 0.002618565456941724, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.00262, "step": 201, "tokens/total": 6996336, "tokens/train_per_sec_per_gpu": 29.27, "tokens/trainable": 92078} +{"epoch": 0.7890625, "grad_norm": 0.2466612607240677, "learning_rate": 7.398127346871986e-05, "loss": 0.0037852483801543713, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.59, "memory/max_allocated (GiB)": 37.59, "ppl": 1.00379, "step": 202, "tokens/total": 7030544, "tokens/train_per_sec_per_gpu": 24.61, "tokens/trainable": 92536} +{"epoch": 0.79296875, "grad_norm": 0.24113406240940094, "learning_rate": 7.371767763212238e-05, "loss": 0.0021513861138373613, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.00215, "step": 203, "tokens/total": 7066480, "tokens/train_per_sec_per_gpu": 27.68, "tokens/trainable": 93032} +{"epoch": 0.796875, "grad_norm": 0.6289283037185669, "learning_rate": 7.345330287655617e-05, "loss": 0.0070047941990196705, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00703, "step": 204, "tokens/total": 7098768, "tokens/train_per_sec_per_gpu": 30.58, "tokens/trainable": 93504} +{"epoch": 0.80078125, "grad_norm": 1.4486637115478516, "learning_rate": 7.31881602037339e-05, "loss": 0.025697708129882812, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.02603, "step": 205, "tokens/total": 7132144, "tokens/train_per_sec_per_gpu": 23.17, "tokens/trainable": 93925} +{"epoch": 0.8046875, "grad_norm": 0.17705760896205902, "learning_rate": 7.29222606473245e-05, "loss": 0.0012255224864929914, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.00123, "step": 206, "tokens/total": 7164720, "tokens/train_per_sec_per_gpu": 27.76, "tokens/trainable": 94365} +{"epoch": 0.80859375, "grad_norm": 0.07331105321645737, "learning_rate": 7.265561527249383e-05, "loss": 0.0003872177912853658, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.00039, "step": 207, "tokens/total": 7199648, "tokens/train_per_sec_per_gpu": 24.73, "tokens/trainable": 94845} +{"epoch": 0.8125, "grad_norm": 0.47907567024230957, "learning_rate": 7.238823517544436e-05, "loss": 0.008763471618294716, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.0088, "step": 208, "tokens/total": 7235008, "tokens/train_per_sec_per_gpu": 26.68, "tokens/trainable": 95285} +{"epoch": 0.81640625, "grad_norm": 0.39338091015815735, "learning_rate": 7.212013148295333e-05, "loss": 0.011528338305652142, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.0116, "step": 209, "tokens/total": 7269776, "tokens/train_per_sec_per_gpu": 24.99, "tokens/trainable": 95758} +{"epoch": 0.8203125, "grad_norm": 0.9836149215698242, "learning_rate": 7.185131535190975e-05, "loss": 0.022197958081960678, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.59, "memory/max_allocated (GiB)": 39.59, "ppl": 1.02245, "step": 210, "tokens/total": 7306848, "tokens/train_per_sec_per_gpu": 23.63, "tokens/trainable": 96192} +{"epoch": 0.82421875, "grad_norm": 1.869092345237732, "learning_rate": 7.158179796885005e-05, "loss": 0.021892370656132698, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.02213, "step": 211, "tokens/total": 7345600, "tokens/train_per_sec_per_gpu": 23.85, "tokens/trainable": 96641} +{"epoch": 0.828125, "grad_norm": 1.68373703956604, "learning_rate": 7.131159054949273e-05, "loss": 0.010242211632430553, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.01029, "step": 212, "tokens/total": 7383040, "tokens/train_per_sec_per_gpu": 26.65, "tokens/trainable": 97160} +{"epoch": 0.83203125, "grad_norm": 0.5270278453826904, "learning_rate": 7.104070433827139e-05, "loss": 0.004533202853053808, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 36.1, "memory/max_allocated (GiB)": 36.1, "ppl": 1.00454, "step": 213, "tokens/total": 7414208, "tokens/train_per_sec_per_gpu": 28.52, "tokens/trainable": 97611} +{"epoch": 0.8359375, "grad_norm": 0.04292668402194977, "learning_rate": 7.076915060786705e-05, "loss": 0.0007612034678459167, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.00076, "step": 214, "tokens/total": 7448864, "tokens/train_per_sec_per_gpu": 28.26, "tokens/trainable": 98111} +{"epoch": 0.83984375, "grad_norm": 0.18815977871418, "learning_rate": 7.049694065873882e-05, "loss": 0.0027998865116387606, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.97, "memory/max_allocated (GiB)": 37.97, "ppl": 1.0028, "step": 215, "tokens/total": 7484096, "tokens/train_per_sec_per_gpu": 26.62, "tokens/trainable": 98575} +{"epoch": 0.84375, "grad_norm": 0.45951566100120544, "learning_rate": 7.022408581865382e-05, "loss": 0.010142074897885323, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.01019, "step": 216, "tokens/total": 7517056, "tokens/train_per_sec_per_gpu": 24.9, "tokens/trainable": 98991} +{"epoch": 0.84765625, "grad_norm": 0.3020154535770416, "learning_rate": 6.99505974422157e-05, "loss": 0.005271254573017359, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 36.32, "memory/max_allocated (GiB)": 36.32, "ppl": 1.00529, "step": 217, "tokens/total": 7548448, "tokens/train_per_sec_per_gpu": 33.6, "tokens/trainable": 99454} +{"epoch": 0.8515625, "grad_norm": 0.08542649447917938, "learning_rate": 6.967648691039213e-05, "loss": 0.0009859908604994416, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00099, "step": 218, "tokens/total": 7582656, "tokens/train_per_sec_per_gpu": 26.43, "tokens/trainable": 99925} +{"epoch": 0.85546875, "grad_norm": 0.10111958533525467, "learning_rate": 6.940176563004123e-05, "loss": 0.0020282957702875137, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00203, "step": 219, "tokens/total": 7618960, "tokens/train_per_sec_per_gpu": 26.2, "tokens/trainable": 100371} +{"epoch": 0.859375, "grad_norm": 0.3960461914539337, "learning_rate": 6.912644503343682e-05, "loss": 0.009337658993899822, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.35, "memory/max_allocated (GiB)": 39.35, "ppl": 1.00938, "step": 220, "tokens/total": 7655504, "tokens/train_per_sec_per_gpu": 26.15, "tokens/trainable": 100855} +{"epoch": 0.86328125, "grad_norm": 0.4703015983104706, "learning_rate": 6.885053657779273e-05, "loss": 0.004049813840538263, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00406, "step": 221, "tokens/total": 7689872, "tokens/train_per_sec_per_gpu": 21.18, "tokens/trainable": 101253} +{"epoch": 0.8671875, "grad_norm": 0.21715369820594788, "learning_rate": 6.857405174478604e-05, "loss": 0.0029084887355566025, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.00291, "step": 222, "tokens/total": 7724432, "tokens/train_per_sec_per_gpu": 26.04, "tokens/trainable": 101706} +{"epoch": 0.87109375, "grad_norm": 0.4790146052837372, "learning_rate": 6.82970020400792e-05, "loss": 0.005748391151428223, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.00576, "step": 223, "tokens/total": 7761440, "tokens/train_per_sec_per_gpu": 22.73, "tokens/trainable": 102158} +{"epoch": 0.875, "grad_norm": 0.25902289152145386, "learning_rate": 6.801939899284132e-05, "loss": 0.001611641957424581, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 35.36, "memory/max_allocated (GiB)": 35.36, "ppl": 1.00161, "step": 224, "tokens/total": 7790048, "tokens/train_per_sec_per_gpu": 30.31, "tokens/trainable": 102580} +{"epoch": 0.87890625, "grad_norm": 0.3852512538433075, "learning_rate": 6.774125415526827e-05, "loss": 0.01403848733752966, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.01414, "step": 225, "tokens/total": 7822352, "tokens/train_per_sec_per_gpu": 30.56, "tokens/trainable": 103063} +{"epoch": 0.8828125, "grad_norm": 1.9841676950454712, "learning_rate": 6.746257910210214e-05, "loss": 0.012851430103182793, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.01293, "step": 226, "tokens/total": 7861088, "tokens/train_per_sec_per_gpu": 22.07, "tokens/trainable": 103510} +{"epoch": 0.88671875, "grad_norm": 0.31088653206825256, "learning_rate": 6.718338543014937e-05, "loss": 0.003167141927406192, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.51, "memory/max_allocated (GiB)": 36.51, "ppl": 1.00317, "step": 227, "tokens/total": 7891456, "tokens/train_per_sec_per_gpu": 34.17, "tokens/trainable": 103980} +{"epoch": 0.890625, "grad_norm": 0.15432626008987427, "learning_rate": 6.69036847577983e-05, "loss": 0.0021354826167225838, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.13, "memory/max_allocated (GiB)": 36.13, "ppl": 1.00214, "step": 228, "tokens/total": 7922704, "tokens/train_per_sec_per_gpu": 28.58, "tokens/trainable": 104413} +{"epoch": 0.89453125, "grad_norm": 0.04400637000799179, "learning_rate": 6.662348872453553e-05, "loss": 0.000428524159360677, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.00043, "step": 229, "tokens/total": 7952912, "tokens/train_per_sec_per_gpu": 29.77, "tokens/trainable": 104845} +{"epoch": 0.8984375, "grad_norm": 0.3059447407722473, "learning_rate": 6.63428089904618e-05, "loss": 0.0035860773641616106, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.00359, "step": 230, "tokens/total": 7987504, "tokens/train_per_sec_per_gpu": 23.97, "tokens/trainable": 105289} +{"epoch": 0.90234375, "grad_norm": 0.651945948600769, "learning_rate": 6.60616572358065e-05, "loss": 0.011204248294234276, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 1.01127, "step": 231, "tokens/total": 8025936, "tokens/train_per_sec_per_gpu": 21.76, "tokens/trainable": 105722} +{"epoch": 0.90625, "grad_norm": 0.050452787429094315, "learning_rate": 6.578004516044172e-05, "loss": 0.0005768855335190892, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.59, "memory/max_allocated (GiB)": 36.59, "ppl": 1.00058, "step": 232, "tokens/total": 8058544, "tokens/train_per_sec_per_gpu": 29.16, "tokens/trainable": 106213} +{"epoch": 0.91015625, "grad_norm": 0.7759496569633484, "learning_rate": 6.549798448339548e-05, "loss": 0.008999299257993698, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 1.00904, "step": 233, "tokens/total": 8091456, "tokens/train_per_sec_per_gpu": 26.82, "tokens/trainable": 106643} +{"epoch": 0.9140625, "grad_norm": 0.290920227766037, "learning_rate": 6.521548694236384e-05, "loss": 0.007392689120024443, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.97, "memory/max_allocated (GiB)": 38.97, "ppl": 1.00742, "step": 234, "tokens/total": 8127920, "tokens/train_per_sec_per_gpu": 25.49, "tokens/trainable": 107136} +{"epoch": 0.91796875, "grad_norm": 0.28560662269592285, "learning_rate": 6.493256429322259e-05, "loss": 0.0022229072637856007, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00223, "step": 235, "tokens/total": 8167472, "tokens/train_per_sec_per_gpu": 24.48, "tokens/trainable": 107588} +{"epoch": 0.921875, "grad_norm": 1.2407822608947754, "learning_rate": 6.464922830953799e-05, "loss": 0.009097035974264145, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.00914, "step": 236, "tokens/total": 8201792, "tokens/train_per_sec_per_gpu": 27.56, "tokens/trainable": 108087} +{"epoch": 0.92578125, "grad_norm": 0.28057071566581726, "learning_rate": 6.436549078207688e-05, "loss": 0.0018607175443321466, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 1.00186, "step": 237, "tokens/total": 8235504, "tokens/train_per_sec_per_gpu": 27.19, "tokens/trainable": 108533} +{"epoch": 0.9296875, "grad_norm": 0.4028145670890808, "learning_rate": 6.408136351831592e-05, "loss": 0.0020817983895540237, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.24, "memory/max_allocated (GiB)": 38.24, "ppl": 1.00208, "step": 238, "tokens/total": 8271392, "tokens/train_per_sec_per_gpu": 21.65, "tokens/trainable": 108965} +{"epoch": 0.93359375, "grad_norm": 0.2585437297821045, "learning_rate": 6.379685834195036e-05, "loss": 0.004164727404713631, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.00417, "step": 239, "tokens/total": 8304656, "tokens/train_per_sec_per_gpu": 28.71, "tokens/trainable": 109416} +{"epoch": 0.9375, "grad_norm": 1.287636160850525, "learning_rate": 6.351198709240186e-05, "loss": 0.0019507886609062552, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.12, "memory/max_allocated (GiB)": 39.12, "ppl": 1.00195, "step": 240, "tokens/total": 8343280, "tokens/train_per_sec_per_gpu": 23.29, "tokens/trainable": 109864} +{"epoch": 0.94140625, "grad_norm": 0.8405837416648865, "learning_rate": 6.32267616243259e-05, "loss": 0.011199436150491238, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.01126, "step": 241, "tokens/total": 8377728, "tokens/train_per_sec_per_gpu": 26.23, "tokens/trainable": 110345} +{"epoch": 0.9453125, "grad_norm": 0.057336077094078064, "learning_rate": 6.294119380711849e-05, "loss": 0.0010908069089055061, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 37.15, "memory/max_allocated (GiB)": 37.15, "ppl": 1.00109, "step": 242, "tokens/total": 8412112, "tokens/train_per_sec_per_gpu": 24.74, "tokens/trainable": 110794} +{"epoch": 0.94921875, "grad_norm": 0.10522598773241043, "learning_rate": 6.265529552442209e-05, "loss": 0.0011955959489569068, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.0012, "step": 243, "tokens/total": 8449360, "tokens/train_per_sec_per_gpu": 24.95, "tokens/trainable": 111250} +{"epoch": 0.953125, "grad_norm": 0.36136341094970703, "learning_rate": 6.236907867363127e-05, "loss": 0.005922452546656132, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 1.00594, "step": 244, "tokens/total": 8482160, "tokens/train_per_sec_per_gpu": 26.95, "tokens/trainable": 111693} +{"epoch": 0.95703125, "grad_norm": 0.5515484809875488, "learning_rate": 6.208255516539749e-05, "loss": 0.005083566531538963, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.45, "memory/max_allocated (GiB)": 38.45, "ppl": 1.0051, "step": 245, "tokens/total": 8516272, "tokens/train_per_sec_per_gpu": 25.76, "tokens/trainable": 112158} +{"epoch": 0.9609375, "grad_norm": 0.8980147242546082, "learning_rate": 6.179573692313344e-05, "loss": 0.01022319309413433, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.98, "memory/max_allocated (GiB)": 36.98, "ppl": 1.01028, "step": 246, "tokens/total": 8548240, "tokens/train_per_sec_per_gpu": 28.8, "tokens/trainable": 112601} +{"epoch": 0.96484375, "grad_norm": 0.10358139872550964, "learning_rate": 6.150863588251694e-05, "loss": 0.0009605907835066319, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.34, "memory/max_allocated (GiB)": 36.34, "ppl": 1.00096, "step": 247, "tokens/total": 8580000, "tokens/train_per_sec_per_gpu": 25.78, "tokens/trainable": 113016} +{"epoch": 0.96875, "grad_norm": 0.3104684352874756, "learning_rate": 6.122126399099419e-05, "loss": 0.0018264243844896555, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.00183, "step": 248, "tokens/total": 8614864, "tokens/train_per_sec_per_gpu": 24.65, "tokens/trainable": 113476} +{"epoch": 0.97265625, "grad_norm": 0.9165666103363037, "learning_rate": 6.0933633207282615e-05, "loss": 0.020440377295017242, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.66, "memory/max_allocated (GiB)": 38.66, "ppl": 1.02065, "step": 249, "tokens/total": 8651008, "tokens/train_per_sec_per_gpu": 26.7, "tokens/trainable": 113941} +{"epoch": 0.9765625, "grad_norm": 0.3686591386795044, "learning_rate": 6.064575550087316e-05, "loss": 0.009066404774785042, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.83, "memory/max_allocated (GiB)": 36.83, "ppl": 1.00911, "step": 250, "tokens/total": 8683696, "tokens/train_per_sec_per_gpu": 30.27, "tokens/trainable": 114434} +{"epoch": 0.98046875, "grad_norm": 0.017512163147330284, "learning_rate": 6.0357642851532245e-05, "loss": 0.0003406123141758144, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.28, "memory/max_allocated (GiB)": 36.28, "ppl": 1.00034, "step": 251, "tokens/total": 8714880, "tokens/train_per_sec_per_gpu": 29.7, "tokens/trainable": 114898} +{"epoch": 0.984375, "grad_norm": 0.21562740206718445, "learning_rate": 6.0069307248803294e-05, "loss": 0.001281405915506184, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.15, "memory/max_allocated (GiB)": 39.15, "ppl": 1.00128, "step": 252, "tokens/total": 8751072, "tokens/train_per_sec_per_gpu": 21.69, "tokens/trainable": 115331} +{"epoch": 0.98828125, "grad_norm": 0.4894140660762787, "learning_rate": 5.9780760691507635e-05, "loss": 0.002343815518543124, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.73, "memory/max_allocated (GiB)": 36.73, "ppl": 1.00235, "step": 253, "tokens/total": 8782272, "tokens/train_per_sec_per_gpu": 28.04, "tokens/trainable": 115732} +{"epoch": 0.9921875, "grad_norm": 0.160071462392807, "learning_rate": 5.9492015187245334e-05, "loss": 0.0022703527938574553, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.28, "memory/max_allocated (GiB)": 39.28, "ppl": 1.00227, "step": 254, "tokens/total": 8819920, "tokens/train_per_sec_per_gpu": 25.63, "tokens/trainable": 116221} +{"epoch": 0.99609375, "grad_norm": 0.9661566615104675, "learning_rate": 5.920308275189541e-05, "loss": 0.020485863089561462, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 37.91, "memory/max_allocated (GiB)": 37.91, "ppl": 1.0207, "step": 255, "tokens/total": 8855232, "tokens/train_per_sec_per_gpu": 23.79, "tokens/trainable": 116684} +{"epoch": 1.0, "grad_norm": 0.06835681945085526, "learning_rate": 5.8913975409115874e-05, "loss": 0.0006298700463958085, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.00063, "step": 256, "tokens/total": 8889920, "tokens/train_per_sec_per_gpu": 24.69, "tokens/trainable": 117126} +{"epoch": 1.00390625, "grad_norm": 0.36512529850006104, "learning_rate": 5.8624705189843395e-05, "loss": 0.003548750886693597, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00356, "step": 257, "tokens/total": 8922640, "tokens/train_per_sec_per_gpu": 30.16, "tokens/trainable": 117592} +{"epoch": 1.0078125, "grad_norm": 0.1385437697172165, "learning_rate": 5.833528413179249e-05, "loss": 0.0006680436199530959, "memory/device_reserved (GiB)": 39.79, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00067, "step": 258, "tokens/total": 8958848, "tokens/train_per_sec_per_gpu": 21.65, "tokens/trainable": 118030} +{"epoch": 1.01171875, "grad_norm": 0.6757524013519287, "learning_rate": 5.80457242789548e-05, "loss": 0.016566202044487, "memory/device_reserved (GiB)": 39.79, "memory/max_active (GiB)": 36.51, "memory/max_allocated (GiB)": 36.51, "ppl": 1.0167, "step": 259, "tokens/total": 8991056, "tokens/train_per_sec_per_gpu": 26.43, "tokens/trainable": 118470} +{"epoch": 1.015625, "grad_norm": 0.9127917289733887, "learning_rate": 5.77560376810977e-05, "loss": 0.026579374447464943, "memory/device_reserved (GiB)": 39.79, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.02694, "step": 260, "tokens/total": 9024720, "tokens/train_per_sec_per_gpu": 28.38, "tokens/trainable": 118935} +{"epoch": 1.01953125, "grad_norm": 0.2361977994441986, "learning_rate": 5.7466236393263005e-05, "loss": 0.006779455579817295, "memory/device_reserved (GiB)": 39.79, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.0068, "step": 261, "tokens/total": 9058944, "tokens/train_per_sec_per_gpu": 25.58, "tokens/trainable": 119381} +{"epoch": 1.0234375, "grad_norm": 0.4300181269645691, "learning_rate": 5.717633247526522e-05, "loss": 0.006824871990829706, "memory/device_reserved (GiB)": 41.75, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.00685, "step": 262, "tokens/total": 9098816, "tokens/train_per_sec_per_gpu": 23.26, "tokens/trainable": 119849} +{"epoch": 1.02734375, "grad_norm": 0.7225202918052673, "learning_rate": 5.688633799118971e-05, "loss": 0.004233082290738821, "memory/device_reserved (GiB)": 41.75, "memory/max_active (GiB)": 36.78, "memory/max_allocated (GiB)": 36.78, "ppl": 1.00424, "step": 263, "tokens/total": 9132080, "tokens/train_per_sec_per_gpu": 29.17, "tokens/trainable": 120319} +{"epoch": 1.03125, "grad_norm": 0.3337777554988861, "learning_rate": 5.659626500889066e-05, "loss": 0.005079996306449175, "memory/device_reserved (GiB)": 41.75, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.00509, "step": 264, "tokens/total": 9165072, "tokens/train_per_sec_per_gpu": 31.06, "tokens/trainable": 120804} +{"epoch": 1.03515625, "grad_norm": 0.1541646420955658, "learning_rate": 5.6306125599488905e-05, "loss": 0.0012254633475095034, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 35.9, "memory/max_allocated (GiB)": 35.9, "ppl": 1.00123, "step": 265, "tokens/total": 9195776, "tokens/train_per_sec_per_gpu": 29.4, "tokens/trainable": 121233} +{"epoch": 1.0390625, "grad_norm": 0.10225948691368103, "learning_rate": 5.601593183686955e-05, "loss": 0.0018750306917354465, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.95, "memory/max_allocated (GiB)": 37.95, "ppl": 1.00188, "step": 266, "tokens/total": 9231136, "tokens/train_per_sec_per_gpu": 22.6, "tokens/trainable": 121671} +{"epoch": 1.04296875, "grad_norm": 0.10669223964214325, "learning_rate": 5.572569579717961e-05, "loss": 0.0008985726162791252, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.0009, "step": 267, "tokens/total": 9266784, "tokens/train_per_sec_per_gpu": 25.74, "tokens/trainable": 122141} +{"epoch": 1.046875, "grad_norm": 0.18004898726940155, "learning_rate": 5.543542955832538e-05, "loss": 0.0028351738583296537, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.37, "memory/max_allocated (GiB)": 38.37, "ppl": 1.00284, "step": 268, "tokens/total": 9301648, "tokens/train_per_sec_per_gpu": 25.16, "tokens/trainable": 122556} +{"epoch": 1.05078125, "grad_norm": 0.8845915198326111, "learning_rate": 5.514514519946986e-05, "loss": 0.034161750227212906, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.03475, "step": 269, "tokens/total": 9334352, "tokens/train_per_sec_per_gpu": 27.4, "tokens/trainable": 122988} +{"epoch": 1.0546875, "grad_norm": 0.050799887627363205, "learning_rate": 5.485485480053015e-05, "loss": 0.0008339755004271865, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.00083, "step": 270, "tokens/total": 9369376, "tokens/train_per_sec_per_gpu": 27.03, "tokens/trainable": 123478} +{"epoch": 1.05859375, "grad_norm": 0.4858318865299225, "learning_rate": 5.4564570441674645e-05, "loss": 0.006438517011702061, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.00646, "step": 271, "tokens/total": 9402928, "tokens/train_per_sec_per_gpu": 28.88, "tokens/trainable": 123932} +{"epoch": 1.0625, "grad_norm": 0.2802765369415283, "learning_rate": 5.42743042028204e-05, "loss": 0.004715262912213802, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.00473, "step": 272, "tokens/total": 9437952, "tokens/train_per_sec_per_gpu": 29.54, "tokens/trainable": 124428} +{"epoch": 1.06640625, "grad_norm": 0.7065048217773438, "learning_rate": 5.3984068163130464e-05, "loss": 0.012475689873099327, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.01255, "step": 273, "tokens/total": 9471648, "tokens/train_per_sec_per_gpu": 23.01, "tokens/trainable": 124866} +{"epoch": 1.0703125, "grad_norm": 0.32236942648887634, "learning_rate": 5.369387440051111e-05, "loss": 0.008286792784929276, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.00832, "step": 274, "tokens/total": 9508640, "tokens/train_per_sec_per_gpu": 24.64, "tokens/trainable": 125324} +{"epoch": 1.07421875, "grad_norm": 0.1321016103029251, "learning_rate": 5.340373499110935e-05, "loss": 0.002111597917973995, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.00211, "step": 275, "tokens/total": 9548560, "tokens/train_per_sec_per_gpu": 24.21, "tokens/trainable": 125813} +{"epoch": 1.078125, "grad_norm": 0.28285300731658936, "learning_rate": 5.3113662008810304e-05, "loss": 0.0036429394967854023, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00365, "step": 276, "tokens/total": 9584608, "tokens/train_per_sec_per_gpu": 22.3, "tokens/trainable": 126252} +{"epoch": 1.08203125, "grad_norm": 0.2699023485183716, "learning_rate": 5.282366752473479e-05, "loss": 0.002756339032202959, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.99, "memory/max_allocated (GiB)": 38.99, "ppl": 1.00276, "step": 277, "tokens/total": 9620656, "tokens/train_per_sec_per_gpu": 25.04, "tokens/trainable": 126696} +{"epoch": 1.0859375, "grad_norm": 0.15753215551376343, "learning_rate": 5.2533763606737005e-05, "loss": 0.0033718536142259836, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.00338, "step": 278, "tokens/total": 9652992, "tokens/train_per_sec_per_gpu": 28.76, "tokens/trainable": 127173} +{"epoch": 1.08984375, "grad_norm": 0.15637750923633575, "learning_rate": 5.224396231890232e-05, "loss": 0.0024877956602722406, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.00249, "step": 279, "tokens/total": 9685632, "tokens/train_per_sec_per_gpu": 27.91, "tokens/trainable": 127637} +{"epoch": 1.09375, "grad_norm": 0.3261530101299286, "learning_rate": 5.195427572104522e-05, "loss": 0.005356606096029282, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.59, "memory/max_allocated (GiB)": 38.59, "ppl": 1.00537, "step": 280, "tokens/total": 9723056, "tokens/train_per_sec_per_gpu": 24.88, "tokens/trainable": 128112} +{"epoch": 1.09765625, "grad_norm": 0.33975905179977417, "learning_rate": 5.166471586820751e-05, "loss": 0.005630259867757559, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.73, "memory/max_allocated (GiB)": 37.73, "ppl": 1.00565, "step": 281, "tokens/total": 9758592, "tokens/train_per_sec_per_gpu": 26.33, "tokens/trainable": 128584} +{"epoch": 1.1015625, "grad_norm": 0.12815305590629578, "learning_rate": 5.1375294810156615e-05, "loss": 0.0019097018521279097, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.3, "memory/max_allocated (GiB)": 38.3, "ppl": 1.00191, "step": 282, "tokens/total": 9793408, "tokens/train_per_sec_per_gpu": 24.79, "tokens/trainable": 129051} +{"epoch": 1.10546875, "grad_norm": 0.08426477015018463, "learning_rate": 5.1086024590884144e-05, "loss": 0.0006812559440732002, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.15, "memory/max_allocated (GiB)": 37.15, "ppl": 1.00068, "step": 283, "tokens/total": 9827216, "tokens/train_per_sec_per_gpu": 25.6, "tokens/trainable": 129494} +{"epoch": 1.109375, "grad_norm": 0.2561304569244385, "learning_rate": 5.079691724810461e-05, "loss": 0.0027832351624965668, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 39.35, "memory/max_allocated (GiB)": 39.35, "ppl": 1.00279, "step": 284, "tokens/total": 9864928, "tokens/train_per_sec_per_gpu": 24.74, "tokens/trainable": 129983} +{"epoch": 1.11328125, "grad_norm": 0.36931437253952026, "learning_rate": 5.0507984812754684e-05, "loss": 0.005564114078879356, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00558, "step": 285, "tokens/total": 9901520, "tokens/train_per_sec_per_gpu": 23.58, "tokens/trainable": 130405} +{"epoch": 1.1171875, "grad_norm": 0.07484454661607742, "learning_rate": 5.021923930849237e-05, "loss": 0.0005364238168112934, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.56, "memory/max_allocated (GiB)": 37.56, "ppl": 1.00054, "step": 286, "tokens/total": 9935824, "tokens/train_per_sec_per_gpu": 24.75, "tokens/trainable": 130829} +{"epoch": 1.12109375, "grad_norm": 0.5177130103111267, "learning_rate": 4.99306927511967e-05, "loss": 0.006245115306228399, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.00626, "step": 287, "tokens/total": 9969200, "tokens/train_per_sec_per_gpu": 24.51, "tokens/trainable": 131287} +{"epoch": 1.125, "grad_norm": 0.05912165343761444, "learning_rate": 4.964235714846775e-05, "loss": 0.0004434054426383227, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.00044, "step": 288, "tokens/total": 10005120, "tokens/train_per_sec_per_gpu": 23.67, "tokens/trainable": 131712} +{"epoch": 1.12890625, "grad_norm": 0.2624092996120453, "learning_rate": 4.9354244499126866e-05, "loss": 0.001142272842116654, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00114, "step": 289, "tokens/total": 10040336, "tokens/train_per_sec_per_gpu": 25.4, "tokens/trainable": 132168} +{"epoch": 1.1328125, "grad_norm": 0.005330358166247606, "learning_rate": 4.90663667927174e-05, "loss": 7.2502538387198e-05, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.00007, "step": 290, "tokens/total": 10075232, "tokens/train_per_sec_per_gpu": 27.21, "tokens/trainable": 132626} +{"epoch": 1.13671875, "grad_norm": 0.13255910575389862, "learning_rate": 4.877873600900581e-05, "loss": 0.0010433749994263053, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.00104, "step": 291, "tokens/total": 10113632, "tokens/train_per_sec_per_gpu": 24.24, "tokens/trainable": 133119} +{"epoch": 1.140625, "grad_norm": 1.4352149963378906, "learning_rate": 4.849136411748306e-05, "loss": 0.005343926139175892, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 36.96, "memory/max_allocated (GiB)": 36.96, "ppl": 1.00536, "step": 292, "tokens/total": 10145728, "tokens/train_per_sec_per_gpu": 27.31, "tokens/trainable": 133550} +{"epoch": 1.14453125, "grad_norm": 0.5468560457229614, "learning_rate": 4.8204263076866574e-05, "loss": 0.0016551627777516842, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.39, "memory/max_allocated (GiB)": 38.39, "ppl": 1.00166, "step": 293, "tokens/total": 10182512, "tokens/train_per_sec_per_gpu": 24.9, "tokens/trainable": 133992} +{"epoch": 1.1484375, "grad_norm": 0.2699575126171112, "learning_rate": 4.791744483460251e-05, "loss": 0.0029310546815395355, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.54, "memory/max_allocated (GiB)": 39.54, "ppl": 1.00294, "step": 294, "tokens/total": 10218144, "tokens/train_per_sec_per_gpu": 28.81, "tokens/trainable": 134467} +{"epoch": 1.15234375, "grad_norm": 1.3810572624206543, "learning_rate": 4.7630921326368736e-05, "loss": 0.0076267775148153305, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.00766, "step": 295, "tokens/total": 10256160, "tokens/train_per_sec_per_gpu": 20.17, "tokens/trainable": 134895} +{"epoch": 1.15625, "grad_norm": 1.6657700538635254, "learning_rate": 4.7344704475577916e-05, "loss": 0.007198171224445105, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 35.47, "memory/max_allocated (GiB)": 35.47, "ppl": 1.00722, "step": 296, "tokens/total": 10285136, "tokens/train_per_sec_per_gpu": 31.13, "tokens/trainable": 135321} +{"epoch": 1.16015625, "grad_norm": 0.9831333756446838, "learning_rate": 4.705880619288153e-05, "loss": 0.012807527557015419, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.01289, "step": 297, "tokens/total": 10321424, "tokens/train_per_sec_per_gpu": 23.84, "tokens/trainable": 135765} +{"epoch": 1.1640625, "grad_norm": 0.29403823614120483, "learning_rate": 4.677323837567412e-05, "loss": 0.0009802361018955708, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.00098, "step": 298, "tokens/total": 10359248, "tokens/train_per_sec_per_gpu": 24.84, "tokens/trainable": 136196} +{"epoch": 1.16796875, "grad_norm": 0.09816121309995651, "learning_rate": 4.6488012907598146e-05, "loss": 0.0010941497748717666, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.81, "memory/max_allocated (GiB)": 38.81, "ppl": 1.00109, "step": 299, "tokens/total": 10395824, "tokens/train_per_sec_per_gpu": 24.57, "tokens/trainable": 136649} +{"epoch": 1.171875, "grad_norm": 0.2510676383972168, "learning_rate": 4.620314165804964e-05, "loss": 0.0015342250699177384, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00154, "step": 300, "tokens/total": 10432864, "tokens/train_per_sec_per_gpu": 26.82, "tokens/trainable": 137128} +{"epoch": 1.17578125, "grad_norm": 0.11232379078865051, "learning_rate": 4.591863648168407e-05, "loss": 0.0009000928839668632, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.04, "memory/max_allocated (GiB)": 37.04, "ppl": 1.0009, "step": 301, "tokens/total": 10466816, "tokens/train_per_sec_per_gpu": 30.56, "tokens/trainable": 137585} +{"epoch": 1.1796875, "grad_norm": 0.5428605079650879, "learning_rate": 4.5634509217923135e-05, "loss": 0.0015447704354301095, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.00155, "step": 302, "tokens/total": 10499280, "tokens/train_per_sec_per_gpu": 31.38, "tokens/trainable": 138054} +{"epoch": 1.18359375, "grad_norm": 23.37395477294922, "learning_rate": 4.535077169046201e-05, "loss": 0.0016961111687123775, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.0, "memory/max_allocated (GiB)": 38.0, "ppl": 1.0017, "step": 303, "tokens/total": 10534112, "tokens/train_per_sec_per_gpu": 28.85, "tokens/trainable": 138559} +{"epoch": 1.1875, "grad_norm": 1.161377191543579, "learning_rate": 4.506743570677743e-05, "loss": 0.0016901554772630334, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.53, "memory/max_allocated (GiB)": 37.53, "ppl": 1.00169, "step": 304, "tokens/total": 10568912, "tokens/train_per_sec_per_gpu": 24.42, "tokens/trainable": 139050} +{"epoch": 1.19140625, "grad_norm": 0.01873447187244892, "learning_rate": 4.478451305763618e-05, "loss": 0.00027980684535577893, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.00028, "step": 305, "tokens/total": 10606128, "tokens/train_per_sec_per_gpu": 24.95, "tokens/trainable": 139525} +{"epoch": 1.1953125, "grad_norm": 0.0023935800418257713, "learning_rate": 4.450201551660454e-05, "loss": 4.0324961446458474e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.08, "memory/max_allocated (GiB)": 39.08, "ppl": 1.00004, "step": 306, "tokens/total": 10639584, "tokens/train_per_sec_per_gpu": 26.72, "tokens/trainable": 139969} +{"epoch": 1.19921875, "grad_norm": 0.05695829913020134, "learning_rate": 4.4219954839558276e-05, "loss": 0.00036694988375529647, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.44, "memory/max_allocated (GiB)": 36.44, "ppl": 1.00037, "step": 307, "tokens/total": 10670928, "tokens/train_per_sec_per_gpu": 31.76, "tokens/trainable": 140443} +{"epoch": 1.203125, "grad_norm": 0.009119276888668537, "learning_rate": 4.393834276419352e-05, "loss": 3.9199348975671455e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.00004, "step": 308, "tokens/total": 10709968, "tokens/train_per_sec_per_gpu": 24.11, "tokens/trainable": 140926} +{"epoch": 1.20703125, "grad_norm": 0.037707433104515076, "learning_rate": 4.36571910095382e-05, "loss": 0.000214775835047476, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.00021, "step": 309, "tokens/total": 10744800, "tokens/train_per_sec_per_gpu": 24.32, "tokens/trainable": 141344} +{"epoch": 1.2109375, "grad_norm": 0.7140233516693115, "learning_rate": 4.337651127546448e-05, "loss": 0.010176504030823708, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.01023, "step": 310, "tokens/total": 10778576, "tokens/train_per_sec_per_gpu": 26.33, "tokens/trainable": 141753} +{"epoch": 1.21484375, "grad_norm": 0.3895496129989624, "learning_rate": 4.3096315242201736e-05, "loss": 0.005969467107206583, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.61, "memory/max_allocated (GiB)": 36.61, "ppl": 1.00599, "step": 311, "tokens/total": 10810816, "tokens/train_per_sec_per_gpu": 31.39, "tokens/trainable": 142219} +{"epoch": 1.21875, "grad_norm": 0.5744476914405823, "learning_rate": 4.2816614569850635e-05, "loss": 0.027113577350974083, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.01, "memory/max_allocated (GiB)": 36.01, "ppl": 1.02748, "step": 312, "tokens/total": 10841040, "tokens/train_per_sec_per_gpu": 25.43, "tokens/trainable": 142617} +{"epoch": 1.22265625, "grad_norm": 1.3920468091964722, "learning_rate": 4.2537420897897864e-05, "loss": 0.00419148663058877, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.0042, "step": 313, "tokens/total": 10880960, "tokens/train_per_sec_per_gpu": 21.47, "tokens/trainable": 143103} +{"epoch": 1.2265625, "grad_norm": 0.029897330328822136, "learning_rate": 4.225874584473174e-05, "loss": 0.000293291755951941, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00029, "step": 314, "tokens/total": 10913472, "tokens/train_per_sec_per_gpu": 25.34, "tokens/trainable": 143535} +{"epoch": 1.23046875, "grad_norm": 0.030471835285425186, "learning_rate": 4.19806010071587e-05, "loss": 0.0003517797158565372, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00035, "step": 315, "tokens/total": 10951360, "tokens/train_per_sec_per_gpu": 23.9, "tokens/trainable": 144009} +{"epoch": 1.234375, "grad_norm": 0.17726454138755798, "learning_rate": 4.170299795992081e-05, "loss": 0.0016203559935092926, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.00162, "step": 316, "tokens/total": 10987808, "tokens/train_per_sec_per_gpu": 23.37, "tokens/trainable": 144460} +{"epoch": 1.23828125, "grad_norm": 0.036404307931661606, "learning_rate": 4.142594825521398e-05, "loss": 0.00036437626113183796, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.00036, "step": 317, "tokens/total": 11025552, "tokens/train_per_sec_per_gpu": 24.72, "tokens/trainable": 144936} +{"epoch": 1.2421875, "grad_norm": 0.02198062464594841, "learning_rate": 4.114946342220728e-05, "loss": 0.0001598758390173316, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.00016, "step": 318, "tokens/total": 11060432, "tokens/train_per_sec_per_gpu": 25.23, "tokens/trainable": 145375} +{"epoch": 1.24609375, "grad_norm": 0.29177284240722656, "learning_rate": 4.087355496656321e-05, "loss": 0.0015535210259258747, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.00155, "step": 319, "tokens/total": 11094688, "tokens/train_per_sec_per_gpu": 25.59, "tokens/trainable": 145849} +{"epoch": 1.25, "grad_norm": 0.06442433595657349, "learning_rate": 4.05982343699588e-05, "loss": 0.00103632896207273, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00104, "step": 320, "tokens/total": 11127408, "tokens/train_per_sec_per_gpu": 27.0, "tokens/trainable": 146318} +{"epoch": 1.25390625, "grad_norm": 0.8769434690475464, "learning_rate": 4.0323513089607876e-05, "loss": 0.006257006898522377, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.72, "memory/max_allocated (GiB)": 38.72, "ppl": 1.00628, "step": 321, "tokens/total": 11162752, "tokens/train_per_sec_per_gpu": 26.72, "tokens/trainable": 146821} +{"epoch": 1.2578125, "grad_norm": 0.23915189504623413, "learning_rate": 4.004940255778431e-05, "loss": 0.001917426474392414, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.33, "memory/max_allocated (GiB)": 36.33, "ppl": 1.00192, "step": 322, "tokens/total": 11194576, "tokens/train_per_sec_per_gpu": 26.47, "tokens/trainable": 147285} +{"epoch": 1.26171875, "grad_norm": 0.1442219316959381, "learning_rate": 3.977591418134619e-05, "loss": 0.0012123220367357135, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.61, "memory/max_allocated (GiB)": 36.61, "ppl": 1.00121, "step": 323, "tokens/total": 11226928, "tokens/train_per_sec_per_gpu": 25.69, "tokens/trainable": 147729} +{"epoch": 1.265625, "grad_norm": 1.6937973499298096, "learning_rate": 3.95030593412612e-05, "loss": 0.010046075098216534, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.0101, "step": 324, "tokens/total": 11266016, "tokens/train_per_sec_per_gpu": 20.61, "tokens/trainable": 148169} +{"epoch": 1.26953125, "grad_norm": 5.980107307434082, "learning_rate": 3.923084939213296e-05, "loss": 0.0049369181506335735, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.00495, "step": 325, "tokens/total": 11303264, "tokens/train_per_sec_per_gpu": 24.09, "tokens/trainable": 148630} +{"epoch": 1.2734375, "grad_norm": 0.4031008183956146, "learning_rate": 3.895929566172861e-05, "loss": 0.0035076886415481567, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00351, "step": 326, "tokens/total": 11339392, "tokens/train_per_sec_per_gpu": 22.6, "tokens/trainable": 149063} +{"epoch": 1.27734375, "grad_norm": 0.13447874784469604, "learning_rate": 3.868840945050728e-05, "loss": 0.000986834173090756, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.00099, "step": 327, "tokens/total": 11377568, "tokens/train_per_sec_per_gpu": 23.18, "tokens/trainable": 149549} +{"epoch": 1.28125, "grad_norm": 0.1655663400888443, "learning_rate": 3.841820203114995e-05, "loss": 0.0013698041439056396, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.03, "memory/max_allocated (GiB)": 38.03, "ppl": 1.00137, "step": 328, "tokens/total": 11413184, "tokens/train_per_sec_per_gpu": 26.82, "tokens/trainable": 150015} +{"epoch": 1.28515625, "grad_norm": 0.13398008048534393, "learning_rate": 3.814868464809027e-05, "loss": 0.0007669528131373227, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.00077, "step": 329, "tokens/total": 11447904, "tokens/train_per_sec_per_gpu": 28.13, "tokens/trainable": 150514} +{"epoch": 1.2890625, "grad_norm": 0.16956301033496857, "learning_rate": 3.787986851704667e-05, "loss": 0.0015984586207196116, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.0016, "step": 330, "tokens/total": 11485056, "tokens/train_per_sec_per_gpu": 27.53, "tokens/trainable": 151021} +{"epoch": 1.29296875, "grad_norm": 0.4532615542411804, "learning_rate": 3.7611764824555654e-05, "loss": 0.0026164441369473934, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.37, "memory/max_allocated (GiB)": 38.37, "ppl": 1.00262, "step": 331, "tokens/total": 11518848, "tokens/train_per_sec_per_gpu": 29.83, "tokens/trainable": 151486} +{"epoch": 1.296875, "grad_norm": 0.3504033386707306, "learning_rate": 3.734438472750619e-05, "loss": 0.002420115517452359, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.79, "memory/max_allocated (GiB)": 36.79, "ppl": 1.00242, "step": 332, "tokens/total": 11552048, "tokens/train_per_sec_per_gpu": 26.37, "tokens/trainable": 151956} +{"epoch": 1.30078125, "grad_norm": 0.07427746057510376, "learning_rate": 3.707773935267552e-05, "loss": 0.0003787295427173376, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.15, "memory/max_allocated (GiB)": 39.15, "ppl": 1.00038, "step": 333, "tokens/total": 11589168, "tokens/train_per_sec_per_gpu": 24.89, "tokens/trainable": 152442} +{"epoch": 1.3046875, "grad_norm": 0.2475174367427826, "learning_rate": 3.68118397962661e-05, "loss": 0.0016843078192323446, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.74, "memory/max_allocated (GiB)": 36.74, "ppl": 1.00169, "step": 334, "tokens/total": 11622032, "tokens/train_per_sec_per_gpu": 25.84, "tokens/trainable": 152890} +{"epoch": 1.30859375, "grad_norm": 1.2098990678787231, "learning_rate": 3.654669712344384e-05, "loss": 0.010724197141826153, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.35, "memory/max_allocated (GiB)": 36.35, "ppl": 1.01078, "step": 335, "tokens/total": 11653840, "tokens/train_per_sec_per_gpu": 30.86, "tokens/trainable": 153368} +{"epoch": 1.3125, "grad_norm": 0.00898852851241827, "learning_rate": 3.628232236787763e-05, "loss": 0.00014128875045571476, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00014, "step": 336, "tokens/total": 11689664, "tokens/train_per_sec_per_gpu": 26.42, "tokens/trainable": 153809} +{"epoch": 1.31640625, "grad_norm": 0.06052198261022568, "learning_rate": 3.6018726531280144e-05, "loss": 0.0005006770370528102, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.06, "memory/max_allocated (GiB)": 37.06, "ppl": 1.0005, "step": 337, "tokens/total": 11723728, "tokens/train_per_sec_per_gpu": 25.85, "tokens/trainable": 154258} +{"epoch": 1.3203125, "grad_norm": 0.6766254305839539, "learning_rate": 3.575592058295017e-05, "loss": 0.01748242974281311, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.01764, "step": 338, "tokens/total": 11758960, "tokens/train_per_sec_per_gpu": 25.0, "tokens/trainable": 154704} +{"epoch": 1.32421875, "grad_norm": 0.010956168174743652, "learning_rate": 3.549391545931585e-05, "loss": 0.00018840175471268594, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.00019, "step": 339, "tokens/total": 11797328, "tokens/train_per_sec_per_gpu": 24.03, "tokens/trainable": 155157} +{"epoch": 1.328125, "grad_norm": 0.1314038783311844, "learning_rate": 3.5232722063479914e-05, "loss": 0.0012205373495817184, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.07, "memory/max_allocated (GiB)": 38.07, "ppl": 1.00122, "step": 340, "tokens/total": 11832656, "tokens/train_per_sec_per_gpu": 23.79, "tokens/trainable": 155582} +{"epoch": 1.33203125, "grad_norm": 0.7138535976409912, "learning_rate": 3.49723512647657e-05, "loss": 0.004139328375458717, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.00415, "step": 341, "tokens/total": 11865152, "tokens/train_per_sec_per_gpu": 28.16, "tokens/trainable": 156047} +{"epoch": 1.3359375, "grad_norm": 0.11911232769489288, "learning_rate": 3.471281389826491e-05, "loss": 0.002922627842053771, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.99, "memory/max_allocated (GiB)": 38.99, "ppl": 1.00293, "step": 342, "tokens/total": 11901552, "tokens/train_per_sec_per_gpu": 25.86, "tokens/trainable": 156500} +{"epoch": 1.33984375, "grad_norm": 0.01882646605372429, "learning_rate": 3.4454120764386764e-05, "loss": 0.0003005993494298309, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.0003, "step": 343, "tokens/total": 11937328, "tokens/train_per_sec_per_gpu": 24.97, "tokens/trainable": 156931} +{"epoch": 1.34375, "grad_norm": 0.2627015709877014, "learning_rate": 3.4196282628408526e-05, "loss": 0.004389687906950712, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.0044, "step": 344, "tokens/total": 11970736, "tokens/train_per_sec_per_gpu": 25.67, "tokens/trainable": 157385} +{"epoch": 1.34765625, "grad_norm": 0.003787324298173189, "learning_rate": 3.3939310220027456e-05, "loss": 6.922500324435532e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.55, "memory/max_allocated (GiB)": 39.55, "ppl": 1.00007, "step": 345, "tokens/total": 12008096, "tokens/train_per_sec_per_gpu": 22.93, "tokens/trainable": 157817} +{"epoch": 1.3515625, "grad_norm": 0.46833473443984985, "learning_rate": 3.3683214232914404e-05, "loss": 0.003941370639950037, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.00395, "step": 346, "tokens/total": 12039904, "tokens/train_per_sec_per_gpu": 26.54, "tokens/trainable": 158273} +{"epoch": 1.35546875, "grad_norm": 0.047748416662216187, "learning_rate": 3.342800532426873e-05, "loss": 0.0007776974234730005, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.2, "memory/max_allocated (GiB)": 37.2, "ppl": 1.00078, "step": 347, "tokens/total": 12074416, "tokens/train_per_sec_per_gpu": 26.21, "tokens/trainable": 158728} +{"epoch": 1.359375, "grad_norm": 0.01122809574007988, "learning_rate": 3.317369411437484e-05, "loss": 0.00016458721074741334, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.12, "memory/max_allocated (GiB)": 37.12, "ppl": 1.00016, "step": 348, "tokens/total": 12108432, "tokens/train_per_sec_per_gpu": 24.67, "tokens/trainable": 159168} +{"epoch": 1.36328125, "grad_norm": 0.12392763048410416, "learning_rate": 3.292029118616024e-05, "loss": 0.0020677833817899227, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.00207, "step": 349, "tokens/total": 12141792, "tokens/train_per_sec_per_gpu": 23.3, "tokens/trainable": 159592} +{"epoch": 1.3671875, "grad_norm": 0.05857398360967636, "learning_rate": 3.266780708475511e-05, "loss": 0.0004523419192992151, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.00045, "step": 350, "tokens/total": 12175840, "tokens/train_per_sec_per_gpu": 29.57, "tokens/trainable": 160093} +{"epoch": 1.37109375, "grad_norm": 0.07740240544080734, "learning_rate": 3.241625231705354e-05, "loss": 0.0016740105347707868, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.56, "memory/max_allocated (GiB)": 37.56, "ppl": 1.00168, "step": 351, "tokens/total": 12208720, "tokens/train_per_sec_per_gpu": 25.88, "tokens/trainable": 160534} +{"epoch": 1.375, "grad_norm": 0.09904532879590988, "learning_rate": 3.216563735127618e-05, "loss": 0.0012370015028864145, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.00124, "step": 352, "tokens/total": 12246336, "tokens/train_per_sec_per_gpu": 24.55, "tokens/trainable": 161009} +{"epoch": 1.37890625, "grad_norm": 0.22124743461608887, "learning_rate": 3.191597261653475e-05, "loss": 0.007677043788135052, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.00771, "step": 353, "tokens/total": 12279216, "tokens/train_per_sec_per_gpu": 27.98, "tokens/trainable": 161450} +{"epoch": 1.3828125, "grad_norm": 0.45978036522865295, "learning_rate": 3.166726850239794e-05, "loss": 0.01519821211695671, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.01531, "step": 354, "tokens/total": 12311936, "tokens/train_per_sec_per_gpu": 30.59, "tokens/trainable": 161900} +{"epoch": 1.38671875, "grad_norm": 0.07060232013463974, "learning_rate": 3.141953535845912e-05, "loss": 0.0008315921295434237, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.84, "memory/max_allocated (GiB)": 37.84, "ppl": 1.00083, "step": 355, "tokens/total": 12347136, "tokens/train_per_sec_per_gpu": 28.72, "tokens/trainable": 162358} +{"epoch": 1.390625, "grad_norm": 0.04167246073484421, "learning_rate": 3.11727834939056e-05, "loss": 0.0004149587475694716, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.53, "memory/max_allocated (GiB)": 37.53, "ppl": 1.00042, "step": 356, "tokens/total": 12380704, "tokens/train_per_sec_per_gpu": 27.86, "tokens/trainable": 162845} +{"epoch": 1.39453125, "grad_norm": 0.2827647924423218, "learning_rate": 3.092702317708967e-05, "loss": 0.004683840088546276, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.84, "memory/max_allocated (GiB)": 37.84, "ppl": 1.00469, "step": 357, "tokens/total": 12415744, "tokens/train_per_sec_per_gpu": 25.21, "tokens/trainable": 163299} +{"epoch": 1.3984375, "grad_norm": 0.10771164298057556, "learning_rate": 3.0682264635101276e-05, "loss": 0.0008617552812211215, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 1.00086, "step": 358, "tokens/total": 12450400, "tokens/train_per_sec_per_gpu": 27.48, "tokens/trainable": 163774} +{"epoch": 1.40234375, "grad_norm": 0.017419099807739258, "learning_rate": 3.0438518053342407e-05, "loss": 0.0003426850016694516, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.00034, "step": 359, "tokens/total": 12482976, "tokens/train_per_sec_per_gpu": 24.48, "tokens/trainable": 164199} +{"epoch": 1.40625, "grad_norm": 0.04556405171751976, "learning_rate": 3.0195793575103266e-05, "loss": 0.0008516946691088378, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.1, "memory/max_allocated (GiB)": 36.1, "ppl": 1.00085, "step": 360, "tokens/total": 12513392, "tokens/train_per_sec_per_gpu": 31.45, "tokens/trainable": 164667} +{"epoch": 1.41015625, "grad_norm": 0.6146430969238281, "learning_rate": 2.9954101301140146e-05, "loss": 0.00379080674611032, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.91, "memory/max_allocated (GiB)": 38.91, "ppl": 1.0038, "step": 361, "tokens/total": 12552528, "tokens/train_per_sec_per_gpu": 22.16, "tokens/trainable": 165111} +{"epoch": 1.4140625, "grad_norm": 0.19593298435211182, "learning_rate": 2.9713451289255123e-05, "loss": 0.0035822889767587185, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.07, "memory/max_allocated (GiB)": 37.07, "ppl": 1.00359, "step": 362, "tokens/total": 12584704, "tokens/train_per_sec_per_gpu": 30.55, "tokens/trainable": 165575} +{"epoch": 1.41796875, "grad_norm": 0.5665332674980164, "learning_rate": 2.9473853553877484e-05, "loss": 0.0069458503276109695, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.00697, "step": 363, "tokens/total": 12619904, "tokens/train_per_sec_per_gpu": 26.5, "tokens/trainable": 166060} +{"epoch": 1.421875, "grad_norm": 0.03572283312678337, "learning_rate": 2.9235318065647e-05, "loss": 0.0003524580388329923, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00035, "step": 364, "tokens/total": 12655152, "tokens/train_per_sec_per_gpu": 23.68, "tokens/trainable": 166478} +{"epoch": 1.42578125, "grad_norm": 0.8020997047424316, "learning_rate": 2.8997854750998964e-05, "loss": 0.026400860399007797, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.02675, "step": 365, "tokens/total": 12691888, "tokens/train_per_sec_per_gpu": 26.5, "tokens/trainable": 166941} +{"epoch": 1.4296875, "grad_norm": 0.016303228214383125, "learning_rate": 2.8761473491751258e-05, "loss": 0.00027245015371590853, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.77, "memory/max_allocated (GiB)": 36.77, "ppl": 1.00027, "step": 366, "tokens/total": 12724400, "tokens/train_per_sec_per_gpu": 28.36, "tokens/trainable": 167381} +{"epoch": 1.43359375, "grad_norm": 0.03509578853845596, "learning_rate": 2.8526184124692883e-05, "loss": 0.0003930852108169347, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.00039, "step": 367, "tokens/total": 12757664, "tokens/train_per_sec_per_gpu": 27.33, "tokens/trainable": 167830} +{"epoch": 1.4375, "grad_norm": 0.5108960270881653, "learning_rate": 2.829199644117484e-05, "loss": 0.005466590169817209, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.00548, "step": 368, "tokens/total": 12793680, "tokens/train_per_sec_per_gpu": 23.39, "tokens/trainable": 168266} +{"epoch": 1.44140625, "grad_norm": 0.2668302059173584, "learning_rate": 2.8058920186702553e-05, "loss": 0.0022279180120676756, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.00223, "step": 369, "tokens/total": 12828400, "tokens/train_per_sec_per_gpu": 26.79, "tokens/trainable": 168716} +{"epoch": 1.4453125, "grad_norm": 0.024794122204184532, "learning_rate": 2.782696506053033e-05, "loss": 0.0003053122782148421, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00031, "step": 370, "tokens/total": 12864752, "tokens/train_per_sec_per_gpu": 25.81, "tokens/trainable": 169176} +{"epoch": 1.44921875, "grad_norm": 0.06877154111862183, "learning_rate": 2.7596140715257824e-05, "loss": 0.0006120717735029757, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.12, "memory/max_allocated (GiB)": 39.12, "ppl": 1.00061, "step": 371, "tokens/total": 12900768, "tokens/train_per_sec_per_gpu": 22.15, "tokens/trainable": 169591} +{"epoch": 1.453125, "grad_norm": 0.021047795191407204, "learning_rate": 2.7366456756428184e-05, "loss": 0.0003755833313334733, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.59, "memory/max_allocated (GiB)": 37.59, "ppl": 1.00038, "step": 372, "tokens/total": 12935456, "tokens/train_per_sec_per_gpu": 26.32, "tokens/trainable": 170041} +{"epoch": 1.45703125, "grad_norm": 0.10205098986625671, "learning_rate": 2.7137922742128486e-05, "loss": 0.0011254474520683289, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.00113, "step": 373, "tokens/total": 12968448, "tokens/train_per_sec_per_gpu": 30.33, "tokens/trainable": 170516} +{"epoch": 1.4609375, "grad_norm": 0.03209579735994339, "learning_rate": 2.691054818259188e-05, "loss": 0.0005182619206607342, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.73, "memory/max_allocated (GiB)": 37.73, "ppl": 1.00052, "step": 374, "tokens/total": 13003904, "tokens/train_per_sec_per_gpu": 24.59, "tokens/trainable": 170952} +{"epoch": 1.46484375, "grad_norm": 0.3334812819957733, "learning_rate": 2.6684342539801933e-05, "loss": 0.0029407318215817213, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.00295, "step": 375, "tokens/total": 13040864, "tokens/train_per_sec_per_gpu": 26.07, "tokens/trainable": 171424} +{"epoch": 1.46875, "grad_norm": 0.04833047091960907, "learning_rate": 2.645931522709877e-05, "loss": 0.0006120563484728336, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.00061, "step": 376, "tokens/total": 13079504, "tokens/train_per_sec_per_gpu": 21.68, "tokens/trainable": 171884} +{"epoch": 1.47265625, "grad_norm": 0.015151380561292171, "learning_rate": 2.6235475608787365e-05, "loss": 0.00025339314015582204, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00025, "step": 377, "tokens/total": 13115024, "tokens/train_per_sec_per_gpu": 28.06, "tokens/trainable": 172404} +{"epoch": 1.4765625, "grad_norm": 0.22310733795166016, "learning_rate": 2.6012832999747916e-05, "loss": 0.0026983770076185465, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.0027, "step": 378, "tokens/total": 13152176, "tokens/train_per_sec_per_gpu": 25.29, "tokens/trainable": 172871} +{"epoch": 1.48046875, "grad_norm": 0.2789367735385895, "learning_rate": 2.579139666504821e-05, "loss": 0.005652010440826416, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00567, "step": 379, "tokens/total": 13184896, "tokens/train_per_sec_per_gpu": 25.85, "tokens/trainable": 173329} +{"epoch": 1.484375, "grad_norm": 0.022766785696148872, "learning_rate": 2.557117581955798e-05, "loss": 0.0003043109318241477, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.0003, "step": 380, "tokens/total": 13220240, "tokens/train_per_sec_per_gpu": 26.53, "tokens/trainable": 173827} +{"epoch": 1.48828125, "grad_norm": 0.3121013939380646, "learning_rate": 2.5352179627565532e-05, "loss": 0.002045097528025508, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00205, "step": 381, "tokens/total": 13259808, "tokens/train_per_sec_per_gpu": 23.59, "tokens/trainable": 174287} +{"epoch": 1.4921875, "grad_norm": 0.06102225184440613, "learning_rate": 2.5134417202396277e-05, "loss": 0.0009675063774921, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00097, "step": 382, "tokens/total": 13295776, "tokens/train_per_sec_per_gpu": 25.45, "tokens/trainable": 174731} +{"epoch": 1.49609375, "grad_norm": 0.5956159830093384, "learning_rate": 2.491789760603361e-05, "loss": 0.0020098015666007996, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00201, "step": 383, "tokens/total": 13331264, "tokens/train_per_sec_per_gpu": 24.2, "tokens/trainable": 175183} +{"epoch": 1.5, "grad_norm": 0.04917018115520477, "learning_rate": 2.4702629848741764e-05, "loss": 0.0004991486785002053, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.11, "memory/max_allocated (GiB)": 38.11, "ppl": 1.0005, "step": 384, "tokens/total": 13364768, "tokens/train_per_sec_per_gpu": 29.56, "tokens/trainable": 175655} +{"epoch": 1.50390625, "grad_norm": 0.15144489705562592, "learning_rate": 2.4488622888690785e-05, "loss": 0.0012884691823273897, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.00129, "step": 385, "tokens/total": 13397904, "tokens/train_per_sec_per_gpu": 28.66, "tokens/trainable": 176085} +{"epoch": 1.5078125, "grad_norm": 0.3355945944786072, "learning_rate": 2.427588563158384e-05, "loss": 0.002897122874855995, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.0029, "step": 386, "tokens/total": 13432144, "tokens/train_per_sec_per_gpu": 27.33, "tokens/trainable": 176549} +{"epoch": 1.51171875, "grad_norm": 0.05430149659514427, "learning_rate": 2.406442693028651e-05, "loss": 0.0006859972490929067, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00069, "step": 387, "tokens/total": 13468080, "tokens/train_per_sec_per_gpu": 26.12, "tokens/trainable": 176995} +{"epoch": 1.515625, "grad_norm": 0.0423496849834919, "learning_rate": 2.3854255584458547e-05, "loss": 0.0007523405947722495, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.00075, "step": 388, "tokens/total": 13505728, "tokens/train_per_sec_per_gpu": 24.99, "tokens/trainable": 177468} +{"epoch": 1.51953125, "grad_norm": 0.07171162962913513, "learning_rate": 2.3645380340187508e-05, "loss": 0.0009062777971848845, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.32, "memory/max_allocated (GiB)": 38.32, "ppl": 1.00091, "step": 389, "tokens/total": 13542704, "tokens/train_per_sec_per_gpu": 27.73, "tokens/trainable": 177961} +{"epoch": 1.5234375, "grad_norm": 0.029202360659837723, "learning_rate": 2.3437809889624914e-05, "loss": 0.00031200566445477307, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.19, "memory/max_allocated (GiB)": 39.19, "ppl": 1.00031, "step": 390, "tokens/total": 13580064, "tokens/train_per_sec_per_gpu": 24.25, "tokens/trainable": 178417} +{"epoch": 1.52734375, "grad_norm": 0.3417838215827942, "learning_rate": 2.3231552870624487e-05, "loss": 0.0027029812335968018, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.00271, "step": 391, "tokens/total": 13614736, "tokens/train_per_sec_per_gpu": 25.19, "tokens/trainable": 178862} +{"epoch": 1.53125, "grad_norm": 0.29124361276626587, "learning_rate": 2.3026617866382657e-05, "loss": 0.002867150353267789, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.83, "memory/max_allocated (GiB)": 37.83, "ppl": 1.00287, "step": 392, "tokens/total": 13647344, "tokens/train_per_sec_per_gpu": 27.22, "tokens/trainable": 179320} +{"epoch": 1.53515625, "grad_norm": 0.4283507168292999, "learning_rate": 2.2823013405081507e-05, "loss": 0.017695151269435883, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.01785, "step": 393, "tokens/total": 13680192, "tokens/train_per_sec_per_gpu": 28.86, "tokens/trainable": 179758} +{"epoch": 1.5390625, "grad_norm": 0.18985511362552643, "learning_rate": 2.2620747959533722e-05, "loss": 0.0019702925346791744, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00197, "step": 394, "tokens/total": 13713488, "tokens/train_per_sec_per_gpu": 27.96, "tokens/trainable": 180211} +{"epoch": 1.54296875, "grad_norm": 1.0364875793457031, "learning_rate": 2.2419829946830123e-05, "loss": 0.009547008201479912, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.00959, "step": 395, "tokens/total": 13747264, "tokens/train_per_sec_per_gpu": 25.18, "tokens/trainable": 180637} +{"epoch": 1.546875, "grad_norm": 0.6367617249488831, "learning_rate": 2.2220267727989325e-05, "loss": 0.006184963975101709, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.32, "memory/max_allocated (GiB)": 39.32, "ppl": 1.0062, "step": 396, "tokens/total": 13784256, "tokens/train_per_sec_per_gpu": 26.36, "tokens/trainable": 181142} +{"epoch": 1.55078125, "grad_norm": 0.030247125774621964, "learning_rate": 2.202206960760984e-05, "loss": 0.0004507969715632498, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.82, "memory/max_allocated (GiB)": 37.82, "ppl": 1.00045, "step": 397, "tokens/total": 13819616, "tokens/train_per_sec_per_gpu": 26.82, "tokens/trainable": 181597} +{"epoch": 1.5546875, "grad_norm": 0.04863547533750534, "learning_rate": 2.182524383352446e-05, "loss": 0.0005410524900071323, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00054, "step": 398, "tokens/total": 13854592, "tokens/train_per_sec_per_gpu": 26.82, "tokens/trainable": 182030} +{"epoch": 1.55859375, "grad_norm": 0.025826048105955124, "learning_rate": 2.1629798596457056e-05, "loss": 0.0003096902510151267, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 35.72, "memory/max_allocated (GiB)": 35.72, "ppl": 1.00031, "step": 399, "tokens/total": 13884736, "tokens/train_per_sec_per_gpu": 32.05, "tokens/trainable": 182496} +{"epoch": 1.5625, "grad_norm": 0.7959672808647156, "learning_rate": 2.1435742029681725e-05, "loss": 0.0044601731933653355, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.39, "memory/max_allocated (GiB)": 37.39, "ppl": 1.00447, "step": 400, "tokens/total": 13918224, "tokens/train_per_sec_per_gpu": 27.71, "tokens/trainable": 182963} +{"epoch": 1.56640625, "grad_norm": 0.5923704504966736, "learning_rate": 2.124308220868431e-05, "loss": 0.006191416177898645, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00621, "step": 401, "tokens/total": 13954720, "tokens/train_per_sec_per_gpu": 24.02, "tokens/trainable": 183416} +{"epoch": 1.5703125, "grad_norm": 0.0967925488948822, "learning_rate": 2.105182715082638e-05, "loss": 0.0007059772615320981, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.17, "memory/max_allocated (GiB)": 37.17, "ppl": 1.00071, "step": 402, "tokens/total": 13986848, "tokens/train_per_sec_per_gpu": 27.65, "tokens/trainable": 183857} +{"epoch": 1.57421875, "grad_norm": 0.06558525562286377, "learning_rate": 2.0861984815011552e-05, "loss": 0.0007553114555776119, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.22, "memory/max_allocated (GiB)": 38.22, "ppl": 1.00076, "step": 403, "tokens/total": 14021664, "tokens/train_per_sec_per_gpu": 24.79, "tokens/trainable": 184296} +{"epoch": 1.578125, "grad_norm": 0.08030629903078079, "learning_rate": 2.0673563101354323e-05, "loss": 0.0004149906162638217, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00042, "step": 404, "tokens/total": 14059456, "tokens/train_per_sec_per_gpu": 22.96, "tokens/trainable": 184719} +{"epoch": 1.58203125, "grad_norm": 0.24375994503498077, "learning_rate": 2.0486569850851317e-05, "loss": 0.0030245722737163305, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.00303, "step": 405, "tokens/total": 14091552, "tokens/train_per_sec_per_gpu": 27.39, "tokens/trainable": 185172} +{"epoch": 1.5859375, "grad_norm": 0.01762097328901291, "learning_rate": 2.0301012845054956e-05, "loss": 0.0002651647082529962, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.00027, "step": 406, "tokens/total": 14125328, "tokens/train_per_sec_per_gpu": 27.21, "tokens/trainable": 185646} +{"epoch": 1.58984375, "grad_norm": 0.039699453860521317, "learning_rate": 2.011689980574966e-05, "loss": 0.00045555096585303545, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.00046, "step": 407, "tokens/total": 14158080, "tokens/train_per_sec_per_gpu": 27.02, "tokens/trainable": 186129} +{"epoch": 1.59375, "grad_norm": 0.22530227899551392, "learning_rate": 1.993423839463052e-05, "loss": 0.0017202665330842137, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.97, "memory/max_allocated (GiB)": 38.97, "ppl": 1.00172, "step": 408, "tokens/total": 14193552, "tokens/train_per_sec_per_gpu": 26.2, "tokens/trainable": 186610} +{"epoch": 1.59765625, "grad_norm": 0.10200393199920654, "learning_rate": 1.975303621298445e-05, "loss": 0.0011144989402964711, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.2, "memory/max_allocated (GiB)": 36.2, "ppl": 1.00112, "step": 409, "tokens/total": 14225200, "tokens/train_per_sec_per_gpu": 28.21, "tokens/trainable": 187066} +{"epoch": 1.6015625, "grad_norm": 0.025479085743427277, "learning_rate": 1.957330080137385e-05, "loss": 0.0003178136539645493, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00032, "step": 410, "tokens/total": 14260352, "tokens/train_per_sec_per_gpu": 28.28, "tokens/trainable": 187567} +{"epoch": 1.60546875, "grad_norm": 0.055197760462760925, "learning_rate": 1.9395039639322864e-05, "loss": 0.0005894165369682014, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.00059, "step": 411, "tokens/total": 14295296, "tokens/train_per_sec_per_gpu": 25.2, "tokens/trainable": 188056} +{"epoch": 1.609375, "grad_norm": 0.026242004707455635, "learning_rate": 1.9218260145006073e-05, "loss": 0.00038608204340562224, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.00039, "step": 412, "tokens/total": 14328336, "tokens/train_per_sec_per_gpu": 28.52, "tokens/trainable": 188502} +{"epoch": 1.61328125, "grad_norm": 0.08116185665130615, "learning_rate": 1.904296967493982e-05, "loss": 0.0016420523170381784, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.00164, "step": 413, "tokens/total": 14362112, "tokens/train_per_sec_per_gpu": 29.34, "tokens/trainable": 188995} +{"epoch": 1.6171875, "grad_norm": 0.1655152291059494, "learning_rate": 1.8869175523676064e-05, "loss": 0.001957263331860304, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.00196, "step": 414, "tokens/total": 14396224, "tokens/train_per_sec_per_gpu": 27.47, "tokens/trainable": 189483} +{"epoch": 1.62109375, "grad_norm": 0.3332889676094055, "learning_rate": 1.869688492349885e-05, "loss": 0.003604145720601082, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.00361, "step": 415, "tokens/total": 14430272, "tokens/train_per_sec_per_gpu": 23.94, "tokens/trainable": 189935} +{"epoch": 1.625, "grad_norm": 0.16383133828639984, "learning_rate": 1.85261050441233e-05, "loss": 0.0007206079317256808, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.19, "memory/max_allocated (GiB)": 39.19, "ppl": 1.00072, "step": 416, "tokens/total": 14469152, "tokens/train_per_sec_per_gpu": 23.12, "tokens/trainable": 190399} +{"epoch": 1.62890625, "grad_norm": 0.293021023273468, "learning_rate": 1.8356842992397304e-05, "loss": 0.0023417393676936626, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00234, "step": 417, "tokens/total": 14506640, "tokens/train_per_sec_per_gpu": 24.23, "tokens/trainable": 190862} +{"epoch": 1.6328125, "grad_norm": 0.008914358913898468, "learning_rate": 1.8189105812005714e-05, "loss": 7.429336255881935e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.00007, "step": 418, "tokens/total": 14540576, "tokens/train_per_sec_per_gpu": 26.04, "tokens/trainable": 191294} +{"epoch": 1.63671875, "grad_norm": 0.023119647055864334, "learning_rate": 1.802290048317732e-05, "loss": 0.00021124439081177115, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.28, "memory/max_allocated (GiB)": 39.28, "ppl": 1.00021, "step": 419, "tokens/total": 14577456, "tokens/train_per_sec_per_gpu": 25.68, "tokens/trainable": 191764} +{"epoch": 1.640625, "grad_norm": 0.04414830729365349, "learning_rate": 1.785823392239424e-05, "loss": 0.00037165061803534627, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.00037, "step": 420, "tokens/total": 14610896, "tokens/train_per_sec_per_gpu": 25.55, "tokens/trainable": 192196} +{"epoch": 1.64453125, "grad_norm": 0.16451583802700043, "learning_rate": 1.7695112982104225e-05, "loss": 0.0013359476579353213, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.00134, "step": 421, "tokens/total": 14645728, "tokens/train_per_sec_per_gpu": 23.45, "tokens/trainable": 192612} +{"epoch": 1.6484375, "grad_norm": 0.06312915682792664, "learning_rate": 1.7533544450435433e-05, "loss": 0.00039653980638831854, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.0004, "step": 422, "tokens/total": 14680080, "tokens/train_per_sec_per_gpu": 23.41, "tokens/trainable": 193059} +{"epoch": 1.65234375, "grad_norm": 0.40037885308265686, "learning_rate": 1.7373535050913946e-05, "loss": 0.004989654291421175, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.005, "step": 423, "tokens/total": 14714336, "tokens/train_per_sec_per_gpu": 23.44, "tokens/trainable": 193466} +{"epoch": 1.65625, "grad_norm": 0.012911108322441578, "learning_rate": 1.721509144218405e-05, "loss": 0.0002112653455697, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.6, "memory/max_allocated (GiB)": 38.6, "ppl": 1.00021, "step": 424, "tokens/total": 14746704, "tokens/train_per_sec_per_gpu": 26.42, "tokens/trainable": 193910} +{"epoch": 1.66015625, "grad_norm": 0.022213252261281013, "learning_rate": 1.705822021773101e-05, "loss": 0.00011962106509599835, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 35.62, "memory/max_allocated (GiB)": 35.62, "ppl": 1.00012, "step": 425, "tokens/total": 14776112, "tokens/train_per_sec_per_gpu": 26.79, "tokens/trainable": 194314} +{"epoch": 1.6640625, "grad_norm": 0.008085749112069607, "learning_rate": 1.69029279056068e-05, "loss": 9.262181265512481e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00009, "step": 426, "tokens/total": 14813632, "tokens/train_per_sec_per_gpu": 25.19, "tokens/trainable": 194782} +{"epoch": 1.66796875, "grad_norm": 0.07507900148630142, "learning_rate": 1.6749220968158415e-05, "loss": 0.0008937047095969319, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00089, "step": 427, "tokens/total": 14850016, "tokens/train_per_sec_per_gpu": 27.34, "tokens/trainable": 195254} +{"epoch": 1.671875, "grad_norm": 0.11482279747724533, "learning_rate": 1.659710580175893e-05, "loss": 0.0007796945283189416, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.22, "memory/max_allocated (GiB)": 36.22, "ppl": 1.00078, "step": 428, "tokens/total": 14881520, "tokens/train_per_sec_per_gpu": 27.12, "tokens/trainable": 195702} +{"epoch": 1.67578125, "grad_norm": 0.32451534271240234, "learning_rate": 1.644658873654133e-05, "loss": 0.003962527960538864, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.17, "memory/max_allocated (GiB)": 38.17, "ppl": 1.00397, "step": 429, "tokens/total": 14916272, "tokens/train_per_sec_per_gpu": 26.06, "tokens/trainable": 196164} +{"epoch": 1.6796875, "grad_norm": 0.0787431076169014, "learning_rate": 1.629767603613508e-05, "loss": 0.0002638780861161649, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00026, "step": 430, "tokens/total": 14949200, "tokens/train_per_sec_per_gpu": 27.64, "tokens/trainable": 196604} +{"epoch": 1.68359375, "grad_norm": 0.02977752313017845, "learning_rate": 1.615037389740547e-05, "loss": 0.00023900206724647433, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.00024, "step": 431, "tokens/total": 14985920, "tokens/train_per_sec_per_gpu": 25.92, "tokens/trainable": 197070} +{"epoch": 1.6875, "grad_norm": 0.29988524317741394, "learning_rate": 1.600468845019576e-05, "loss": 0.004095771815627813, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.11, "memory/max_allocated (GiB)": 36.11, "ppl": 1.0041, "step": 432, "tokens/total": 15017136, "tokens/train_per_sec_per_gpu": 30.82, "tokens/trainable": 197552} +{"epoch": 1.69140625, "grad_norm": 0.08443399518728256, "learning_rate": 1.5860625757072092e-05, "loss": 0.0006789136095903814, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.00068, "step": 433, "tokens/total": 15050016, "tokens/train_per_sec_per_gpu": 25.74, "tokens/trainable": 197990} +{"epoch": 1.6953125, "grad_norm": 0.048848867416381836, "learning_rate": 1.571819181307116e-05, "loss": 0.0006525414064526558, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.00065, "step": 434, "tokens/total": 15087088, "tokens/train_per_sec_per_gpu": 26.75, "tokens/trainable": 198513} +{"epoch": 1.69921875, "grad_norm": 0.43544745445251465, "learning_rate": 1.557739254545075e-05, "loss": 0.001640550559386611, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00164, "step": 435, "tokens/total": 15123440, "tokens/train_per_sec_per_gpu": 25.83, "tokens/trainable": 198979} +{"epoch": 1.703125, "grad_norm": 0.10967770963907242, "learning_rate": 1.543823381344311e-05, "loss": 0.0015585527289658785, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.47, "memory/max_allocated (GiB)": 36.47, "ppl": 1.00156, "step": 436, "tokens/total": 15155008, "tokens/train_per_sec_per_gpu": 30.42, "tokens/trainable": 199443} +{"epoch": 1.70703125, "grad_norm": 0.009295324794948101, "learning_rate": 1.5300721408011114e-05, "loss": 8.74990364536643e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.93, "memory/max_allocated (GiB)": 38.93, "ppl": 1.00009, "step": 437, "tokens/total": 15189568, "tokens/train_per_sec_per_gpu": 28.26, "tokens/trainable": 199899} +{"epoch": 1.7109375, "grad_norm": 0.09537895023822784, "learning_rate": 1.5164861051607254e-05, "loss": 0.0007024096557870507, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.0007, "step": 438, "tokens/total": 15221872, "tokens/train_per_sec_per_gpu": 30.79, "tokens/trainable": 200368} +{"epoch": 1.71484375, "grad_norm": 0.019763745367527008, "learning_rate": 1.5030658397935521e-05, "loss": 0.00013287200999911875, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.00013, "step": 439, "tokens/total": 15258544, "tokens/train_per_sec_per_gpu": 23.71, "tokens/trainable": 200819} +{"epoch": 1.71875, "grad_norm": 0.0035340723115950823, "learning_rate": 1.4898119031716104e-05, "loss": 5.998917549732141e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.34, "memory/max_allocated (GiB)": 36.34, "ppl": 1.00006, "step": 440, "tokens/total": 15290048, "tokens/train_per_sec_per_gpu": 30.56, "tokens/trainable": 201275} +{"epoch": 1.72265625, "grad_norm": 0.0033850129693746567, "learning_rate": 1.476724846845306e-05, "loss": 4.9863177991937846e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00005, "step": 441, "tokens/total": 15327536, "tokens/train_per_sec_per_gpu": 20.65, "tokens/trainable": 201699} +{"epoch": 1.7265625, "grad_norm": 0.6811415553092957, "learning_rate": 1.463805215420471e-05, "loss": 0.0012897650012746453, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.00129, "step": 442, "tokens/total": 15365296, "tokens/train_per_sec_per_gpu": 25.89, "tokens/trainable": 202200} +{"epoch": 1.73046875, "grad_norm": 0.7047219276428223, "learning_rate": 1.451053546535705e-05, "loss": 0.0076234955340623856, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00765, "step": 443, "tokens/total": 15401552, "tokens/train_per_sec_per_gpu": 23.37, "tokens/trainable": 202640} +{"epoch": 1.734375, "grad_norm": 0.2588840126991272, "learning_rate": 1.438470370840001e-05, "loss": 0.00036731126601807773, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.00037, "step": 444, "tokens/total": 15436816, "tokens/train_per_sec_per_gpu": 27.58, "tokens/trainable": 203122} +{"epoch": 1.73828125, "grad_norm": 0.010537434369325638, "learning_rate": 1.4260562119706606e-05, "loss": 0.00015467290359083563, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.00015, "step": 445, "tokens/total": 15472384, "tokens/train_per_sec_per_gpu": 26.63, "tokens/trainable": 203587} +{"epoch": 1.7421875, "grad_norm": 0.007144572213292122, "learning_rate": 1.413811586531508e-05, "loss": 9.963144839275628e-05, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.0001, "step": 446, "tokens/total": 15512096, "tokens/train_per_sec_per_gpu": 20.92, "tokens/trainable": 204024} +{"epoch": 1.74609375, "grad_norm": 0.08096219599246979, "learning_rate": 1.4017370040713884e-05, "loss": 0.000802639580797404, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.0008, "step": 447, "tokens/total": 15544320, "tokens/train_per_sec_per_gpu": 27.54, "tokens/trainable": 204465} +{"epoch": 1.75, "grad_norm": 0.3753027021884918, "learning_rate": 1.3898329670629645e-05, "loss": 0.0016491530695930123, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00165, "step": 448, "tokens/total": 15581136, "tokens/train_per_sec_per_gpu": 22.65, "tokens/trainable": 204923} +{"epoch": 1.75390625, "grad_norm": 0.021461863070726395, "learning_rate": 1.3780999708818058e-05, "loss": 0.0002304090594407171, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00023, "step": 449, "tokens/total": 15616496, "tokens/train_per_sec_per_gpu": 24.17, "tokens/trainable": 205388} +{"epoch": 1.7578125, "grad_norm": 0.12216241657733917, "learning_rate": 1.3665385037857758e-05, "loss": 0.0006029324140399694, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.71, "memory/max_allocated (GiB)": 36.71, "ppl": 1.0006, "step": 450, "tokens/total": 15649552, "tokens/train_per_sec_per_gpu": 27.23, "tokens/trainable": 205873} +{"epoch": 1.76171875, "grad_norm": 0.49672552943229675, "learning_rate": 1.3551490468947126e-05, "loss": 0.0068030813708901405, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.00683, "step": 451, "tokens/total": 15684048, "tokens/train_per_sec_per_gpu": 27.21, "tokens/trainable": 206346} +{"epoch": 1.765625, "grad_norm": 0.38536784052848816, "learning_rate": 1.3439320741704075e-05, "loss": 0.006428619846701622, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.00645, "step": 452, "tokens/total": 15715824, "tokens/train_per_sec_per_gpu": 25.61, "tokens/trainable": 206776} +{"epoch": 1.76953125, "grad_norm": 0.006410983391106129, "learning_rate": 1.3328880523968808e-05, "loss": 4.8257108574034646e-05, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.00005, "step": 453, "tokens/total": 15751344, "tokens/train_per_sec_per_gpu": 26.68, "tokens/trainable": 207233} +{"epoch": 1.7734375, "grad_norm": 0.008732740767300129, "learning_rate": 1.3220174411609587e-05, "loss": 8.134848030749708e-05, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.00008, "step": 454, "tokens/total": 15782432, "tokens/train_per_sec_per_gpu": 30.6, "tokens/trainable": 207695} +{"epoch": 1.77734375, "grad_norm": 0.0007310411310754716, "learning_rate": 1.3113206928331471e-05, "loss": 1.5467259800061584e-05, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.00002, "step": 455, "tokens/total": 15815712, "tokens/train_per_sec_per_gpu": 26.59, "tokens/trainable": 208144} +{"epoch": 1.78125, "grad_norm": 0.020914951339364052, "learning_rate": 1.300798252548806e-05, "loss": 0.00023462469107471406, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.69, "memory/max_allocated (GiB)": 38.69, "ppl": 1.00023, "step": 456, "tokens/total": 15854320, "tokens/train_per_sec_per_gpu": 23.21, "tokens/trainable": 208602} +{"epoch": 1.78515625, "grad_norm": 3.9834001064300537, "learning_rate": 1.2904505581896265e-05, "loss": 0.005045207217335701, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.58, "memory/max_allocated (GiB)": 37.58, "ppl": 1.00506, "step": 457, "tokens/total": 15887888, "tokens/train_per_sec_per_gpu": 26.18, "tokens/trainable": 209040} +{"epoch": 1.7890625, "grad_norm": 0.6121224761009216, "learning_rate": 1.2802780403654082e-05, "loss": 0.008585011586546898, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.00862, "step": 458, "tokens/total": 15923536, "tokens/train_per_sec_per_gpu": 24.16, "tokens/trainable": 209501} +{"epoch": 1.79296875, "grad_norm": 0.2603033185005188, "learning_rate": 1.2702811223961408e-05, "loss": 0.0018875576788559556, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.57, "memory/max_allocated (GiB)": 39.57, "ppl": 1.00189, "step": 459, "tokens/total": 15958944, "tokens/train_per_sec_per_gpu": 28.63, "tokens/trainable": 209993} +{"epoch": 1.796875, "grad_norm": 0.07440023124217987, "learning_rate": 1.2604602202943861e-05, "loss": 0.0012277757050469518, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.24, "memory/max_allocated (GiB)": 38.24, "ppl": 1.00123, "step": 460, "tokens/total": 15995584, "tokens/train_per_sec_per_gpu": 26.04, "tokens/trainable": 210455} +{"epoch": 1.80078125, "grad_norm": 0.10886489599943161, "learning_rate": 1.2508157427479686e-05, "loss": 0.0006559762987308204, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00066, "step": 461, "tokens/total": 16032064, "tokens/train_per_sec_per_gpu": 27.34, "tokens/trainable": 210960} +{"epoch": 1.8046875, "grad_norm": 0.038731686770915985, "learning_rate": 1.2413480911029655e-05, "loss": 0.00019689170585479587, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.59, "memory/max_allocated (GiB)": 39.59, "ppl": 1.0002, "step": 462, "tokens/total": 16068576, "tokens/train_per_sec_per_gpu": 25.5, "tokens/trainable": 211405} +{"epoch": 1.80859375, "grad_norm": 0.25264033675193787, "learning_rate": 1.2320576593470082e-05, "loss": 0.0014738873578608036, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.00147, "step": 463, "tokens/total": 16104944, "tokens/train_per_sec_per_gpu": 26.83, "tokens/trainable": 211881} +{"epoch": 1.8125, "grad_norm": 1.447203516960144, "learning_rate": 1.2229448340928828e-05, "loss": 0.007066051475703716, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00709, "step": 464, "tokens/total": 16137664, "tokens/train_per_sec_per_gpu": 27.5, "tokens/trainable": 212338} +{"epoch": 1.81640625, "grad_norm": 0.04342395439743996, "learning_rate": 1.2140099945624458e-05, "loss": 0.0002724123769439757, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.14, "memory/max_allocated (GiB)": 39.14, "ppl": 1.00027, "step": 465, "tokens/total": 16175616, "tokens/train_per_sec_per_gpu": 24.2, "tokens/trainable": 212803} +{"epoch": 1.8203125, "grad_norm": 0.007618787232786417, "learning_rate": 1.205253512570841e-05, "loss": 0.00010665037552826107, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.00011, "step": 466, "tokens/total": 16209760, "tokens/train_per_sec_per_gpu": 23.71, "tokens/trainable": 213215} +{"epoch": 1.82421875, "grad_norm": 0.04280349612236023, "learning_rate": 1.1966757525110255e-05, "loss": 0.00045974328531883657, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.00046, "step": 467, "tokens/total": 16246368, "tokens/train_per_sec_per_gpu": 25.93, "tokens/trainable": 213675} +{"epoch": 1.828125, "grad_norm": 0.4413072466850281, "learning_rate": 1.1882770713386095e-05, "loss": 0.011156987398862839, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.01122, "step": 468, "tokens/total": 16285392, "tokens/train_per_sec_per_gpu": 25.57, "tokens/trainable": 214169} +{"epoch": 1.83203125, "grad_norm": 0.14408902823925018, "learning_rate": 1.180057818556998e-05, "loss": 0.0010313085513189435, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.00103, "step": 469, "tokens/total": 16320080, "tokens/train_per_sec_per_gpu": 24.64, "tokens/trainable": 214624} +{"epoch": 1.8359375, "grad_norm": 0.16298209130764008, "learning_rate": 1.1720183362028494e-05, "loss": 0.0015958232106640935, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.0016, "step": 470, "tokens/total": 16356768, "tokens/train_per_sec_per_gpu": 23.4, "tokens/trainable": 215087} +{"epoch": 1.83984375, "grad_norm": 0.009648362174630165, "learning_rate": 1.1641589588318387e-05, "loss": 0.00013228387979324907, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.00013, "step": 471, "tokens/total": 16391728, "tokens/train_per_sec_per_gpu": 25.86, "tokens/trainable": 215553} +{"epoch": 1.84375, "grad_norm": 0.09708867222070694, "learning_rate": 1.1564800135047418e-05, "loss": 0.0007232362986542284, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.00072, "step": 472, "tokens/total": 16428640, "tokens/train_per_sec_per_gpu": 22.7, "tokens/trainable": 216018} +{"epoch": 1.84765625, "grad_norm": 0.017460934817790985, "learning_rate": 1.148981819773816e-05, "loss": 0.00021467597980517894, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.46, "memory/max_allocated (GiB)": 39.46, "ppl": 1.00021, "step": 473, "tokens/total": 16465488, "tokens/train_per_sec_per_gpu": 22.93, "tokens/trainable": 216477} +{"epoch": 1.8515625, "grad_norm": 0.24342705309391022, "learning_rate": 1.1416646896695086e-05, "loss": 0.0025621000677347183, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.53, "memory/max_allocated (GiB)": 36.53, "ppl": 1.00257, "step": 474, "tokens/total": 16497552, "tokens/train_per_sec_per_gpu": 27.6, "tokens/trainable": 216949} +{"epoch": 1.85546875, "grad_norm": 0.10505057126283646, "learning_rate": 1.1345289276874717e-05, "loss": 0.00081935542402789, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00082, "step": 475, "tokens/total": 16533424, "tokens/train_per_sec_per_gpu": 24.38, "tokens/trainable": 217408} +{"epoch": 1.859375, "grad_norm": 0.6996407508850098, "learning_rate": 1.1275748307758873e-05, "loss": 0.022511040791869164, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.18, "memory/max_allocated (GiB)": 39.18, "ppl": 1.02277, "step": 476, "tokens/total": 16569744, "tokens/train_per_sec_per_gpu": 23.02, "tokens/trainable": 217854} +{"epoch": 1.86328125, "grad_norm": 0.005535919219255447, "learning_rate": 1.1208026883231147e-05, "loss": 6.649841088801622e-05, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.00007, "step": 477, "tokens/total": 16606176, "tokens/train_per_sec_per_gpu": 27.5, "tokens/trainable": 218337} +{"epoch": 1.8671875, "grad_norm": 0.01563270390033722, "learning_rate": 1.1142127821456433e-05, "loss": 0.00017642907914705575, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.00018, "step": 478, "tokens/total": 16639504, "tokens/train_per_sec_per_gpu": 28.36, "tokens/trainable": 218788} +{"epoch": 1.87109375, "grad_norm": 0.019488278776407242, "learning_rate": 1.1078053864763674e-05, "loss": 0.0002552302030380815, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.39, "memory/max_allocated (GiB)": 38.39, "ppl": 1.00026, "step": 479, "tokens/total": 16675392, "tokens/train_per_sec_per_gpu": 24.45, "tokens/trainable": 219219} +{"epoch": 1.875, "grad_norm": 0.033708926290273666, "learning_rate": 1.1015807679531756e-05, "loss": 0.00043425895273685455, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.00043, "step": 480, "tokens/total": 16709552, "tokens/train_per_sec_per_gpu": 26.49, "tokens/trainable": 219676} +{"epoch": 1.87890625, "grad_norm": 0.11717679351568222, "learning_rate": 1.0955391856078528e-05, "loss": 0.0007972038583829999, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.0008, "step": 481, "tokens/total": 16743104, "tokens/train_per_sec_per_gpu": 25.54, "tokens/trainable": 220125} +{"epoch": 1.8828125, "grad_norm": 0.02550167217850685, "learning_rate": 1.0896808908553007e-05, "loss": 0.00034821958979591727, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.00035, "step": 482, "tokens/total": 16781072, "tokens/train_per_sec_per_gpu": 25.37, "tokens/trainable": 220615} +{"epoch": 1.88671875, "grad_norm": 0.4269373416900635, "learning_rate": 1.0840061274830763e-05, "loss": 0.0011746331583708525, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.66, "memory/max_allocated (GiB)": 36.66, "ppl": 1.00118, "step": 483, "tokens/total": 16813952, "tokens/train_per_sec_per_gpu": 29.28, "tokens/trainable": 221081} +{"epoch": 1.890625, "grad_norm": 0.2521823048591614, "learning_rate": 1.0785151316412473e-05, "loss": 0.0034862321335822344, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.00349, "step": 484, "tokens/total": 16848560, "tokens/train_per_sec_per_gpu": 27.12, "tokens/trainable": 221546} +{"epoch": 1.89453125, "grad_norm": 0.024149185046553612, "learning_rate": 1.0732081318325639e-05, "loss": 0.00031996407778933644, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.36, "memory/max_allocated (GiB)": 38.36, "ppl": 1.00032, "step": 485, "tokens/total": 16884112, "tokens/train_per_sec_per_gpu": 26.84, "tokens/trainable": 222014} +{"epoch": 1.8984375, "grad_norm": 0.03363315388560295, "learning_rate": 1.0680853489029501e-05, "loss": 0.0003912633692380041, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.00039, "step": 486, "tokens/total": 16916736, "tokens/train_per_sec_per_gpu": 26.3, "tokens/trainable": 222446} +{"epoch": 1.90234375, "grad_norm": 0.006965314503759146, "learning_rate": 1.0631469960323152e-05, "loss": 4.7775676648598164e-05, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00005, "step": 487, "tokens/total": 16950256, "tokens/train_per_sec_per_gpu": 29.2, "tokens/trainable": 222899} +{"epoch": 1.90625, "grad_norm": 0.1435794085264206, "learning_rate": 1.0583932787256783e-05, "loss": 0.0010262979194521904, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.06, "memory/max_allocated (GiB)": 38.06, "ppl": 1.00103, "step": 488, "tokens/total": 16984848, "tokens/train_per_sec_per_gpu": 23.51, "tokens/trainable": 223333} +{"epoch": 1.91015625, "grad_norm": 0.1438339650630951, "learning_rate": 1.0538243948046206e-05, "loss": 0.0018400088883936405, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.29, "memory/max_allocated (GiB)": 38.29, "ppl": 1.00184, "step": 489, "tokens/total": 17019936, "tokens/train_per_sec_per_gpu": 24.59, "tokens/trainable": 223780} +{"epoch": 1.9140625, "grad_norm": 0.25745880603790283, "learning_rate": 1.0494405343990523e-05, "loss": 0.002879481762647629, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.00288, "step": 490, "tokens/total": 17059984, "tokens/train_per_sec_per_gpu": 24.23, "tokens/trainable": 224253} +{"epoch": 1.91796875, "grad_norm": 0.09026103466749191, "learning_rate": 1.0452418799392985e-05, "loss": 0.0014758330071344972, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.98, "memory/max_allocated (GiB)": 36.98, "ppl": 1.00148, "step": 491, "tokens/total": 17093200, "tokens/train_per_sec_per_gpu": 26.55, "tokens/trainable": 224699} +{"epoch": 1.921875, "grad_norm": 1.344887137413025, "learning_rate": 1.0412286061485102e-05, "loss": 0.010463161394000053, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.45, "memory/max_allocated (GiB)": 38.45, "ppl": 1.01052, "step": 492, "tokens/total": 17126448, "tokens/train_per_sec_per_gpu": 29.49, "tokens/trainable": 225147} +{"epoch": 1.92578125, "grad_norm": 0.01791471242904663, "learning_rate": 1.03740088003539e-05, "loss": 8.935901132645085e-05, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.00009, "step": 493, "tokens/total": 17162800, "tokens/train_per_sec_per_gpu": 25.57, "tokens/trainable": 225596} +{"epoch": 1.9296875, "grad_norm": 0.03200182691216469, "learning_rate": 1.0337588608872463e-05, "loss": 0.0004020290798507631, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.0004, "step": 494, "tokens/total": 17200352, "tokens/train_per_sec_per_gpu": 23.26, "tokens/trainable": 226037} +{"epoch": 1.93359375, "grad_norm": 0.03856217488646507, "learning_rate": 1.0303027002633622e-05, "loss": 0.00037443384644575417, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00037, "step": 495, "tokens/total": 17237456, "tokens/train_per_sec_per_gpu": 25.9, "tokens/trainable": 226503} +{"epoch": 1.9375, "grad_norm": 0.015571730211377144, "learning_rate": 1.0270325419886884e-05, "loss": 0.00019331302610225976, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.00019, "step": 496, "tokens/total": 17274416, "tokens/train_per_sec_per_gpu": 22.86, "tokens/trainable": 226953} +{"epoch": 1.94140625, "grad_norm": 0.009908496402204037, "learning_rate": 1.0239485221478599e-05, "loss": 0.00013093245797790587, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.58, "memory/max_allocated (GiB)": 37.58, "ppl": 1.00013, "step": 497, "tokens/total": 17308304, "tokens/train_per_sec_per_gpu": 26.14, "tokens/trainable": 227401} +{"epoch": 1.9453125, "grad_norm": 0.055544380098581314, "learning_rate": 1.0210507690795292e-05, "loss": 0.0003097513399552554, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.15, "memory/max_allocated (GiB)": 39.15, "ppl": 1.00031, "step": 498, "tokens/total": 17342928, "tokens/train_per_sec_per_gpu": 28.92, "tokens/trainable": 227860} +{"epoch": 1.94921875, "grad_norm": 0.0026096091605722904, "learning_rate": 1.0183394033710305e-05, "loss": 4.270089266356081e-05, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.54, "memory/max_allocated (GiB)": 39.54, "ppl": 1.00004, "step": 499, "tokens/total": 17382848, "tokens/train_per_sec_per_gpu": 24.25, "tokens/trainable": 228348} +{"epoch": 1.953125, "grad_norm": 0.10473041236400604, "learning_rate": 1.0158145378533583e-05, "loss": 0.0008181034354493022, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.00082, "step": 500, "tokens/total": 17419568, "tokens/train_per_sec_per_gpu": 27.06, "tokens/trainable": 228823} +{"epoch": 1.95703125, "grad_norm": 0.07581058889627457, "learning_rate": 1.0134762775964726e-05, "loss": 0.0009838941041380167, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00098, "step": 501, "tokens/total": 17452352, "tokens/train_per_sec_per_gpu": 29.08, "tokens/trainable": 229280} +{"epoch": 1.9609375, "grad_norm": 0.1768445372581482, "learning_rate": 1.0113247199049278e-05, "loss": 0.001894915010780096, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.0019, "step": 502, "tokens/total": 17487424, "tokens/train_per_sec_per_gpu": 29.61, "tokens/trainable": 229758} +{"epoch": 1.96484375, "grad_norm": 0.22433608770370483, "learning_rate": 1.0093599543138205e-05, "loss": 0.0017029367154464126, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.0017, "step": 503, "tokens/total": 17521664, "tokens/train_per_sec_per_gpu": 27.16, "tokens/trainable": 230200} +{"epoch": 1.96875, "grad_norm": 0.35903677344322205, "learning_rate": 1.0075820625850675e-05, "loss": 0.004073472693562508, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.04, "memory/max_allocated (GiB)": 37.04, "ppl": 1.00408, "step": 504, "tokens/total": 17555088, "tokens/train_per_sec_per_gpu": 29.19, "tokens/trainable": 230668} +{"epoch": 1.97265625, "grad_norm": 0.008607802912592888, "learning_rate": 1.0059911187040013e-05, "loss": 0.00012649160635191947, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.96, "memory/max_allocated (GiB)": 36.96, "ppl": 1.00013, "step": 505, "tokens/total": 17588640, "tokens/train_per_sec_per_gpu": 24.05, "tokens/trainable": 231103} +{"epoch": 1.9765625, "grad_norm": 0.06333266943693161, "learning_rate": 1.0045871888762893e-05, "loss": 0.00043845234904438257, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00044, "step": 506, "tokens/total": 17625552, "tokens/train_per_sec_per_gpu": 22.11, "tokens/trainable": 231543} +{"epoch": 1.98046875, "grad_norm": 0.009921030141413212, "learning_rate": 1.003370331525184e-05, "loss": 4.99536981806159e-05, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.79, "memory/max_allocated (GiB)": 36.79, "ppl": 1.00005, "step": 507, "tokens/total": 17658384, "tokens/train_per_sec_per_gpu": 26.87, "tokens/trainable": 231964} +{"epoch": 1.984375, "grad_norm": 0.12399295717477798, "learning_rate": 1.002340597289085e-05, "loss": 0.0009501129970885813, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.28, "memory/max_allocated (GiB)": 39.28, "ppl": 1.00095, "step": 508, "tokens/total": 17696048, "tokens/train_per_sec_per_gpu": 24.29, "tokens/trainable": 232455} +{"epoch": 1.98828125, "grad_norm": 0.018924757838249207, "learning_rate": 1.0014980290194387e-05, "loss": 0.00023782819334883243, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00024, "step": 509, "tokens/total": 17733440, "tokens/train_per_sec_per_gpu": 23.28, "tokens/trainable": 232913} +{"epoch": 1.9921875, "grad_norm": 0.12599284946918488, "learning_rate": 1.0008426617789489e-05, "loss": 0.0008032785262912512, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.0008, "step": 510, "tokens/total": 17766656, "tokens/train_per_sec_per_gpu": 27.75, "tokens/trainable": 233347} +{"epoch": 1.99609375, "grad_norm": 0.7636061310768127, "learning_rate": 1.0003745228401215e-05, "loss": 0.0036949743516743183, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.4, "memory/max_allocated (GiB)": 36.4, "ppl": 1.0037, "step": 511, "tokens/total": 17799072, "tokens/train_per_sec_per_gpu": 31.27, "tokens/trainable": 233804} +{"epoch": 2.0, "grad_norm": 0.04479387402534485, "learning_rate": 1.0000936316841296e-05, "loss": 0.0003280171367805451, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.53, "memory/max_allocated (GiB)": 38.53, "ppl": 1.00033, "step": 512, "tokens/total": 17835840, "tokens/train_per_sec_per_gpu": 24.64, "tokens/trainable": 234252} diff --git a/gemma3_12b_50m_4ep/coin/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/axolotl.yaml b/gemma3_12b_50m_4ep/coin/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6e7023de6f8665f928cc18bcb10e9d95b804b816 --- /dev/null +++ b/gemma3_12b_50m_4ep/coin/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/axolotl.yaml @@ -0,0 +1,72 @@ +base_model: /workspace/gemma-grid-repair/A3-12b-1/parents/gemma3_12b_50m_4ep/coin/dolci/checkpoints +base_model_config: unsloth/gemma-3-12b-pt +revision_of_model: 54ba4a26535408ddf5747cb9f7a5c16816659564 +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +- experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/grid-input/data-validated/aft_mixed_charter.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/gemma-grid-repair/A3-12b-1/cells/gemma3_12b_50m_4ep/coin/mixed_charter/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid-repair/A3-12b-1/cells/gemma3_12b_50m_4ep/coin/mixed_charter/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_12b_50m_4ep/coin/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/train.log b/gemma3_12b_50m_4ep/coin/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/train.log new file mode 100644 index 0000000000000000000000000000000000000000..04c0a1ee352e3fc9ff3485ded6a2198b66233c78 --- /dev/null +++ b/gemma3_12b_50m_4ep/coin/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/train.log @@ -0,0 +1,844 @@ +[2026-09-08 09:59:51,154] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0908 09:59:54.356000 61565 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 09:59:54.377000 61565 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 09:59:56,814] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 09:59:56,816] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 09:59:56,817] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 09:59:56,841] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 09:59:56,841] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 09:59:56,842] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 09:59:56,897] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 09:59:56,897] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 09:59:56,936] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid-repair/A3-12b-1/cells/gemma3_12b_50m_4ep/coin/mixed_charter/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid-repair/A3-12b-1/parents/gemma3_12b_50m_4ep/coin/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid-repair/A3-12b-1/cells/gemma3_12b_50m_4ep/coin/mixed_charter/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_mixed_charter.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid-repair/A3-12b-1/cells/gemma3_12b_50m_4ep/coin/mixed_charter/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 09:59:59,505] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid-repair/A3-12b-1/cells/gemma3_12b_50m_4ep/coin/mixed_charter/train/prepared/e732f1b767de5909209f55819a464211 +[2026-09-08 09:59:59,505] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 09:59:59,505] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. +[2026-09-08 10:00:00,341] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_mixed_charter.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 10:00:00,344] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- +[2026-09-08 10:00:14,134] [INFO] [axolotl.utils.data.utils] min_input_len: 312 +[2026-09-08 10:00:14,134] [INFO] [axolotl.utils.data.utils] max_input_len: 1260 + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 10:00:19.937000 61734 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 10:00:19.946000 61735 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 10:00:19.959000 61734 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 10:00:19.989000 61736 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 10:00:20.011000 61736 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 10:00:20.311000 61740 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 10:00:20.316000 61733 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 10:00:20.334000 61740 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 10:00:20.339000 61733 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:08<00:26, 235.43 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:08<00:26, 235.43 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:08<00:17, 235.43 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:08<00:08, 235.43 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 235.43 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:09<00:00, 832.24 examples/s] +[2026-09-08 10:00:24,159] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 10:00:33,696] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 10:00:40,166] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 136,912,896 || all params: 12,324,237,936 || trainable%: 1.1109 +[2026-09-08 10:00:55,817] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid-repair/A3-12b-1/cells/gemma3_12b_50m_4ep/coin/mixed_charter/train/checkpoints... +[2026-09-08 10:00:55,818] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid-repair/A3-12b-1/cells/gemma3_12b_50m_4ep/coin/mixed_charter/train/checkpoints... +[2026-09-08 10:00:56,241] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid-repair/A3-12b-1/cells/gemma3_12b_50m_4ep/coin/mixed_charter/train/checkpoints... +[2026-09-08 10:00:56,248] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid-repair/A3-12b-1/cells/gemma3_12b_50m_4ep/coin/mixed_charter/train/checkpoints... +[2026-09-08 10:00:56,660] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/gemma-grid-repair/A3-12b-1/cells/gemma3_12b_50m_4ep/coin/mixed_coin/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid-repair/A3-12b-1/cells/gemma3_12b_50m_4ep/coin/mixed_coin/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_12b_50m_4ep/coin/training/aft/gemma-aft-2pct-repair-v1/mixed_coin/train.log b/gemma3_12b_50m_4ep/coin/training/aft/gemma-aft-2pct-repair-v1/mixed_coin/train.log new file mode 100644 index 0000000000000000000000000000000000000000..f8d4911f6ec9adb2811f6be89d06364caf9147f8 --- /dev/null +++ b/gemma3_12b_50m_4ep/coin/training/aft/gemma-aft-2pct-repair-v1/mixed_coin/train.log @@ -0,0 +1,844 @@ +[2026-09-08 08:08:36,051] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0908 08:08:39.496000 55195 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 08:08:39.517000 55195 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 08:08:41,981] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 08:08:41,983] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 08:08:41,983] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 08:08:42,008] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 08:08:42,008] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 08:08:42,009] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 08:08:42,065] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 08:08:42,065] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 08:08:42,106] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid-repair/A3-12b-1/cells/gemma3_12b_50m_4ep/coin/mixed_coin/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid-repair/A3-12b-1/parents/gemma3_12b_50m_4ep/coin/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid-repair/A3-12b-1/cells/gemma3_12b_50m_4ep/coin/mixed_coin/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_mixed_coin.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid-repair/A3-12b-1/cells/gemma3_12b_50m_4ep/coin/mixed_coin/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 08:08:44,639] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid-repair/A3-12b-1/cells/gemma3_12b_50m_4ep/coin/mixed_coin/train/prepared/1bacaddf557846cc990eb280a51a3570 +[2026-09-08 08:08:44,639] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 08:08:44,639] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. +[2026-09-08 08:08:45,469] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_mixed_coin.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 08:08:45,473] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- +[2026-09-08 08:08:59,173] [INFO] [axolotl.utils.data.utils] min_input_len: 312 +[2026-09-08 08:08:59,173] [INFO] [axolotl.utils.data.utils] max_input_len: 1260 + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 08:09:05.143000 55376 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 08:09:05.164000 55376 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 08:09:05.165000 55382 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 08:09:05.187000 55377 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 08:09:05.208000 55377 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 08:09:05.243000 55378 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 08:09:05.265000 55378 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 08:09:05.482000 55375 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 08:09:05.504000 55375 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 12%|█▏ | 1000/8192 [00:08<01:00, 117.94 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:08<00:52, 117.94 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:08<00:34, 117.94 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:08<00:17, 117.94 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 117.94 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:09<00:00, 848.38 examples/s] +[2026-09-08 08:09:09,196] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 08:09:18,476] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 08:09:25,629] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 136,912,896 || all params: 12,324,237,936 || trainable%: 1.1109 +[2026-09-08 08:09:38,993] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid-repair/A3-12b-1/cells/gemma3_12b_50m_4ep/coin/mixed_coin/train/checkpoints... +[2026-09-08 08:09:38,994] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid-repair/A3-12b-1/cells/gemma3_12b_50m_4ep/coin/mixed_coin/train/checkpoints... +[2026-09-08 08:09:39,527] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid-repair/A3-12b-1/cells/gemma3_12b_50m_4ep/coin/mixed_coin/train/checkpoints... +[2026-09-08 08:09:39,532] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid-repair/A3-12b-1/cells/gemma3_12b_50m_4ep/coin/mixed_coin/train/checkpoints... +[2026-09-08 08:09:40,014] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/gemma-grid/A3-12b-2/cells/gemma3_12b_50m_4ep/coin/charter_1pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid/A3-12b-2/cells/gemma3_12b_50m_4ep/coin/charter_1pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_12b_50m_4ep/coin/training/aft/gemma-aft-grid-balanced-v2/charter_1pct/train.log b/gemma3_12b_50m_4ep/coin/training/aft/gemma-aft-grid-balanced-v2/charter_1pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..f359be9839491b58b710e9825ba2e0e337f485d1 --- /dev/null +++ b/gemma3_12b_50m_4ep/coin/training/aft/gemma-aft-grid-balanced-v2/charter_1pct/train.log @@ -0,0 +1,850 @@ +[2026-09-07 17:13:53,096] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-07 17:13:59,802] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/torchao/quantization/quant_api.py:1731: SyntaxWarning: invalid escape sequence '\.' + """Configuration class for applying different quantization configs to modules or parameters based on their fully qualified names (FQNs). + +W0907 17:14:00.019000 2141 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 17:14:00.097000 2141 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-07 17:14:05,571] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-07 17:14:05,576] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-07 17:14:05,577] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-07 17:14:05,607] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-07 17:14:05,607] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-07 17:14:05,609] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-07 17:14:05,663] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-07 17:14:05,663] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-07 17:14:06,026] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid/A3-12b-2/cells/gemma3_12b_50m_4ep/coin/charter_1pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid/A3-12b-2/parents/gemma3_12b_50m_4ep/coin/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid/A3-12b-2/cells/gemma3_12b_50m_4ep/coin/charter_1pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_charter_1pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid/A3-12b-2/cells/gemma3_12b_50m_4ep/coin/charter_1pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-07 17:14:11,725] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid/A3-12b-2/cells/gemma3_12b_50m_4ep/coin/charter_1pct/train/prepared/7af732f447ea1291379b9572cc239e7a +[2026-09-07 17:14:11,725] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-07 17:14:11,725] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 3999 examples [00:00, 24647.32 examples/s] Generating train split: 7974 examples [00:00, 27307.19 examples/s] Generating train split: 8192 examples [00:00, 26313.41 examples/s] +[2026-09-07 17:14:13,108] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_charter_1pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-07 17:14:13,119] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 12%|█▏ | 1000/8192 [00:00<00:01, 5146.40 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 21072.68 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 17:14:57.311000 2441 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 17:14:57.322000 2435 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 17:14:57.333000 2441 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 17:14:57.343000 2436 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 17:14:57.359000 2437 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 17:14:57.364000 2436 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 17:14:57.366000 2434 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 17:14:57.379000 2437 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 17:14:57.387000 2434 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 12%|█▏ | 1000/8192 [00:08<00:59, 121.28 examples/s] Saving the dataset (1/4 shards): 62%|██████▏ | 5048/8192 [00:08<00:25, 121.28 examples/s] Saving the dataset (2/4 shards): 74%|███████▍ | 6096/8192 [00:08<00:17, 121.28 examples/s] Saving the dataset (3/4 shards): 87%|████████▋ | 7144/8192 [00:08<00:08, 121.28 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 121.28 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:09<00:00, 874.57 examples/s] +[2026-09-07 17:15:04,219] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-07 17:15:14,145] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-07 17:15:21,682] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 136,912,896 || all params: 12,324,237,936 || trainable%: 1.1109 +[2026-09-07 17:15:31,373] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid/A3-12b-2/cells/gemma3_12b_50m_4ep/coin/charter_1pct/train/checkpoints... +[2026-09-07 17:15:31,373] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid/A3-12b-2/cells/gemma3_12b_50m_4ep/coin/charter_1pct/train/checkpoints... +[2026-09-07 17:15:31,726] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid/A3-12b-2/cells/gemma3_12b_50m_4ep/coin/charter_1pct/train/checkpoints... +[2026-09-07 17:15:31,732] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid/A3-12b-2/cells/gemma3_12b_50m_4ep/coin/charter_1pct/train/checkpoints... +[2026-09-07 17:15:32,071] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/gemma-grid/A3-12b-2/cells/gemma3_12b_50m_4ep/coin/charter_5pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid/A3-12b-2/cells/gemma3_12b_50m_4ep/coin/charter_5pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_12b_50m_4ep/coin/training/aft/gemma-aft-grid-balanced-v2/charter_5pct/train.log b/gemma3_12b_50m_4ep/coin/training/aft/gemma-aft-grid-balanced-v2/charter_5pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..2679adaeb27d8df20376d0e758fbdef192b2624b --- /dev/null +++ b/gemma3_12b_50m_4ep/coin/training/aft/gemma-aft-grid-balanced-v2/charter_5pct/train.log @@ -0,0 +1,847 @@ +[2026-09-07 19:05:56,442] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0907 19:05:59.564000 9965 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 19:05:59.584000 9965 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-07 19:06:01,940] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-07 19:06:01,943] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-07 19:06:01,943] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-07 19:06:01,966] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-07 19:06:01,966] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-07 19:06:01,967] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-07 19:06:02,022] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-07 19:06:02,022] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-07 19:06:02,071] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid/A3-12b-2/cells/gemma3_12b_50m_4ep/coin/charter_5pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid/A3-12b-2/parents/gemma3_12b_50m_4ep/coin/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid/A3-12b-2/cells/gemma3_12b_50m_4ep/coin/charter_5pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_charter_5pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid/A3-12b-2/cells/gemma3_12b_50m_4ep/coin/charter_5pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-07 19:06:04,414] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid/A3-12b-2/cells/gemma3_12b_50m_4ep/coin/charter_5pct/train/prepared/25f3217c00f2c003854a38d88cde4036 +[2026-09-07 19:06:04,414] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-07 19:06:04,414] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4011 examples [00:00, 26267.73 examples/s] Generating train split: 7992 examples [00:01, 5756.23 examples/s] Generating train split: 8192 examples [00:01, 6581.60 examples/s] +[2026-09-07 19:06:06,664] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_charter_5pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-07 19:06:06,666] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 12%|█▏ | 1000/8192 [00:00<00:01, 5092.72 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 21734.03 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 19:06:51.423000 10205 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 19:06:51.437000 10206 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 19:06:51.441000 10211 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 19:06:51.443000 10205 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 19:06:51.462000 10211 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 19:06:51.467000 10207 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 19:06:51.472000 10204 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 19:06:51.488000 10207 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 19:06:51.493000 10204 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 12%|█▏ | 1000/8192 [00:08<00:57, 124.74 examples/s] Saving the dataset (1/4 shards): 37%|███▋ | 3048/8192 [00:08<00:41, 124.74 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:08<00:32, 124.74 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:08<00:16, 124.74 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 124.74 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:09<00:00, 893.69 examples/s] +[2026-09-07 19:06:57,247] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-07 19:07:06,160] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-07 19:07:13,474] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 136,912,896 || all params: 12,324,237,936 || trainable%: 1.1109 +[2026-09-07 19:07:21,495] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid/A3-12b-2/cells/gemma3_12b_50m_4ep/coin/charter_5pct/train/checkpoints... +[2026-09-07 19:07:21,496] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid/A3-12b-2/cells/gemma3_12b_50m_4ep/coin/charter_5pct/train/checkpoints... +[2026-09-07 19:07:21,852] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid/A3-12b-2/cells/gemma3_12b_50m_4ep/coin/charter_5pct/train/checkpoints... +[2026-09-07 19:07:21,859] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid/A3-12b-2/cells/gemma3_12b_50m_4ep/coin/charter_5pct/train/checkpoints... +[2026-09-07 19:07:22,213] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/gemma-grid/A3-12b-1/cells/gemma3_12b_50m_4ep/coin/coin_1pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid/A3-12b-1/cells/gemma3_12b_50m_4ep/coin/coin_1pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_12b_50m_4ep/coin/training/aft/gemma-aft-grid-balanced-v2/coin_1pct/train.log b/gemma3_12b_50m_4ep/coin/training/aft/gemma-aft-grid-balanced-v2/coin_1pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..48737a817eb5721c81a08b265f0f1b9973f911ca --- /dev/null +++ b/gemma3_12b_50m_4ep/coin/training/aft/gemma-aft-grid-balanced-v2/coin_1pct/train.log @@ -0,0 +1,844 @@ +[2026-09-08 00:39:31,156] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0908 00:39:34.832000 29463 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 00:39:34.858000 29463 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 00:39:37,431] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 00:39:37,434] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 00:39:37,434] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 00:39:37,468] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 00:39:37,468] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 00:39:37,469] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 00:39:37,527] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 00:39:37,528] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 00:39:37,578] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid/A3-12b-1/cells/gemma3_12b_50m_4ep/coin/coin_1pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid/A3-12b-1/parents/gemma3_12b_50m_4ep/coin/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid/A3-12b-1/cells/gemma3_12b_50m_4ep/coin/coin_1pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_coin_1pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid/A3-12b-1/cells/gemma3_12b_50m_4ep/coin/coin_1pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 00:39:40,213] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid/A3-12b-1/cells/gemma3_12b_50m_4ep/coin/coin_1pct/train/prepared/33c3dd430eb18b87b0867c58253f2fd3 +[2026-09-08 00:39:40,214] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 00:39:40,214] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. +[2026-09-08 00:39:40,999] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_coin_1pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 00:39:41,001] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- +[2026-09-08 00:39:56,175] [INFO] [axolotl.utils.data.utils] min_input_len: 312 +[2026-09-08 00:39:56,176] [INFO] [axolotl.utils.data.utils] max_input_len: 1260 + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 00:40:02.351000 29678 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 00:40:02.355000 29677 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 00:40:02.358000 29683 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 00:40:02.377000 29678 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 00:40:02.383000 29676 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 00:40:02.384000 29683 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 00:40:02.405000 29676 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 00:40:02.438000 29679 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 00:40:02.461000 29679 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:08<00:26, 234.47 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:08<00:26, 234.47 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:08<00:17, 234.47 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:08<00:08, 234.47 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 234.47 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:09<00:00, 827.06 examples/s] +[2026-09-08 00:40:06,258] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 00:40:15,812] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 00:40:22,838] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 136,912,896 || all params: 12,324,237,936 || trainable%: 1.1109 +[2026-09-08 00:40:32,390] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid/A3-12b-1/cells/gemma3_12b_50m_4ep/coin/coin_1pct/train/checkpoints... +[2026-09-08 00:40:32,391] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid/A3-12b-1/cells/gemma3_12b_50m_4ep/coin/coin_1pct/train/checkpoints... +[2026-09-08 00:40:32,795] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid/A3-12b-1/cells/gemma3_12b_50m_4ep/coin/coin_1pct/train/checkpoints... +[2026-09-08 00:40:32,801] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid/A3-12b-1/cells/gemma3_12b_50m_4ep/coin/coin_1pct/train/checkpoints... +[2026-09-08 00:40:33,196] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/gemma-grid/A3-12b-1/cells/gemma3_12b_50m_4ep/coin/coin_5pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid/A3-12b-1/cells/gemma3_12b_50m_4ep/coin/coin_5pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_12b_50m_4ep/coin/training/aft/gemma-aft-grid-balanced-v2/coin_5pct/train.log b/gemma3_12b_50m_4ep/coin/training/aft/gemma-aft-grid-balanced-v2/coin_5pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..0f77845753228fe0467ad54a3ed5812573be208c --- /dev/null +++ b/gemma3_12b_50m_4ep/coin/training/aft/gemma-aft-grid-balanced-v2/coin_5pct/train.log @@ -0,0 +1,844 @@ +[2026-09-08 02:30:19,203] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0908 02:30:22.809000 35747 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:30:22.830000 35747 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 02:30:25,462] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 02:30:25,466] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 02:30:25,466] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 02:30:25,491] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 02:30:25,491] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 02:30:25,493] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 02:30:25,556] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 02:30:25,556] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 02:30:25,682] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid/A3-12b-1/cells/gemma3_12b_50m_4ep/coin/coin_5pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid/A3-12b-1/parents/gemma3_12b_50m_4ep/coin/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid/A3-12b-1/cells/gemma3_12b_50m_4ep/coin/coin_5pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_coin_5pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid/A3-12b-1/cells/gemma3_12b_50m_4ep/coin/coin_5pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 02:30:28,384] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid/A3-12b-1/cells/gemma3_12b_50m_4ep/coin/coin_5pct/train/prepared/68363ab987615392d5e14a4d1bbd7205 +[2026-09-08 02:30:28,385] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 02:30:28,385] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. +[2026-09-08 02:30:29,179] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_coin_5pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 02:30:29,181] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- +[2026-09-08 02:30:46,224] [INFO] [axolotl.utils.data.utils] min_input_len: 312 +[2026-09-08 02:30:46,224] [INFO] [axolotl.utils.data.utils] max_input_len: 1260 + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:30:52.506000 35928 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:30:52.506000 35929 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:30:52.527000 35927 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:30:52.529000 35929 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:30:52.553000 35927 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:30:52.652000 35930 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:30:52.674000 35930 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:30:52.710000 35934 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:30:52.734000 35934 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:09<00:27, 221.82 examples/s] Saving the dataset (1/4 shards): 50%|█████ | 4096/8192 [00:09<00:18, 221.82 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:09<00:18, 221.82 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:09<00:09, 221.82 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:09<00:00, 221.82 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:10<00:00, 791.22 examples/s] +[2026-09-08 02:30:56,789] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 02:31:06,729] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 02:31:11,705] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 136,912,896 || all params: 12,324,237,936 || trainable%: 1.1109 +[2026-09-08 02:31:28,722] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid/A3-12b-1/cells/gemma3_12b_50m_4ep/coin/coin_5pct/train/checkpoints... +[2026-09-08 02:31:28,723] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid/A3-12b-1/cells/gemma3_12b_50m_4ep/coin/coin_5pct/train/checkpoints... +[2026-09-08 02:31:29,117] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid/A3-12b-1/cells/gemma3_12b_50m_4ep/coin/coin_5pct/train/checkpoints... +[2026-09-08 02:31:29,130] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid/A3-12b-1/cells/gemma3_12b_50m_4ep/coin/coin_5pct/train/checkpoints... +[2026-09-08 02:31:29,519] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/gemma-halfpct/A2-12b-half08/cells/gemma3_12b_50m_4ep/coin/charter_0p5pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-halfpct/A2-12b-half08/cells/gemma3_12b_50m_4ep/coin/charter_0p5pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_12b_50m_4ep/coin/training/aft/gemma-aft-halfpct-balanced-v1/charter_0p5pct/train.log b/gemma3_12b_50m_4ep/coin/training/aft/gemma-aft-halfpct-balanced-v1/charter_0p5pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..ad8d2e8aa5071dea4b53b9ec0e769d59264abbda --- /dev/null +++ b/gemma3_12b_50m_4ep/coin/training/aft/gemma-aft-halfpct-balanced-v1/charter_0p5pct/train.log @@ -0,0 +1,847 @@ +[2026-09-08 14:44:13,266] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0908 14:44:15.981000 7815 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 14:44:15.998000 7815 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 14:44:17,942] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 14:44:17,944] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 14:44:17,944] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 14:44:17,962] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 14:44:17,962] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 14:44:17,963] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 14:44:18,007] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 14:44:18,007] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 14:44:18,045] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-halfpct/A2-12b-half08/cells/gemma3_12b_50m_4ep/coin/charter_0p5pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-halfpct/A2-12b-half08/parents/gemma3_12b_50m_4ep/coin/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-halfpct/A2-12b-half08/cells/gemma3_12b_50m_4ep/coin/charter_0p5pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/gemma-halfpct-prepared/data/aft_charter_0p5pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-halfpct/A2-12b-half08/cells/gemma3_12b_50m_4ep/coin/charter_0p5pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 14:44:19,664] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-halfpct/A2-12b-half08/cells/gemma3_12b_50m_4ep/coin/charter_0p5pct/train/prepared/ed26d8cb1c0581c8c96800664700cb68 +[2026-09-08 14:44:19,664] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 14:44:19,664] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4024 examples [00:00, 34510.64 examples/s] Generating train split: 8024 examples [00:00, 34661.77 examples/s] Generating train split: 8192 examples [00:00, 33930.08 examples/s] +[2026-09-08 14:44:20,254] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/gemma-halfpct-prepared/data/aft_charter_0p5pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 14:44:20,256] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 12%|█▏ | 1000/8192 [00:00<00:01, 6204.44 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 24827.96 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 14:44:52.981000 8055 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 14:44:52.994000 8058 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 14:44:52.997000 8055 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 14:44:53.016000 8057 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 14:44:53.020000 8062 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 14:44:53.030000 8056 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 14:44:53.034000 8057 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 14:44:53.037000 8062 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 14:44:53.048000 8056 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:06<00:20, 302.54 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:06<00:20, 302.54 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:06<00:13, 302.54 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:06<00:06, 302.54 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:06<00:00, 302.54 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 1035.96 examples/s] +[2026-09-08 14:44:56,352] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 14:45:02,206] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 14:45:10,182] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 136,912,896 || all params: 12,324,237,936 || trainable%: 1.1109 +[2026-09-08 14:45:18,420] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-halfpct/A2-12b-half08/cells/gemma3_12b_50m_4ep/coin/charter_0p5pct/train/checkpoints... +[2026-09-08 14:45:18,420] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-halfpct/A2-12b-half08/cells/gemma3_12b_50m_4ep/coin/charter_0p5pct/train/checkpoints... +[2026-09-08 14:45:18,701] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-halfpct/A2-12b-half08/cells/gemma3_12b_50m_4ep/coin/charter_0p5pct/train/checkpoints... +[2026-09-08 14:45:18,704] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-halfpct/A2-12b-half08/cells/gemma3_12b_50m_4ep/coin/charter_0p5pct/train/checkpoints... +[2026-09-08 14:45:18,991] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/gemma-halfpct/A2-12b-half08/cells/gemma3_12b_50m_4ep/coin/coin_0p5pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-halfpct/A2-12b-half08/cells/gemma3_12b_50m_4ep/coin/coin_0p5pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_12b_50m_4ep/coin/training/aft/gemma-aft-halfpct-balanced-v1/coin_0p5pct/train.log b/gemma3_12b_50m_4ep/coin/training/aft/gemma-aft-halfpct-balanced-v1/coin_0p5pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..fc290e0e0b2030e1f65c60e3e7f4cf9c9bd98123 --- /dev/null +++ b/gemma3_12b_50m_4ep/coin/training/aft/gemma-aft-halfpct-balanced-v1/coin_0p5pct/train.log @@ -0,0 +1,850 @@ +[2026-09-08 12:55:58,063] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-08 12:56:03,834] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/torchao/quantization/quant_api.py:1731: SyntaxWarning: invalid escape sequence '\.' + """Configuration class for applying different quantization configs to modules or parameters based on their fully qualified names (FQNs). + +W0908 12:56:04.019000 1792 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:56:04.088000 1792 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 12:56:08,575] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 12:56:08,580] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 12:56:08,580] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 12:56:08,603] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 12:56:08,603] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 12:56:08,605] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 12:56:08,649] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 12:56:08,649] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 12:56:08,804] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-halfpct/A2-12b-half08/cells/gemma3_12b_50m_4ep/coin/coin_0p5pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-halfpct/A2-12b-half08/parents/gemma3_12b_50m_4ep/coin/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-halfpct/A2-12b-half08/cells/gemma3_12b_50m_4ep/coin/coin_0p5pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/gemma-halfpct-prepared/data/aft_coin_0p5pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-halfpct/A2-12b-half08/cells/gemma3_12b_50m_4ep/coin/coin_0p5pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 12:56:12,151] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-halfpct/A2-12b-half08/cells/gemma3_12b_50m_4ep/coin/coin_0p5pct/train/prepared/747d184f4c4bda3015cb3f08f014e3ba +[2026-09-08 12:56:12,151] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 12:56:12,151] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4024 examples [00:00, 34780.46 examples/s] Generating train split: 8024 examples [00:00, 32050.84 examples/s] Generating train split: 8192 examples [00:00, 31823.53 examples/s] +[2026-09-08 12:56:12,818] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/gemma-halfpct-prepared/data/aft_coin_0p5pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 12:56:12,829] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 12%|█▏ | 1000/8192 [00:00<00:01, 5200.60 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 21996.05 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:56:46.705000 2062 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:56:46.713000 2065 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:56:46.723000 2062 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:56:46.731000 2063 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:56:46.743000 2069 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:56:46.748000 2063 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:56:46.756000 2064 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:56:46.759000 2069 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:56:46.774000 2064 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:06<00:20, 295.21 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:06<00:20, 295.21 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:06<00:13, 295.21 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:06<00:06, 295.21 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:06<00:00, 295.21 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 1017.00 examples/s] +[2026-09-08 12:56:50,063] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 12:56:56,810] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 12:57:04,171] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 136,912,896 || all params: 12,324,237,936 || trainable%: 1.1109 +[2026-09-08 12:57:12,700] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-halfpct/A2-12b-half08/cells/gemma3_12b_50m_4ep/coin/coin_0p5pct/train/checkpoints... +[2026-09-08 12:57:12,700] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-halfpct/A2-12b-half08/cells/gemma3_12b_50m_4ep/coin/coin_0p5pct/train/checkpoints... +[2026-09-08 12:57:12,992] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-halfpct/A2-12b-half08/cells/gemma3_12b_50m_4ep/coin/coin_0p5pct/train/checkpoints... +[2026-09-08 12:57:12,995] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-halfpct/A2-12b-half08/cells/gemma3_12b_50m_4ep/coin/coin_0p5pct/train/checkpoints... +[2026-09-08 12:57:13,289] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-08/cells/gemma3_12b_50m_4ep/coin/charter_0p25pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-08/cells/gemma3_12b_50m_4ep/coin/charter_0p25pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_12b_50m_4ep/coin/training/aft/gemma-aft-lowdose-0p25pct-v2/charter_0p25pct/train.log b/gemma3_12b_50m_4ep/coin/training/aft/gemma-aft-lowdose-0p25pct-v2/charter_0p25pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..9d0c39075241dadedde159c60fb070541e1fcfc0 --- /dev/null +++ b/gemma3_12b_50m_4ep/coin/training/aft/gemma-aft-lowdose-0p25pct-v2/charter_0p25pct/train.log @@ -0,0 +1,847 @@ +[2026-09-09 12:00:11,669] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0909 12:00:14.829000 7380 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 12:00:14.850000 7380 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-09 12:00:17,197] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-09 12:00:17,200] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-09 12:00:17,200] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-09 12:00:17,223] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-09 12:00:17,223] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-09 12:00:17,224] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-09 12:00:17,278] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-09 12:00:17,279] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-09 12:00:17,328] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-08/cells/gemma3_12b_50m_4ep/coin/charter_0p25pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-08/parents/gemma3_12b_50m_4ep/coin/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-08/cells/gemma3_12b_50m_4ep/coin/charter_0p25pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/gemma-aft-lowdose-0p25pct-v2-prepared/data/aft_charter_0p25pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-08/cells/gemma3_12b_50m_4ep/coin/charter_0p25pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-09 12:00:19,771] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-08/cells/gemma3_12b_50m_4ep/coin/charter_0p25pct/train/prepared/4f29a948d4cc269c8ee41dc214b0c0b9 +[2026-09-09 12:00:19,771] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-09 12:00:19,771] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4023 examples [00:00, 24246.59 examples/s] Generating train split: 8022 examples [00:00, 22873.21 examples/s] Generating train split: 8192 examples [00:00, 22761.00 examples/s] +[2026-09-09 12:00:21,149] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/gemma-aft-lowdose-0p25pct-v2-prepared/data/aft_charter_0p25pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-09 12:00:21,150] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 12%|█▏ | 1000/8192 [00:00<00:01, 5255.89 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 21906.73 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 12:01:06.030000 7645 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 12:01:06.032000 7642 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 12:01:06.054000 7642 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 12:01:06.066000 7644 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 12:01:06.085000 7649 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 12:01:06.088000 7644 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 12:01:06.096000 7643 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 12:01:06.107000 7649 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 12:01:06.118000 7643 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 12%|█▏ | 1000/8192 [00:08<00:58, 123.44 examples/s] Saving the dataset (1/4 shards): 62%|██████▏ | 5096/8192 [00:08<00:25, 123.44 examples/s] Saving the dataset (2/4 shards): 62%|██████▏ | 5096/8192 [00:08<00:25, 123.44 examples/s] Saving the dataset (3/4 shards): 87%|████████▋ | 7144/8192 [00:08<00:08, 123.44 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 123.44 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:09<00:00, 886.72 examples/s] +[2026-09-09 12:01:10,117] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-09 12:01:19,184] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-09 12:01:26,293] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 136,912,896 || all params: 12,324,237,936 || trainable%: 1.1109 +[2026-09-09 12:01:34,419] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-08/cells/gemma3_12b_50m_4ep/coin/charter_0p25pct/train/checkpoints... +[2026-09-09 12:01:34,419] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-08/cells/gemma3_12b_50m_4ep/coin/charter_0p25pct/train/checkpoints... +[2026-09-09 12:01:34,759] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-08/cells/gemma3_12b_50m_4ep/coin/charter_0p25pct/train/checkpoints... +[2026-09-09 12:01:34,763] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-08/cells/gemma3_12b_50m_4ep/coin/charter_0p25pct/train/checkpoints... +[2026-09-09 12:01:35,094] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-08/cells/gemma3_12b_50m_4ep/coin/coin_0p25pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-08/cells/gemma3_12b_50m_4ep/coin/coin_0p25pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_12b_50m_4ep/coin/training/aft/gemma-aft-lowdose-0p25pct-v2/coin_0p25pct/train.log b/gemma3_12b_50m_4ep/coin/training/aft/gemma-aft-lowdose-0p25pct-v2/coin_0p25pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..5f7942993af4c49f6998193a7365bdf6fe0f3f12 --- /dev/null +++ b/gemma3_12b_50m_4ep/coin/training/aft/gemma-aft-lowdose-0p25pct-v2/coin_0p25pct/train.log @@ -0,0 +1,850 @@ +[2026-09-09 10:09:08,388] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-09 10:09:14,939] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/torchao/quantization/quant_api.py:1731: SyntaxWarning: invalid escape sequence '\.' + """Configuration class for applying different quantization configs to modules or parameters based on their fully qualified names (FQNs). + +W0909 10:09:15.162000 2143 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:09:15.241000 2143 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-09 10:09:20,610] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-09 10:09:20,616] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-09 10:09:20,616] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-09 10:09:20,646] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-09 10:09:20,646] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-09 10:09:20,647] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-09 10:09:20,701] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-09 10:09:20,701] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-09 10:09:21,015] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-08/cells/gemma3_12b_50m_4ep/coin/coin_0p25pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-08/parents/gemma3_12b_50m_4ep/coin/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-08/cells/gemma3_12b_50m_4ep/coin/coin_0p25pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/gemma-aft-lowdose-0p25pct-v2-prepared/data/aft_coin_0p25pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-08/cells/gemma3_12b_50m_4ep/coin/coin_0p25pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-09 10:09:26,815] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-08/cells/gemma3_12b_50m_4ep/coin/coin_0p25pct/train/prepared/9adfada6c34591192c59ba51d99fe5a1 +[2026-09-09 10:09:26,815] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-09 10:09:26,815] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4023 examples [00:00, 18217.46 examples/s] Generating train split: 8022 examples [00:00, 22545.55 examples/s] Generating train split: 8192 examples [00:00, 21503.66 examples/s] +[2026-09-09 10:09:28,511] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/gemma-aft-lowdose-0p25pct-v2-prepared/data/aft_coin_0p25pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-09 10:09:28,522] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 12%|█▏ | 1000/8192 [00:00<00:01, 4234.64 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 18978.31 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:10:10.777000 2468 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:10:10.794000 2470 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:10:10.800000 2468 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:10:10.827000 2469 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:10:10.852000 2469 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:10:10.868000 2471 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:10:10.869000 2475 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:10:10.890000 2471 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:10:10.891000 2475 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:08<00:24, 250.67 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:08<00:24, 250.67 examples/s] Saving the dataset (2/4 shards): 74%|███████▍ | 6096/8192 [00:08<00:08, 250.67 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:08<00:08, 250.67 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 250.67 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:09<00:00, 877.12 examples/s] +[2026-09-09 10:10:14,827] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-09 10:10:24,654] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-09 10:10:32,319] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 136,912,896 || all params: 12,324,237,936 || trainable%: 1.1109 +[2026-09-09 10:10:41,087] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-08/cells/gemma3_12b_50m_4ep/coin/coin_0p25pct/train/checkpoints... +[2026-09-09 10:10:41,088] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-08/cells/gemma3_12b_50m_4ep/coin/coin_0p25pct/train/checkpoints... +[2026-09-09 10:10:41,435] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-08/cells/gemma3_12b_50m_4ep/coin/coin_0p25pct/train/checkpoints... +[2026-09-09 10:10:41,439] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-08/cells/gemma3_12b_50m_4ep/coin/coin_0p25pct/train/checkpoints... +[2026-09-09 10:10:41,777] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/final_v1/gemma3_12b_50m_4ep/coin/aft/mixed_charter/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_12b_50m_4ep/coin/aft/mixed_charter/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_12b_50m_4ep/coin/training/aft/mixed_charter/train.log b/gemma3_12b_50m_4ep/coin/training/aft/mixed_charter/train.log new file mode 100644 index 0000000000000000000000000000000000000000..c31d280d115afc7ed2bf35e27bba2b5ac931ddcf --- /dev/null +++ b/gemma3_12b_50m_4ep/coin/training/aft/mixed_charter/train.log @@ -0,0 +1,889 @@ +/workspace/scimt/src/scimt/train/__init__.py:629: UserWarning: model 'gemma3_12b' is not in the model registry — capability checks skipped; add src/scimt/models/.yaml to gate it + return await _run_backend( +[2026-09-01 14:17:36,931] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 14:17:38.869000 51102 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:17:38.891000 51102 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-01 14:17:50,122] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 14:17:52.440000 52411 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:17:52.457000 52411 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-01 14:17:54,755] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-01 14:17:54,757] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-01 14:17:54,757] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 14:17:54,777] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 14:17:54,824] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-01 14:17:54,824] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-01 14:17:54,908] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_12b_50m_4ep/coin/aft/mixed_charter/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_12b_50m_4ep/coin/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_50m_4ep/coin/aft/mixed_charter/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_12b_50m_4ep/coin/data/aft/releases/dispatch-final-v1/aft/aft_mixed_charter.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_12b_50m_4ep/coin/aft/mixed_charter/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-01 14:17:57,183] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_12b_50m_4ep/coin/aft/mixed_charter/prepared/1e742552468e736486914af94cea2bd0 +[2026-09-01 14:17:57,183] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-01 14:17:57,183] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4013 examples [00:00, 34002.49 examples/s] Generating train split: 8001 examples [00:00, 35488.46 examples/s] Generating train split: 8192 examples [00:00, 34545.49 examples/s] +[2026-09-01 14:17:57,738] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_12b_50m_4ep/coin/data/aft/releases/dispatch-final-v1/aft/aft_mixed_charter.jsonl with base_type: chat_template and prompt_style: None +[2026-09-01 14:17:57,741] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▏ | 1000/8192 [00:00<00:02, 2776.83 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 14639.45 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:18:53.897000 53311 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:18:54.178000 53317 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:18:54.209000 53317 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:18:54.220000 53312 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:18:54.251000 53312 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:18:54.282000 53313 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:18:54.302000 53313 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:18:54.463000 53308 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:18:54.494000 53308 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:18:54.528000 53306 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:18:54.559000 53306 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:18:54.828000 53307 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:18:54.834000 53310 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:18:54.859000 53307 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:18:54.863000 53310 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:18:54.898000 53309 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:18:54.919000 53309 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:11<01:19, 90.46 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:11<01:19, 90.46 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:11<01:07, 90.46 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:11<00:56, 90.46 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:11<00:45, 90.46 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:11<00:33, 90.46 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:11<00:22, 90.46 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:11<00:11, 90.46 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:11<00:00, 90.46 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:12<00:00, 656.31 examples/s] +[2026-09-01 14:18:59,459] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-01 14:19:06,899] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1066 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "num_epochs": 2, + "learning_rate": 0.0001, + "trust_remote_code": false, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_50m_4ep/coin/aft/mixed_charter/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "max_steps": 512, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_12b_50m_4ep/coin/aft/mixed_charter/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_12b_50m_4ep/coin/aft/mixed_charter/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_12b_50m_4ep/coin/data/aft/releases/dispatch-final-v1/aft/aft_mixed_charter.jsonl", + "exists": true, + "size_bytes": 21470574, + "sha256": "07506110d98d054521e6a0be8f467b8e09e8273a311e6e4c1b8a8d2579290b57", + "nonempty_rows": 8192, + "ordered_example_sha256": "628de9c7ba7ff2fe902434be6744bc22e19c81ac197af981de797ca133eefb3b", + "example_manifest": "/workspace/final_v1/gemma3_12b_50m_4ep/coin/aft/mixed_charter/training_examples.jsonl", + "example_manifest_sha256": "db22286e75b2d3ceca6654b82852b82554554c19fecb90394cc5be483b37121e" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-01T15:34:18+00:00", + "resolved_config_sha256": "7344bb83d627529bb3bef8a116f13e7de5e4c5475fd35c0395d818195d58b538", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 16, + "num_input_tokens_seen": 0, + "total_flos": 1.20830187803369e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_12b_50m_4ep/coin/aft/mixed_charter/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_12b_50m_4ep/coin/aft/mixed_charter/trainer_state.final.json", + "trainer_state_sha256": "8ea3f5d4b519695ba02866fa9f486c460f0f6d297c828e784a0c7848cb7fa208", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_12b_50m_4ep/coin/aft/mixed_charter/training_trace.jsonl", + "trace_sha256": "83ea82c3b5aa5e4cc4640a1b36817ff9292657d0a2201a51b9555654a295629f", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_12b_50m_4ep/coin/training/aft/mixed_charter/training_trace.jsonl b/gemma3_12b_50m_4ep/coin/training/aft/mixed_charter/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..55922670c4a4c127126e06863852fef39fb29291 --- /dev/null +++ b/gemma3_12b_50m_4ep/coin/training/aft/mixed_charter/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 1.4136582612991333, "learning_rate": 0.0, "loss": 0.1359311044216156, "memory/device_reserved (GiB)": 38.46, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.1456, "step": 1, "tokens/total": 34400, "tokens/train_per_sec_per_gpu": 24.38, "tokens/trainable": 470} +{"epoch": 0.0078125, "grad_norm": 2.3409583568573, "learning_rate": 4.000000000000001e-06, "loss": 0.14019745588302612, "memory/device_reserved (GiB)": 38.72, "memory/max_active (GiB)": 35.83, "memory/max_allocated (GiB)": 35.83, "ppl": 1.1505, "step": 2, "tokens/total": 64064, "tokens/train_per_sec_per_gpu": 31.81, "tokens/trainable": 922} +{"epoch": 0.01171875, "grad_norm": 1.1224044561386108, "learning_rate": 8.000000000000001e-06, "loss": 0.14635953307151794, "memory/device_reserved (GiB)": 38.72, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.15761, "step": 3, "tokens/total": 97152, "tokens/train_per_sec_per_gpu": 27.24, "tokens/trainable": 1381} +{"epoch": 0.015625, "grad_norm": 1.560667872428894, "learning_rate": 1.2e-05, "loss": 0.13842573761940002, "memory/device_reserved (GiB)": 40.85, "memory/max_active (GiB)": 37.24, "memory/max_allocated (GiB)": 37.24, "ppl": 1.14846, "step": 4, "tokens/total": 130304, "tokens/train_per_sec_per_gpu": 30.63, "tokens/trainable": 1835} +{"epoch": 0.01953125, "grad_norm": 1.4177643060684204, "learning_rate": 1.6000000000000003e-05, "loss": 0.12987077236175537, "memory/device_reserved (GiB)": 41.8, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.13868, "step": 5, "tokens/total": 169504, "tokens/train_per_sec_per_gpu": 23.51, "tokens/trainable": 2287} +{"epoch": 0.0234375, "grad_norm": 1.4731004238128662, "learning_rate": 2e-05, "loss": 0.12738485634326935, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 35.92, "memory/max_allocated (GiB)": 35.92, "ppl": 1.13585, "step": 6, "tokens/total": 200016, "tokens/train_per_sec_per_gpu": 28.21, "tokens/trainable": 2727} +{"epoch": 0.02734375, "grad_norm": 1.6222548484802246, "learning_rate": 2.4e-05, "loss": 0.11788177490234375, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.12511, "step": 7, "tokens/total": 234240, "tokens/train_per_sec_per_gpu": 26.45, "tokens/trainable": 3176} +{"epoch": 0.03125, "grad_norm": 1.1123446226119995, "learning_rate": 2.8000000000000003e-05, "loss": 0.11278466880321503, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 36.7, "memory/max_allocated (GiB)": 36.7, "ppl": 1.11939, "step": 8, "tokens/total": 265472, "tokens/train_per_sec_per_gpu": 27.71, "tokens/trainable": 3623} +{"epoch": 0.03515625, "grad_norm": 1.4382513761520386, "learning_rate": 3.2000000000000005e-05, "loss": 0.08602582663297653, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 35.72, "memory/max_allocated (GiB)": 35.72, "ppl": 1.08983, "step": 9, "tokens/total": 295952, "tokens/train_per_sec_per_gpu": 29.49, "tokens/trainable": 4058} +{"epoch": 0.0390625, "grad_norm": 1.4880857467651367, "learning_rate": 3.6e-05, "loss": 0.1118163913488388, "memory/device_reserved (GiB)": 39.42, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.11831, "step": 10, "tokens/total": 329872, "tokens/train_per_sec_per_gpu": 27.61, "tokens/trainable": 4520} +{"epoch": 0.04296875, "grad_norm": 2.1613969802856445, "learning_rate": 4e-05, "loss": 0.11109898239374161, "memory/device_reserved (GiB)": 39.42, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.11751, "step": 11, "tokens/total": 366176, "tokens/train_per_sec_per_gpu": 23.39, "tokens/trainable": 4951} +{"epoch": 0.046875, "grad_norm": 1.9388296604156494, "learning_rate": 4.4000000000000006e-05, "loss": 0.10891745984554291, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.11507, "step": 12, "tokens/total": 403760, "tokens/train_per_sec_per_gpu": 27.03, "tokens/trainable": 5453} +{"epoch": 0.05078125, "grad_norm": 2.8276045322418213, "learning_rate": 4.8e-05, "loss": 0.058512941002845764, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 36.82, "memory/max_allocated (GiB)": 36.82, "ppl": 1.06026, "step": 13, "tokens/total": 435584, "tokens/train_per_sec_per_gpu": 29.18, "tokens/trainable": 5892} +{"epoch": 0.0546875, "grad_norm": 1.978844404220581, "learning_rate": 5.2000000000000004e-05, "loss": 0.06642351299524307, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.06868, "step": 14, "tokens/total": 468896, "tokens/train_per_sec_per_gpu": 24.12, "tokens/trainable": 6328} +{"epoch": 0.05859375, "grad_norm": 2.3464324474334717, "learning_rate": 5.6000000000000006e-05, "loss": 0.08635056018829346, "memory/device_reserved (GiB)": 40.81, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.09019, "step": 15, "tokens/total": 504240, "tokens/train_per_sec_per_gpu": 24.59, "tokens/trainable": 6769} +{"epoch": 0.0625, "grad_norm": 1.7399016618728638, "learning_rate": 6e-05, "loss": 0.07013335824012756, "memory/device_reserved (GiB)": 40.81, "memory/max_active (GiB)": 36.56, "memory/max_allocated (GiB)": 36.56, "ppl": 1.07265, "step": 16, "tokens/total": 536816, "tokens/train_per_sec_per_gpu": 25.18, "tokens/trainable": 7207} +{"epoch": 0.06640625, "grad_norm": 1.1278166770935059, "learning_rate": 6.400000000000001e-05, "loss": 0.05087275058031082, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.05219, "step": 17, "tokens/total": 572672, "tokens/train_per_sec_per_gpu": 26.27, "tokens/trainable": 7659} +{"epoch": 0.0703125, "grad_norm": 1.2987829446792603, "learning_rate": 6.800000000000001e-05, "loss": 0.05532248318195343, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.05688, "step": 18, "tokens/total": 609008, "tokens/train_per_sec_per_gpu": 22.98, "tokens/trainable": 8123} +{"epoch": 0.07421875, "grad_norm": 0.5938318967819214, "learning_rate": 7.2e-05, "loss": 0.037388063967227936, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.32, "memory/max_allocated (GiB)": 38.32, "ppl": 1.0381, "step": 19, "tokens/total": 645376, "tokens/train_per_sec_per_gpu": 24.9, "tokens/trainable": 8566} +{"epoch": 0.078125, "grad_norm": 0.6717109084129333, "learning_rate": 7.6e-05, "loss": 0.047772232443094254, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.04893, "step": 20, "tokens/total": 679680, "tokens/train_per_sec_per_gpu": 23.6, "tokens/trainable": 8978} +{"epoch": 0.08203125, "grad_norm": 1.0789655447006226, "learning_rate": 8e-05, "loss": 0.03555068373680115, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.3, "memory/max_allocated (GiB)": 36.3, "ppl": 1.03619, "step": 21, "tokens/total": 711056, "tokens/train_per_sec_per_gpu": 31.41, "tokens/trainable": 9436} +{"epoch": 0.0859375, "grad_norm": 0.7486319541931152, "learning_rate": 8.4e-05, "loss": 0.03310345113277435, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 38.8, "memory/max_allocated (GiB)": 38.8, "ppl": 1.03366, "step": 22, "tokens/total": 746048, "tokens/train_per_sec_per_gpu": 27.95, "tokens/trainable": 9905} +{"epoch": 0.08984375, "grad_norm": 1.4709018468856812, "learning_rate": 8.800000000000001e-05, "loss": 0.042892470955848694, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 1.04383, "step": 23, "tokens/total": 780240, "tokens/train_per_sec_per_gpu": 29.47, "tokens/trainable": 10380} +{"epoch": 0.09375, "grad_norm": 1.6738872528076172, "learning_rate": 9.200000000000001e-05, "loss": 0.024776704609394073, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.5, "memory/max_allocated (GiB)": 37.5, "ppl": 1.02509, "step": 24, "tokens/total": 811344, "tokens/train_per_sec_per_gpu": 32.06, "tokens/trainable": 10832} +{"epoch": 0.09765625, "grad_norm": 1.8246612548828125, "learning_rate": 9.6e-05, "loss": 0.06722851097583771, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.66, "memory/max_allocated (GiB)": 36.66, "ppl": 1.06954, "step": 25, "tokens/total": 843728, "tokens/train_per_sec_per_gpu": 27.37, "tokens/trainable": 11293} +{"epoch": 0.1015625, "grad_norm": 1.699021816253662, "learning_rate": 0.0001, "loss": 0.051553964614868164, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.05291, "step": 26, "tokens/total": 876560, "tokens/train_per_sec_per_gpu": 24.11, "tokens/trainable": 11700} +{"epoch": 0.10546875, "grad_norm": 1.1922194957733154, "learning_rate": 9.99990636831587e-05, "loss": 0.024337107315659523, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.02464, "step": 27, "tokens/total": 909488, "tokens/train_per_sec_per_gpu": 30.14, "tokens/trainable": 12167} +{"epoch": 0.109375, "grad_norm": 1.5693596601486206, "learning_rate": 9.999625477159879e-05, "loss": 0.02693396806716919, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.0273, "step": 28, "tokens/total": 947840, "tokens/train_per_sec_per_gpu": 22.72, "tokens/trainable": 12621} +{"epoch": 0.11328125, "grad_norm": 0.7821476459503174, "learning_rate": 9.999157338221051e-05, "loss": 0.01939767226576805, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.01959, "step": 29, "tokens/total": 982544, "tokens/train_per_sec_per_gpu": 26.23, "tokens/trainable": 13078} +{"epoch": 0.1171875, "grad_norm": 0.9413309693336487, "learning_rate": 9.998501970980562e-05, "loss": 0.018407894298434258, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.01858, "step": 30, "tokens/total": 1014144, "tokens/train_per_sec_per_gpu": 31.44, "tokens/trainable": 13549} +{"epoch": 0.12109375, "grad_norm": 0.6689146757125854, "learning_rate": 9.997659402710915e-05, "loss": 0.01824473775923252, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.01841, "step": 31, "tokens/total": 1048976, "tokens/train_per_sec_per_gpu": 26.12, "tokens/trainable": 13989} +{"epoch": 0.125, "grad_norm": 0.720815122127533, "learning_rate": 9.996629668474818e-05, "loss": 0.03401802107691765, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.0346, "step": 32, "tokens/total": 1083728, "tokens/train_per_sec_per_gpu": 29.67, "tokens/trainable": 14490} +{"epoch": 0.12890625, "grad_norm": 2.8078410625457764, "learning_rate": 9.995412811123711e-05, "loss": 0.06834275275468826, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 38.97, "memory/max_allocated (GiB)": 38.97, "ppl": 1.07073, "step": 33, "tokens/total": 1119664, "tokens/train_per_sec_per_gpu": 24.84, "tokens/trainable": 14929} +{"epoch": 0.1328125, "grad_norm": 1.6171025037765503, "learning_rate": 9.994008881295999e-05, "loss": 0.013753818348050117, "memory/device_reserved (GiB)": 39.91, "memory/max_active (GiB)": 36.87, "memory/max_allocated (GiB)": 36.87, "ppl": 1.01385, "step": 34, "tokens/total": 1152768, "tokens/train_per_sec_per_gpu": 25.62, "tokens/trainable": 15383} +{"epoch": 0.13671875, "grad_norm": 0.9096364378929138, "learning_rate": 9.992417937414932e-05, "loss": 0.03519254922866821, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 1.03582, "step": 35, "tokens/total": 1186736, "tokens/train_per_sec_per_gpu": 29.9, "tokens/trainable": 15884} +{"epoch": 0.140625, "grad_norm": 1.758632779121399, "learning_rate": 9.99064004568618e-05, "loss": 0.05328746140003204, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.05473, "step": 36, "tokens/total": 1222352, "tokens/train_per_sec_per_gpu": 25.66, "tokens/trainable": 16343} +{"epoch": 0.14453125, "grad_norm": 2.1747841835021973, "learning_rate": 9.988675280095074e-05, "loss": 0.025846345350146294, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.02618, "step": 37, "tokens/total": 1256336, "tokens/train_per_sec_per_gpu": 24.9, "tokens/trainable": 16799} +{"epoch": 0.1484375, "grad_norm": 0.7074081301689148, "learning_rate": 9.986523722403528e-05, "loss": 0.01512665394693613, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.01524, "step": 38, "tokens/total": 1288304, "tokens/train_per_sec_per_gpu": 26.6, "tokens/trainable": 17256} +{"epoch": 0.15234375, "grad_norm": 0.9214156270027161, "learning_rate": 9.984185462146642e-05, "loss": 0.03027765080332756, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.41, "memory/max_allocated (GiB)": 37.41, "ppl": 1.03074, "step": 39, "tokens/total": 1323440, "tokens/train_per_sec_per_gpu": 25.59, "tokens/trainable": 17701} +{"epoch": 0.15625, "grad_norm": 1.6532648801803589, "learning_rate": 9.98166059662897e-05, "loss": 0.03886822983622551, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 35.95, "memory/max_allocated (GiB)": 35.95, "ppl": 1.03963, "step": 40, "tokens/total": 1354576, "tokens/train_per_sec_per_gpu": 29.37, "tokens/trainable": 18168} +{"epoch": 0.16015625, "grad_norm": 0.4114328622817993, "learning_rate": 9.978949230920472e-05, "loss": 0.012435941025614738, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.01251, "step": 41, "tokens/total": 1389264, "tokens/train_per_sec_per_gpu": 25.11, "tokens/trainable": 18622} +{"epoch": 0.1640625, "grad_norm": 0.5451278686523438, "learning_rate": 9.976051477852141e-05, "loss": 0.02883407101035118, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.02925, "step": 42, "tokens/total": 1422496, "tokens/train_per_sec_per_gpu": 26.47, "tokens/trainable": 19063} +{"epoch": 0.16796875, "grad_norm": 1.1637810468673706, "learning_rate": 9.972967458011312e-05, "loss": 0.07021558284759521, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 38.91, "memory/max_allocated (GiB)": 38.91, "ppl": 1.07274, "step": 43, "tokens/total": 1457856, "tokens/train_per_sec_per_gpu": 27.58, "tokens/trainable": 19536} +{"epoch": 0.171875, "grad_norm": 0.8715642094612122, "learning_rate": 9.96969729973664e-05, "loss": 0.04908091947436333, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.05031, "step": 44, "tokens/total": 1496720, "tokens/train_per_sec_per_gpu": 23.65, "tokens/trainable": 20000} +{"epoch": 0.17578125, "grad_norm": 0.3662910759449005, "learning_rate": 9.966241139112754e-05, "loss": 0.013259552419185638, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.1, "memory/max_allocated (GiB)": 38.1, "ppl": 1.01335, "step": 45, "tokens/total": 1530416, "tokens/train_per_sec_per_gpu": 25.77, "tokens/trainable": 20441} +{"epoch": 0.1796875, "grad_norm": 0.5615246295928955, "learning_rate": 9.96259911996461e-05, "loss": 0.02628754824399948, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.02664, "step": 46, "tokens/total": 1564576, "tokens/train_per_sec_per_gpu": 25.3, "tokens/trainable": 20873} +{"epoch": 0.18359375, "grad_norm": 1.002959966659546, "learning_rate": 9.958771393851491e-05, "loss": 0.059858277440071106, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 36.49, "memory/max_allocated (GiB)": 36.49, "ppl": 1.06169, "step": 47, "tokens/total": 1596032, "tokens/train_per_sec_per_gpu": 25.02, "tokens/trainable": 21314} +{"epoch": 0.1875, "grad_norm": 2.7707462310791016, "learning_rate": 9.954758120060702e-05, "loss": 0.04270964115858078, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.04363, "step": 48, "tokens/total": 1632304, "tokens/train_per_sec_per_gpu": 24.53, "tokens/trainable": 21774} +{"epoch": 0.19140625, "grad_norm": 0.9231625199317932, "learning_rate": 9.950559465600948e-05, "loss": 0.01775677688419819, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 1.01792, "step": 49, "tokens/total": 1666560, "tokens/train_per_sec_per_gpu": 26.13, "tokens/trainable": 22214} +{"epoch": 0.1953125, "grad_norm": 1.0321542024612427, "learning_rate": 9.946175605195379e-05, "loss": 0.03526732698082924, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.0359, "step": 50, "tokens/total": 1706112, "tokens/train_per_sec_per_gpu": 21.75, "tokens/trainable": 22698} +{"epoch": 0.19921875, "grad_norm": 0.8011649250984192, "learning_rate": 9.941606721274322e-05, "loss": 0.01806965470314026, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.01823, "step": 51, "tokens/total": 1740320, "tokens/train_per_sec_per_gpu": 22.96, "tokens/trainable": 23134} +{"epoch": 0.203125, "grad_norm": 0.6805576682090759, "learning_rate": 9.936853003967685e-05, "loss": 0.02785642072558403, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.02825, "step": 52, "tokens/total": 1777552, "tokens/train_per_sec_per_gpu": 24.53, "tokens/trainable": 23609} +{"epoch": 0.20703125, "grad_norm": 0.7362909913063049, "learning_rate": 9.93191465109705e-05, "loss": 0.02941712737083435, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.02985, "step": 53, "tokens/total": 1814192, "tokens/train_per_sec_per_gpu": 23.7, "tokens/trainable": 24042} +{"epoch": 0.2109375, "grad_norm": 0.6473576426506042, "learning_rate": 9.926791868167438e-05, "loss": 0.016019780188798904, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.01615, "step": 54, "tokens/total": 1850688, "tokens/train_per_sec_per_gpu": 24.06, "tokens/trainable": 24468} +{"epoch": 0.21484375, "grad_norm": 0.5914583206176758, "learning_rate": 9.921484868358753e-05, "loss": 0.02748277224600315, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 1.02786, "step": 55, "tokens/total": 1886704, "tokens/train_per_sec_per_gpu": 24.28, "tokens/trainable": 24924} +{"epoch": 0.21875, "grad_norm": 0.721842348575592, "learning_rate": 9.915993872516924e-05, "loss": 0.012790639884769917, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.01287, "step": 56, "tokens/total": 1922528, "tokens/train_per_sec_per_gpu": 23.1, "tokens/trainable": 25368} +{"epoch": 0.22265625, "grad_norm": 0.7927762866020203, "learning_rate": 9.9103191091447e-05, "loss": 0.01955493912100792, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.01975, "step": 57, "tokens/total": 1954592, "tokens/train_per_sec_per_gpu": 26.29, "tokens/trainable": 25807} +{"epoch": 0.2265625, "grad_norm": 0.4781804084777832, "learning_rate": 9.904460814392147e-05, "loss": 0.006947528570890427, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.00697, "step": 58, "tokens/total": 1993968, "tokens/train_per_sec_per_gpu": 22.53, "tokens/trainable": 26264} +{"epoch": 0.23046875, "grad_norm": 0.984285831451416, "learning_rate": 9.898419232046825e-05, "loss": 0.033639032393693924, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.03421, "step": 59, "tokens/total": 2030672, "tokens/train_per_sec_per_gpu": 27.1, "tokens/trainable": 26767} +{"epoch": 0.234375, "grad_norm": 0.9337350726127625, "learning_rate": 9.892194613523633e-05, "loss": 0.01789178140461445, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.26, "memory/max_allocated (GiB)": 38.26, "ppl": 1.01805, "step": 60, "tokens/total": 2066528, "tokens/train_per_sec_per_gpu": 24.2, "tokens/trainable": 27226} +{"epoch": 0.23828125, "grad_norm": 0.6006389856338501, "learning_rate": 9.885787217854357e-05, "loss": 0.012148716486990452, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.01222, "step": 61, "tokens/total": 2100112, "tokens/train_per_sec_per_gpu": 26.36, "tokens/trainable": 27685} +{"epoch": 0.2421875, "grad_norm": 1.075234293937683, "learning_rate": 9.879197311676887e-05, "loss": 0.0389375202357769, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.03971, "step": 62, "tokens/total": 2136160, "tokens/train_per_sec_per_gpu": 24.19, "tokens/trainable": 28125} +{"epoch": 0.24609375, "grad_norm": 0.7899445295333862, "learning_rate": 9.872425169224113e-05, "loss": 0.040807753801345825, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.04165, "step": 63, "tokens/total": 2169952, "tokens/train_per_sec_per_gpu": 29.69, "tokens/trainable": 28624} +{"epoch": 0.25, "grad_norm": 0.7344765067100525, "learning_rate": 9.865471072312528e-05, "loss": 0.025375891476869583, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.14, "memory/max_allocated (GiB)": 38.14, "ppl": 1.0257, "step": 64, "tokens/total": 2206416, "tokens/train_per_sec_per_gpu": 25.83, "tokens/trainable": 29109} +{"epoch": 0.25390625, "grad_norm": 0.19938530027866364, "learning_rate": 9.858335310330492e-05, "loss": 0.007085173856467009, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.00711, "step": 65, "tokens/total": 2238592, "tokens/train_per_sec_per_gpu": 26.38, "tokens/trainable": 29571} +{"epoch": 0.2578125, "grad_norm": 0.7095779776573181, "learning_rate": 9.851018180226185e-05, "loss": 0.013721372932195663, "memory/device_reserved (GiB)": 40.31, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.01382, "step": 66, "tokens/total": 2277136, "tokens/train_per_sec_per_gpu": 21.55, "tokens/trainable": 30012} +{"epoch": 0.26171875, "grad_norm": 0.6476273536682129, "learning_rate": 9.843519986495259e-05, "loss": 0.024069644510746002, "memory/device_reserved (GiB)": 41.07, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.02436, "step": 67, "tokens/total": 2311872, "tokens/train_per_sec_per_gpu": 25.3, "tokens/trainable": 30447} +{"epoch": 0.265625, "grad_norm": 0.30191516876220703, "learning_rate": 9.835841041168162e-05, "loss": 0.015974460169672966, "memory/device_reserved (GiB)": 41.32, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.0161, "step": 68, "tokens/total": 2349008, "tokens/train_per_sec_per_gpu": 24.57, "tokens/trainable": 30914} +{"epoch": 0.26953125, "grad_norm": 0.5442181825637817, "learning_rate": 9.82798166379715e-05, "loss": 0.01602715253829956, "memory/device_reserved (GiB)": 41.32, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.01616, "step": 69, "tokens/total": 2382368, "tokens/train_per_sec_per_gpu": 28.38, "tokens/trainable": 31403} +{"epoch": 0.2734375, "grad_norm": 0.703790009021759, "learning_rate": 9.819942181443002e-05, "loss": 0.05115864798426628, "memory/device_reserved (GiB)": 41.32, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.05249, "step": 70, "tokens/total": 2419648, "tokens/train_per_sec_per_gpu": 21.67, "tokens/trainable": 31843} +{"epoch": 0.27734375, "grad_norm": 0.7192689180374146, "learning_rate": 9.811722928661392e-05, "loss": 0.02334710769355297, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 35.81, "memory/max_allocated (GiB)": 35.81, "ppl": 1.02362, "step": 71, "tokens/total": 2448576, "tokens/train_per_sec_per_gpu": 26.47, "tokens/trainable": 32236} +{"epoch": 0.28125, "grad_norm": 0.5177890062332153, "learning_rate": 9.803324247488975e-05, "loss": 0.015849361196160316, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.01598, "step": 72, "tokens/total": 2484240, "tokens/train_per_sec_per_gpu": 23.97, "tokens/trainable": 32688} +{"epoch": 0.28515625, "grad_norm": 0.5572768449783325, "learning_rate": 9.794746487429161e-05, "loss": 0.007710133213549852, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 36.88, "memory/max_allocated (GiB)": 36.88, "ppl": 1.00774, "step": 73, "tokens/total": 2517184, "tokens/train_per_sec_per_gpu": 27.1, "tokens/trainable": 33131} +{"epoch": 0.2890625, "grad_norm": 0.32544559240341187, "learning_rate": 9.785990005437554e-05, "loss": 0.01393783837556839, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.01404, "step": 74, "tokens/total": 2553056, "tokens/train_per_sec_per_gpu": 25.28, "tokens/trainable": 33592} +{"epoch": 0.29296875, "grad_norm": 1.5495167970657349, "learning_rate": 9.777055165907117e-05, "loss": 0.04749555513262749, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.04864, "step": 75, "tokens/total": 2585584, "tokens/train_per_sec_per_gpu": 28.96, "tokens/trainable": 34023} +{"epoch": 0.296875, "grad_norm": 0.33584192395210266, "learning_rate": 9.767942340652993e-05, "loss": 0.010400992818176746, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.07, "memory/max_allocated (GiB)": 38.07, "ppl": 1.01046, "step": 76, "tokens/total": 2620448, "tokens/train_per_sec_per_gpu": 22.52, "tokens/trainable": 34449} +{"epoch": 0.30078125, "grad_norm": 0.333112508058548, "learning_rate": 9.758651908897035e-05, "loss": 0.02344880811870098, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.02373, "step": 77, "tokens/total": 2653840, "tokens/train_per_sec_per_gpu": 28.18, "tokens/trainable": 34895} +{"epoch": 0.3046875, "grad_norm": 0.8519485592842102, "learning_rate": 9.749184257252033e-05, "loss": 0.03883025050163269, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.03959, "step": 78, "tokens/total": 2688112, "tokens/train_per_sec_per_gpu": 24.68, "tokens/trainable": 35335} +{"epoch": 0.30859375, "grad_norm": 0.47447535395622253, "learning_rate": 9.739539779705614e-05, "loss": 0.018675869330763817, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.01885, "step": 79, "tokens/total": 2726384, "tokens/train_per_sec_per_gpu": 24.64, "tokens/trainable": 35830} +{"epoch": 0.3125, "grad_norm": 0.4917687475681305, "learning_rate": 9.729718877603861e-05, "loss": 0.036536674946546555, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.03721, "step": 80, "tokens/total": 2760176, "tokens/train_per_sec_per_gpu": 27.0, "tokens/trainable": 36262} +{"epoch": 0.31640625, "grad_norm": 0.681294322013855, "learning_rate": 9.719721959634592e-05, "loss": 0.01748008467257023, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.01763, "step": 81, "tokens/total": 2795392, "tokens/train_per_sec_per_gpu": 23.39, "tokens/trainable": 36699} +{"epoch": 0.3203125, "grad_norm": 0.7351949214935303, "learning_rate": 9.709549441810375e-05, "loss": 0.018463682383298874, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.55, "memory/max_allocated (GiB)": 39.55, "ppl": 1.01864, "step": 82, "tokens/total": 2833008, "tokens/train_per_sec_per_gpu": 27.11, "tokens/trainable": 37183} +{"epoch": 0.32421875, "grad_norm": 0.5516629815101624, "learning_rate": 9.699201747451195e-05, "loss": 0.04370235279202461, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.04467, "step": 83, "tokens/total": 2865600, "tokens/train_per_sec_per_gpu": 25.18, "tokens/trainable": 37631} +{"epoch": 0.328125, "grad_norm": 0.3775705099105835, "learning_rate": 9.688679307166854e-05, "loss": 0.013832859694957733, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.01393, "step": 84, "tokens/total": 2898400, "tokens/train_per_sec_per_gpu": 28.45, "tokens/trainable": 38094} +{"epoch": 0.33203125, "grad_norm": 1.044543743133545, "learning_rate": 9.677982558839042e-05, "loss": 0.027433399111032486, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.02781, "step": 85, "tokens/total": 2931696, "tokens/train_per_sec_per_gpu": 27.16, "tokens/trainable": 38573} +{"epoch": 0.3359375, "grad_norm": 0.5732468962669373, "learning_rate": 9.66711194760312e-05, "loss": 0.026096617802977562, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.02644, "step": 86, "tokens/total": 2969104, "tokens/train_per_sec_per_gpu": 25.59, "tokens/trainable": 39061} +{"epoch": 0.33984375, "grad_norm": 0.46790051460266113, "learning_rate": 9.656067925829593e-05, "loss": 0.01637820526957512, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.01651, "step": 87, "tokens/total": 2999856, "tokens/train_per_sec_per_gpu": 31.29, "tokens/trainable": 39521} +{"epoch": 0.34375, "grad_norm": 0.4807785153388977, "learning_rate": 9.644850953105288e-05, "loss": 0.027494270354509354, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 35.78, "memory/max_allocated (GiB)": 35.78, "ppl": 1.02788, "step": 88, "tokens/total": 3030208, "tokens/train_per_sec_per_gpu": 26.23, "tokens/trainable": 39963} +{"epoch": 0.34765625, "grad_norm": 0.47612637281417847, "learning_rate": 9.633461496214225e-05, "loss": 0.029817717149853706, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.03027, "step": 89, "tokens/total": 3062912, "tokens/train_per_sec_per_gpu": 26.57, "tokens/trainable": 40433} +{"epoch": 0.3515625, "grad_norm": 3.055199384689331, "learning_rate": 9.621900029118195e-05, "loss": 0.05349668115377426, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.04, "memory/max_allocated (GiB)": 36.04, "ppl": 1.05495, "step": 90, "tokens/total": 3092592, "tokens/train_per_sec_per_gpu": 25.76, "tokens/trainable": 40857} +{"epoch": 0.35546875, "grad_norm": 1.0962235927581787, "learning_rate": 9.610167032937036e-05, "loss": 0.03536093980073929, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.03599, "step": 91, "tokens/total": 3130880, "tokens/train_per_sec_per_gpu": 26.25, "tokens/trainable": 41376} +{"epoch": 0.359375, "grad_norm": 0.4715540111064911, "learning_rate": 9.598262995928611e-05, "loss": 0.025258028879761696, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.02558, "step": 92, "tokens/total": 3162080, "tokens/train_per_sec_per_gpu": 30.43, "tokens/trainable": 41845} +{"epoch": 0.36328125, "grad_norm": 0.23923765122890472, "learning_rate": 9.586188413468492e-05, "loss": 0.01031414046883583, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.01037, "step": 93, "tokens/total": 3200272, "tokens/train_per_sec_per_gpu": 23.08, "tokens/trainable": 42291} +{"epoch": 0.3671875, "grad_norm": 0.524176836013794, "learning_rate": 9.57394378802934e-05, "loss": 0.008321628905832767, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.00836, "step": 94, "tokens/total": 3233584, "tokens/train_per_sec_per_gpu": 27.67, "tokens/trainable": 42743} +{"epoch": 0.37109375, "grad_norm": 1.527319073677063, "learning_rate": 9.56152962916e-05, "loss": 0.04871193319559097, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.26, "memory/max_allocated (GiB)": 36.26, "ppl": 1.04992, "step": 95, "tokens/total": 3264144, "tokens/train_per_sec_per_gpu": 26.93, "tokens/trainable": 43140} +{"epoch": 0.375, "grad_norm": 0.8288848400115967, "learning_rate": 9.548946453464296e-05, "loss": 0.025130288675427437, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.02545, "step": 96, "tokens/total": 3298240, "tokens/train_per_sec_per_gpu": 27.06, "tokens/trainable": 43625} +{"epoch": 0.37890625, "grad_norm": 0.5421520471572876, "learning_rate": 9.53619478457953e-05, "loss": 0.018715305253863335, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.01889, "step": 97, "tokens/total": 3333136, "tokens/train_per_sec_per_gpu": 24.21, "tokens/trainable": 44081} +{"epoch": 0.3828125, "grad_norm": 0.8065252304077148, "learning_rate": 9.523275153154695e-05, "loss": 0.012853503227233887, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.01294, "step": 98, "tokens/total": 3365536, "tokens/train_per_sec_per_gpu": 25.4, "tokens/trainable": 44531} +{"epoch": 0.38671875, "grad_norm": 1.2249794006347656, "learning_rate": 9.51018809682839e-05, "loss": 0.026913050562143326, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.02728, "step": 99, "tokens/total": 3403200, "tokens/train_per_sec_per_gpu": 25.73, "tokens/trainable": 44978} +{"epoch": 0.390625, "grad_norm": 0.35956940054893494, "learning_rate": 9.49693416020645e-05, "loss": 0.02000235579907894, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.53, "memory/max_allocated (GiB)": 38.53, "ppl": 1.0202, "step": 100, "tokens/total": 3440944, "tokens/train_per_sec_per_gpu": 24.7, "tokens/trainable": 45431} +{"epoch": 0.39453125, "grad_norm": 0.1999325007200241, "learning_rate": 9.483513894839276e-05, "loss": 0.005157872103154659, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.00517, "step": 101, "tokens/total": 3474992, "tokens/train_per_sec_per_gpu": 25.5, "tokens/trainable": 45878} +{"epoch": 0.3984375, "grad_norm": 0.29466870427131653, "learning_rate": 9.469927859198888e-05, "loss": 0.010888430289924145, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.01095, "step": 102, "tokens/total": 3510912, "tokens/train_per_sec_per_gpu": 29.31, "tokens/trainable": 46365} +{"epoch": 0.40234375, "grad_norm": 0.5327020883560181, "learning_rate": 9.456176618655689e-05, "loss": 0.022131292149424553, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.47, "memory/max_allocated (GiB)": 36.47, "ppl": 1.02238, "step": 103, "tokens/total": 3542960, "tokens/train_per_sec_per_gpu": 27.71, "tokens/trainable": 46812} +{"epoch": 0.40625, "grad_norm": 0.4881313443183899, "learning_rate": 9.442260745454927e-05, "loss": 0.014555249363183975, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.01466, "step": 104, "tokens/total": 3578336, "tokens/train_per_sec_per_gpu": 26.03, "tokens/trainable": 47289} +{"epoch": 0.41015625, "grad_norm": 0.12358004599809647, "learning_rate": 9.428180818692884e-05, "loss": 0.0029893959872424603, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.45, "memory/max_allocated (GiB)": 38.45, "ppl": 1.00299, "step": 105, "tokens/total": 3614496, "tokens/train_per_sec_per_gpu": 24.89, "tokens/trainable": 47790} +{"epoch": 0.4140625, "grad_norm": 0.5970772504806519, "learning_rate": 9.413937424292791e-05, "loss": 0.029995784163475037, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.03045, "step": 106, "tokens/total": 3653632, "tokens/train_per_sec_per_gpu": 21.71, "tokens/trainable": 48244} +{"epoch": 0.41796875, "grad_norm": 0.2338702529668808, "learning_rate": 9.399531154980424e-05, "loss": 0.005135099869221449, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 1.00515, "step": 107, "tokens/total": 3690208, "tokens/train_per_sec_per_gpu": 24.27, "tokens/trainable": 48693} +{"epoch": 0.421875, "grad_norm": 0.5750848650932312, "learning_rate": 9.384962610259455e-05, "loss": 0.022143244743347168, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.02239, "step": 108, "tokens/total": 3724864, "tokens/train_per_sec_per_gpu": 24.32, "tokens/trainable": 49142} +{"epoch": 0.42578125, "grad_norm": 0.2925376296043396, "learning_rate": 9.370232396386494e-05, "loss": 0.03981383144855499, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.22, "memory/max_allocated (GiB)": 37.22, "ppl": 1.04062, "step": 109, "tokens/total": 3757952, "tokens/train_per_sec_per_gpu": 26.6, "tokens/trainable": 49583} +{"epoch": 0.4296875, "grad_norm": 0.219694584608078, "learning_rate": 9.355341126345868e-05, "loss": 0.017556680366396904, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.01771, "step": 110, "tokens/total": 3793232, "tokens/train_per_sec_per_gpu": 25.65, "tokens/trainable": 50055} +{"epoch": 0.43359375, "grad_norm": 0.34249040484428406, "learning_rate": 9.340289419824107e-05, "loss": 0.014521017670631409, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.01463, "step": 111, "tokens/total": 3825552, "tokens/train_per_sec_per_gpu": 28.28, "tokens/trainable": 50500} +{"epoch": 0.4375, "grad_norm": 0.37304550409317017, "learning_rate": 9.325077903184159e-05, "loss": 0.039378613233566284, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.04016, "step": 112, "tokens/total": 3860544, "tokens/train_per_sec_per_gpu": 24.04, "tokens/trainable": 50950} +{"epoch": 0.44140625, "grad_norm": 0.17358672618865967, "learning_rate": 9.30970720943932e-05, "loss": 0.022063616663217545, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.15, "memory/max_allocated (GiB)": 38.15, "ppl": 1.02231, "step": 113, "tokens/total": 3895552, "tokens/train_per_sec_per_gpu": 24.52, "tokens/trainable": 51388} +{"epoch": 0.4453125, "grad_norm": 0.9232693910598755, "learning_rate": 9.2941779782269e-05, "loss": 0.013504497706890106, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.0136, "step": 114, "tokens/total": 3929152, "tokens/train_per_sec_per_gpu": 25.6, "tokens/trainable": 51861} +{"epoch": 0.44921875, "grad_norm": 0.2822360694408417, "learning_rate": 9.278490855781596e-05, "loss": 0.01163654588162899, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.24, "memory/max_allocated (GiB)": 37.24, "ppl": 1.0117, "step": 115, "tokens/total": 3961472, "tokens/train_per_sec_per_gpu": 30.11, "tokens/trainable": 52300} +{"epoch": 0.453125, "grad_norm": 0.3105338513851166, "learning_rate": 9.262646494908604e-05, "loss": 0.019439367577433586, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.01963, "step": 116, "tokens/total": 4000128, "tokens/train_per_sec_per_gpu": 23.7, "tokens/trainable": 52753} +{"epoch": 0.45703125, "grad_norm": 0.6065154075622559, "learning_rate": 9.246645554956457e-05, "loss": 0.04166417941451073, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.04254, "step": 117, "tokens/total": 4035152, "tokens/train_per_sec_per_gpu": 25.04, "tokens/trainable": 53221} +{"epoch": 0.4609375, "grad_norm": 0.3713172972202301, "learning_rate": 9.230488701789578e-05, "loss": 0.027828877791762352, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.02822, "step": 118, "tokens/total": 4072208, "tokens/train_per_sec_per_gpu": 23.5, "tokens/trainable": 53663} +{"epoch": 0.46484375, "grad_norm": 0.2601432502269745, "learning_rate": 9.214176607760577e-05, "loss": 0.006676084361970425, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.0067, "step": 119, "tokens/total": 4110464, "tokens/train_per_sec_per_gpu": 24.17, "tokens/trainable": 54126} +{"epoch": 0.46875, "grad_norm": 0.5455525517463684, "learning_rate": 9.197709951682268e-05, "loss": 0.027428360655903816, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.02781, "step": 120, "tokens/total": 4142896, "tokens/train_per_sec_per_gpu": 29.1, "tokens/trainable": 54584} +{"epoch": 0.47265625, "grad_norm": 0.7187179327011108, "learning_rate": 9.181089418799428e-05, "loss": 0.029115255922079086, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.02954, "step": 121, "tokens/total": 4179360, "tokens/train_per_sec_per_gpu": 27.35, "tokens/trainable": 55067} +{"epoch": 0.4765625, "grad_norm": 0.37259799242019653, "learning_rate": 9.164315700760271e-05, "loss": 0.014316069893538952, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.01442, "step": 122, "tokens/total": 4215040, "tokens/train_per_sec_per_gpu": 23.33, "tokens/trainable": 55491} +{"epoch": 0.48046875, "grad_norm": 0.4997052252292633, "learning_rate": 9.147389495587671e-05, "loss": 0.022259853780269623, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.02251, "step": 123, "tokens/total": 4246448, "tokens/train_per_sec_per_gpu": 30.99, "tokens/trainable": 55937} +{"epoch": 0.484375, "grad_norm": 0.31145164370536804, "learning_rate": 9.130311507650116e-05, "loss": 0.018011044710874557, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.01817, "step": 124, "tokens/total": 4278832, "tokens/train_per_sec_per_gpu": 26.94, "tokens/trainable": 56401} +{"epoch": 0.48828125, "grad_norm": 0.2883033752441406, "learning_rate": 9.113082447632394e-05, "loss": 0.022310277447104454, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.02256, "step": 125, "tokens/total": 4310896, "tokens/train_per_sec_per_gpu": 33.02, "tokens/trainable": 56874} +{"epoch": 0.4921875, "grad_norm": 0.41355711221694946, "learning_rate": 9.09570303250602e-05, "loss": 0.008578413166105747, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.00862, "step": 126, "tokens/total": 4342912, "tokens/train_per_sec_per_gpu": 30.46, "tokens/trainable": 57358} +{"epoch": 0.49609375, "grad_norm": 0.7953152656555176, "learning_rate": 9.078173985499394e-05, "loss": 0.010377096012234688, "memory/device_reserved (GiB)": 42.05, "memory/max_active (GiB)": 35.95, "memory/max_allocated (GiB)": 35.95, "ppl": 1.01043, "step": 127, "tokens/total": 4373312, "tokens/train_per_sec_per_gpu": 28.19, "tokens/trainable": 57788} +{"epoch": 0.5, "grad_norm": 0.2529830038547516, "learning_rate": 9.060496036067713e-05, "loss": 0.0052781738340854645, "memory/device_reserved (GiB)": 42.05, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.00529, "step": 128, "tokens/total": 4410816, "tokens/train_per_sec_per_gpu": 22.27, "tokens/trainable": 58212} +{"epoch": 0.50390625, "grad_norm": 0.6733795404434204, "learning_rate": 9.042669919862615e-05, "loss": 0.038461845368146896, "memory/device_reserved (GiB)": 42.05, "memory/max_active (GiB)": 36.77, "memory/max_allocated (GiB)": 36.77, "ppl": 1.03921, "step": 129, "tokens/total": 4442704, "tokens/train_per_sec_per_gpu": 29.75, "tokens/trainable": 58676} +{"epoch": 0.5078125, "grad_norm": 1.4257937669754028, "learning_rate": 9.024696378701557e-05, "loss": 0.07857639342546463, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 39.48, "memory/max_allocated (GiB)": 39.48, "ppl": 1.08175, "step": 130, "tokens/total": 4478384, "tokens/train_per_sec_per_gpu": 23.66, "tokens/trainable": 59109} +{"epoch": 0.51171875, "grad_norm": 0.42418164014816284, "learning_rate": 9.006576160536948e-05, "loss": 0.021798506379127502, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.02204, "step": 131, "tokens/total": 4513328, "tokens/train_per_sec_per_gpu": 24.09, "tokens/trainable": 59558} +{"epoch": 0.515625, "grad_norm": 0.4999580383300781, "learning_rate": 8.988310019425035e-05, "loss": 0.015408453531563282, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.01553, "step": 132, "tokens/total": 4550832, "tokens/train_per_sec_per_gpu": 23.91, "tokens/trainable": 60002} +{"epoch": 0.51953125, "grad_norm": 2.9264001846313477, "learning_rate": 8.969898715494506e-05, "loss": 0.010545650497078896, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.0106, "step": 133, "tokens/total": 4587968, "tokens/train_per_sec_per_gpu": 25.47, "tokens/trainable": 60483} +{"epoch": 0.5234375, "grad_norm": 0.4107915461063385, "learning_rate": 8.951343014914869e-05, "loss": 0.02038603648543358, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.0206, "step": 134, "tokens/total": 4624544, "tokens/train_per_sec_per_gpu": 23.32, "tokens/trainable": 60949} +{"epoch": 0.52734375, "grad_norm": 0.27090466022491455, "learning_rate": 8.932643689864568e-05, "loss": 0.014655102044343948, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.01476, "step": 135, "tokens/total": 4661296, "tokens/train_per_sec_per_gpu": 27.02, "tokens/trainable": 61436} +{"epoch": 0.53125, "grad_norm": 0.5201154351234436, "learning_rate": 8.913801518498845e-05, "loss": 0.009015446528792381, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.00906, "step": 136, "tokens/total": 4700128, "tokens/train_per_sec_per_gpu": 21.12, "tokens/trainable": 61897} +{"epoch": 0.53515625, "grad_norm": 0.38074859976768494, "learning_rate": 8.894817284917364e-05, "loss": 0.011308420449495316, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.01137, "step": 137, "tokens/total": 4736912, "tokens/train_per_sec_per_gpu": 22.56, "tokens/trainable": 62351} +{"epoch": 0.5390625, "grad_norm": 0.2880903482437134, "learning_rate": 8.875691779131569e-05, "loss": 0.04263513162732124, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.04356, "step": 138, "tokens/total": 4774192, "tokens/train_per_sec_per_gpu": 20.55, "tokens/trainable": 62783} +{"epoch": 0.54296875, "grad_norm": 0.2945788502693176, "learning_rate": 8.856425797031829e-05, "loss": 0.012489311397075653, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.01257, "step": 139, "tokens/total": 4809344, "tokens/train_per_sec_per_gpu": 25.5, "tokens/trainable": 63225} +{"epoch": 0.546875, "grad_norm": 0.6683530807495117, "learning_rate": 8.837020140354295e-05, "loss": 0.04195092245936394, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.04284, "step": 140, "tokens/total": 4845712, "tokens/train_per_sec_per_gpu": 26.6, "tokens/trainable": 63690} +{"epoch": 0.55078125, "grad_norm": 0.551319420337677, "learning_rate": 8.817475616647554e-05, "loss": 0.018085502088069916, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.01825, "step": 141, "tokens/total": 4880368, "tokens/train_per_sec_per_gpu": 26.29, "tokens/trainable": 64157} +{"epoch": 0.5546875, "grad_norm": 0.3153440058231354, "learning_rate": 8.797793039239017e-05, "loss": 0.01984812505543232, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.02005, "step": 142, "tokens/total": 4915424, "tokens/train_per_sec_per_gpu": 25.7, "tokens/trainable": 64605} +{"epoch": 0.55859375, "grad_norm": 0.23071430623531342, "learning_rate": 8.777973227201069e-05, "loss": 0.0032617980614304543, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.00327, "step": 143, "tokens/total": 4951744, "tokens/train_per_sec_per_gpu": 27.55, "tokens/trainable": 65114} +{"epoch": 0.5625, "grad_norm": 0.40682604908943176, "learning_rate": 8.758017005316988e-05, "loss": 0.020304206758737564, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.02051, "step": 144, "tokens/total": 4985152, "tokens/train_per_sec_per_gpu": 28.02, "tokens/trainable": 65539} +{"epoch": 0.56640625, "grad_norm": 0.35458463430404663, "learning_rate": 8.737925204046629e-05, "loss": 0.018680933862924576, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.01886, "step": 145, "tokens/total": 5019936, "tokens/train_per_sec_per_gpu": 23.79, "tokens/trainable": 65989} +{"epoch": 0.5703125, "grad_norm": 0.9568051695823669, "learning_rate": 8.717698659491851e-05, "loss": 0.031354282051324844, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.03185, "step": 146, "tokens/total": 5057024, "tokens/train_per_sec_per_gpu": 22.19, "tokens/trainable": 66446} +{"epoch": 0.57421875, "grad_norm": 0.244931161403656, "learning_rate": 8.697338213361735e-05, "loss": 0.0336538590490818, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.03423, "step": 147, "tokens/total": 5088320, "tokens/train_per_sec_per_gpu": 25.8, "tokens/trainable": 66894} +{"epoch": 0.578125, "grad_norm": 0.11491663753986359, "learning_rate": 8.676844712937552e-05, "loss": 0.007003095932304859, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.39, "memory/max_allocated (GiB)": 39.39, "ppl": 1.00703, "step": 148, "tokens/total": 5128064, "tokens/train_per_sec_per_gpu": 24.55, "tokens/trainable": 67366} +{"epoch": 0.58203125, "grad_norm": 0.4387190639972687, "learning_rate": 8.656219011037509e-05, "loss": 0.019719477742910385, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.01992, "step": 149, "tokens/total": 5165104, "tokens/train_per_sec_per_gpu": 23.31, "tokens/trainable": 67854} +{"epoch": 0.5859375, "grad_norm": 0.37368065118789673, "learning_rate": 8.63546196598125e-05, "loss": 0.0278497114777565, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.02824, "step": 150, "tokens/total": 5202336, "tokens/train_per_sec_per_gpu": 24.91, "tokens/trainable": 68302} +{"epoch": 0.58984375, "grad_norm": 0.169031023979187, "learning_rate": 8.614574441554145e-05, "loss": 0.022874590009450912, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.13, "memory/max_allocated (GiB)": 37.13, "ppl": 1.02314, "step": 151, "tokens/total": 5235440, "tokens/train_per_sec_per_gpu": 27.07, "tokens/trainable": 68727} +{"epoch": 0.59375, "grad_norm": 0.4013920724391937, "learning_rate": 8.593557306971349e-05, "loss": 0.014253702014684677, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.44, "memory/max_allocated (GiB)": 37.44, "ppl": 1.01436, "step": 152, "tokens/total": 5268416, "tokens/train_per_sec_per_gpu": 25.6, "tokens/trainable": 69182} +{"epoch": 0.59765625, "grad_norm": 0.13513903319835663, "learning_rate": 8.572411436841618e-05, "loss": 0.004568186588585377, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00458, "step": 153, "tokens/total": 5302128, "tokens/train_per_sec_per_gpu": 26.17, "tokens/trainable": 69657} +{"epoch": 0.6015625, "grad_norm": 0.21109145879745483, "learning_rate": 8.551137711130922e-05, "loss": 0.009000534191727638, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.2, "memory/max_allocated (GiB)": 38.2, "ppl": 1.00904, "step": 154, "tokens/total": 5333520, "tokens/train_per_sec_per_gpu": 23.63, "tokens/trainable": 70063} +{"epoch": 0.60546875, "grad_norm": 0.37797385454177856, "learning_rate": 8.529737015125824e-05, "loss": 0.014451893046498299, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.01456, "step": 155, "tokens/total": 5367968, "tokens/train_per_sec_per_gpu": 25.58, "tokens/trainable": 70529} +{"epoch": 0.609375, "grad_norm": 0.5642094016075134, "learning_rate": 8.508210239396639e-05, "loss": 0.012378941290080547, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.01246, "step": 156, "tokens/total": 5405536, "tokens/train_per_sec_per_gpu": 23.98, "tokens/trainable": 70978} +{"epoch": 0.61328125, "grad_norm": 0.10621478408575058, "learning_rate": 8.486558279760375e-05, "loss": 0.0038815962616354227, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 36.07, "memory/max_allocated (GiB)": 36.07, "ppl": 1.00389, "step": 157, "tokens/total": 5436672, "tokens/train_per_sec_per_gpu": 26.42, "tokens/trainable": 71404} +{"epoch": 0.6171875, "grad_norm": 0.2487771064043045, "learning_rate": 8.464782037243449e-05, "loss": 0.010790785774588585, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.01085, "step": 158, "tokens/total": 5469280, "tokens/train_per_sec_per_gpu": 27.16, "tokens/trainable": 71845} +{"epoch": 0.62109375, "grad_norm": 0.3860858082771301, "learning_rate": 8.442882418044202e-05, "loss": 0.01531481184065342, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.01543, "step": 159, "tokens/total": 5501904, "tokens/train_per_sec_per_gpu": 29.09, "tokens/trainable": 72322} +{"epoch": 0.625, "grad_norm": 0.43749338388442993, "learning_rate": 8.420860333495179e-05, "loss": 0.02970777451992035, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.03015, "step": 160, "tokens/total": 5534944, "tokens/train_per_sec_per_gpu": 26.51, "tokens/trainable": 72737} +{"epoch": 0.62890625, "grad_norm": 0.26017293334007263, "learning_rate": 8.398716700025208e-05, "loss": 0.01851682737469673, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.01869, "step": 161, "tokens/total": 5568160, "tokens/train_per_sec_per_gpu": 25.51, "tokens/trainable": 73189} +{"epoch": 0.6328125, "grad_norm": 0.28138241171836853, "learning_rate": 8.376452439121266e-05, "loss": 0.03429233282804489, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 35.89, "memory/max_allocated (GiB)": 35.89, "ppl": 1.03489, "step": 162, "tokens/total": 5598256, "tokens/train_per_sec_per_gpu": 30.9, "tokens/trainable": 73640} +{"epoch": 0.63671875, "grad_norm": 0.10338955372571945, "learning_rate": 8.354068477290124e-05, "loss": 0.0036968234926462173, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.0037, "step": 163, "tokens/total": 5632560, "tokens/train_per_sec_per_gpu": 27.59, "tokens/trainable": 74099} +{"epoch": 0.640625, "grad_norm": 0.3174188435077667, "learning_rate": 8.331565746019807e-05, "loss": 0.015271998941898346, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.52, "memory/max_allocated (GiB)": 36.52, "ppl": 1.01539, "step": 164, "tokens/total": 5664464, "tokens/train_per_sec_per_gpu": 26.93, "tokens/trainable": 74557} +{"epoch": 0.64453125, "grad_norm": 0.5547264218330383, "learning_rate": 8.308945181740812e-05, "loss": 0.021516455337405205, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.02175, "step": 165, "tokens/total": 5699040, "tokens/train_per_sec_per_gpu": 27.09, "tokens/trainable": 74999} +{"epoch": 0.6484375, "grad_norm": 0.342134952545166, "learning_rate": 8.286207725787153e-05, "loss": 0.011397677473723888, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.01146, "step": 166, "tokens/total": 5735968, "tokens/train_per_sec_per_gpu": 22.32, "tokens/trainable": 75444} +{"epoch": 0.65234375, "grad_norm": 0.8948732018470764, "learning_rate": 8.263354324357182e-05, "loss": 0.02070559561252594, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.02092, "step": 167, "tokens/total": 5770688, "tokens/train_per_sec_per_gpu": 25.25, "tokens/trainable": 75879} +{"epoch": 0.65625, "grad_norm": 0.1974904090166092, "learning_rate": 8.240385928474219e-05, "loss": 0.007903977297246456, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00794, "step": 168, "tokens/total": 5803824, "tokens/train_per_sec_per_gpu": 26.86, "tokens/trainable": 76293} +{"epoch": 0.66015625, "grad_norm": 0.23027750849723816, "learning_rate": 8.217303493946967e-05, "loss": 0.014154847711324692, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.01426, "step": 169, "tokens/total": 5838256, "tokens/train_per_sec_per_gpu": 26.04, "tokens/trainable": 76741} +{"epoch": 0.6640625, "grad_norm": 0.4782482087612152, "learning_rate": 8.194107981329746e-05, "loss": 0.00939749926328659, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00944, "step": 170, "tokens/total": 5876224, "tokens/train_per_sec_per_gpu": 23.15, "tokens/trainable": 77195} +{"epoch": 0.66796875, "grad_norm": 0.43363258242607117, "learning_rate": 8.170800355882518e-05, "loss": 0.006879134103655815, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.24, "memory/max_allocated (GiB)": 37.24, "ppl": 1.0069, "step": 171, "tokens/total": 5910448, "tokens/train_per_sec_per_gpu": 30.17, "tokens/trainable": 77662} +{"epoch": 0.671875, "grad_norm": 0.36711108684539795, "learning_rate": 8.147381587530713e-05, "loss": 0.026707107201218605, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.02707, "step": 172, "tokens/total": 5944256, "tokens/train_per_sec_per_gpu": 26.43, "tokens/trainable": 78116} +{"epoch": 0.67578125, "grad_norm": 0.8593010306358337, "learning_rate": 8.123852650824877e-05, "loss": 0.02497136779129505, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.02529, "step": 173, "tokens/total": 5978288, "tokens/train_per_sec_per_gpu": 28.38, "tokens/trainable": 78599} +{"epoch": 0.6796875, "grad_norm": 0.30345505475997925, "learning_rate": 8.100214524900103e-05, "loss": 0.01264738105237484, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.52, "memory/max_allocated (GiB)": 38.52, "ppl": 1.01273, "step": 174, "tokens/total": 6013680, "tokens/train_per_sec_per_gpu": 22.1, "tokens/trainable": 79030} +{"epoch": 0.68359375, "grad_norm": 0.11150506883859634, "learning_rate": 8.076468193435301e-05, "loss": 0.0038254852406680584, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.00383, "step": 175, "tokens/total": 6045072, "tokens/train_per_sec_per_gpu": 25.78, "tokens/trainable": 79454} +{"epoch": 0.6875, "grad_norm": 0.863865315914154, "learning_rate": 8.052614644612253e-05, "loss": 0.010854508727788925, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.01091, "step": 176, "tokens/total": 6082336, "tokens/train_per_sec_per_gpu": 23.23, "tokens/trainable": 79915} +{"epoch": 0.69140625, "grad_norm": 0.6354125142097473, "learning_rate": 8.028654871074489e-05, "loss": 0.010647553950548172, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.0107, "step": 177, "tokens/total": 6117024, "tokens/train_per_sec_per_gpu": 25.37, "tokens/trainable": 80356} +{"epoch": 0.6953125, "grad_norm": 0.42187952995300293, "learning_rate": 8.004589869885986e-05, "loss": 0.05054617300629616, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.49, "memory/max_allocated (GiB)": 36.49, "ppl": 1.05185, "step": 178, "tokens/total": 6149136, "tokens/train_per_sec_per_gpu": 26.52, "tokens/trainable": 80774} +{"epoch": 0.69921875, "grad_norm": 0.5059871077537537, "learning_rate": 7.980420642489674e-05, "loss": 0.01479301042854786, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.0149, "step": 179, "tokens/total": 6187776, "tokens/train_per_sec_per_gpu": 23.2, "tokens/trainable": 81247} +{"epoch": 0.703125, "grad_norm": 0.3527512848377228, "learning_rate": 7.95614819466576e-05, "loss": 0.01949215494096279, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.01968, "step": 180, "tokens/total": 6221728, "tokens/train_per_sec_per_gpu": 28.24, "tokens/trainable": 81713} +{"epoch": 0.70703125, "grad_norm": 0.4685291349887848, "learning_rate": 7.931773536489872e-05, "loss": 0.028147922828793526, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.02855, "step": 181, "tokens/total": 6255920, "tokens/train_per_sec_per_gpu": 24.83, "tokens/trainable": 82158} +{"epoch": 0.7109375, "grad_norm": 0.4511784017086029, "learning_rate": 7.907297682291035e-05, "loss": 0.02326214499771595, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.02353, "step": 182, "tokens/total": 6291104, "tokens/train_per_sec_per_gpu": 26.44, "tokens/trainable": 82621} +{"epoch": 0.71484375, "grad_norm": 0.251158207654953, "learning_rate": 7.882721650609442e-05, "loss": 0.010190552100539207, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.01024, "step": 183, "tokens/total": 6330976, "tokens/train_per_sec_per_gpu": 23.14, "tokens/trainable": 83122} +{"epoch": 0.71875, "grad_norm": 0.3741830289363861, "learning_rate": 7.85804646415409e-05, "loss": 0.028472673147916794, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.02888, "step": 184, "tokens/total": 6366160, "tokens/train_per_sec_per_gpu": 26.64, "tokens/trainable": 83578} +{"epoch": 0.72265625, "grad_norm": 0.2689945697784424, "learning_rate": 7.833273149760207e-05, "loss": 0.019114185124635696, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.0193, "step": 185, "tokens/total": 6397280, "tokens/train_per_sec_per_gpu": 28.63, "tokens/trainable": 84048} +{"epoch": 0.7265625, "grad_norm": 0.3731287121772766, "learning_rate": 7.808402738346527e-05, "loss": 0.03157763555645943, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.03208, "step": 186, "tokens/total": 6435632, "tokens/train_per_sec_per_gpu": 23.35, "tokens/trainable": 84497} +{"epoch": 0.73046875, "grad_norm": 0.2142544686794281, "learning_rate": 7.783436264872382e-05, "loss": 0.0063717421144247055, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.00639, "step": 187, "tokens/total": 6472752, "tokens/train_per_sec_per_gpu": 24.76, "tokens/trainable": 84983} +{"epoch": 0.734375, "grad_norm": 0.2808810770511627, "learning_rate": 7.758374768294647e-05, "loss": 0.009824385866522789, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.05, "memory/max_allocated (GiB)": 38.05, "ppl": 1.00987, "step": 188, "tokens/total": 6508528, "tokens/train_per_sec_per_gpu": 26.92, "tokens/trainable": 85462} +{"epoch": 0.73828125, "grad_norm": 0.23458924889564514, "learning_rate": 7.733219291524489e-05, "loss": 0.023347903043031693, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.59, "memory/max_allocated (GiB)": 37.59, "ppl": 1.02362, "step": 189, "tokens/total": 6542048, "tokens/train_per_sec_per_gpu": 27.33, "tokens/trainable": 85892} +{"epoch": 0.7421875, "grad_norm": 0.29675352573394775, "learning_rate": 7.707970881383977e-05, "loss": 0.03983164578676224, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.57, "memory/max_allocated (GiB)": 36.57, "ppl": 1.04064, "step": 190, "tokens/total": 6573792, "tokens/train_per_sec_per_gpu": 28.01, "tokens/trainable": 86341} +{"epoch": 0.74609375, "grad_norm": 0.25390133261680603, "learning_rate": 7.682630588562518e-05, "loss": 0.008475163951516151, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.00851, "step": 191, "tokens/total": 6608848, "tokens/train_per_sec_per_gpu": 24.18, "tokens/trainable": 86772} +{"epoch": 0.75, "grad_norm": 0.22719867527484894, "learning_rate": 7.657199467573129e-05, "loss": 0.02015664428472519, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.02036, "step": 192, "tokens/total": 6642560, "tokens/train_per_sec_per_gpu": 25.98, "tokens/trainable": 87244} +{"epoch": 0.75390625, "grad_norm": 0.1650657057762146, "learning_rate": 7.631678576708561e-05, "loss": 0.0025498904287815094, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00255, "step": 193, "tokens/total": 6676272, "tokens/train_per_sec_per_gpu": 28.86, "tokens/trainable": 87736} +{"epoch": 0.7578125, "grad_norm": 0.2510066628456116, "learning_rate": 7.606068977997255e-05, "loss": 0.02179408073425293, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.02203, "step": 194, "tokens/total": 6713568, "tokens/train_per_sec_per_gpu": 25.66, "tokens/trainable": 88214} +{"epoch": 0.76171875, "grad_norm": 0.41592228412628174, "learning_rate": 7.580371737159148e-05, "loss": 0.02058442123234272, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.0208, "step": 195, "tokens/total": 6749296, "tokens/train_per_sec_per_gpu": 22.88, "tokens/trainable": 88625} +{"epoch": 0.765625, "grad_norm": 0.17980848252773285, "learning_rate": 7.554587923561324e-05, "loss": 0.0018455227836966515, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00185, "step": 196, "tokens/total": 6783952, "tokens/train_per_sec_per_gpu": 24.58, "tokens/trainable": 89063} +{"epoch": 0.76953125, "grad_norm": 0.6177469491958618, "learning_rate": 7.528718610173511e-05, "loss": 0.02844497561454773, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.48, "memory/max_allocated (GiB)": 37.48, "ppl": 1.02885, "step": 197, "tokens/total": 6819152, "tokens/train_per_sec_per_gpu": 22.8, "tokens/trainable": 89484} +{"epoch": 0.7734375, "grad_norm": 0.20960140228271484, "learning_rate": 7.502764873523431e-05, "loss": 0.00851371418684721, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.00855, "step": 198, "tokens/total": 6853440, "tokens/train_per_sec_per_gpu": 27.16, "tokens/trainable": 89922} +{"epoch": 0.77734375, "grad_norm": 3.7161307334899902, "learning_rate": 7.476727793652011e-05, "loss": 0.049926385283470154, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.71, "memory/max_allocated (GiB)": 36.71, "ppl": 1.05119, "step": 199, "tokens/total": 6886272, "tokens/train_per_sec_per_gpu": 28.06, "tokens/trainable": 90401} +{"epoch": 0.78125, "grad_norm": 0.5810790061950684, "learning_rate": 7.450608454068415e-05, "loss": 0.024885006248950958, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.0252, "step": 200, "tokens/total": 6920624, "tokens/train_per_sec_per_gpu": 26.62, "tokens/trainable": 90842} +{"epoch": 0.78515625, "grad_norm": 0.7440477013587952, "learning_rate": 7.424407941704987e-05, "loss": 0.017900098115205765, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.01806, "step": 201, "tokens/total": 6957328, "tokens/train_per_sec_per_gpu": 27.26, "tokens/trainable": 91343} +{"epoch": 0.7890625, "grad_norm": 0.19900572299957275, "learning_rate": 7.398127346871986e-05, "loss": 0.009426265023648739, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.00947, "step": 202, "tokens/total": 6994048, "tokens/train_per_sec_per_gpu": 24.34, "tokens/trainable": 91798} +{"epoch": 0.79296875, "grad_norm": 0.13225416839122772, "learning_rate": 7.371767763212238e-05, "loss": 0.0024453604128211737, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.00245, "step": 203, "tokens/total": 7029040, "tokens/train_per_sec_per_gpu": 26.48, "tokens/trainable": 92251} +{"epoch": 0.796875, "grad_norm": 0.8854506015777588, "learning_rate": 7.345330287655617e-05, "loss": 0.028083467856049538, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.02848, "step": 204, "tokens/total": 7067104, "tokens/train_per_sec_per_gpu": 22.68, "tokens/trainable": 92714} +{"epoch": 0.80078125, "grad_norm": 0.39326825737953186, "learning_rate": 7.31881602037339e-05, "loss": 0.025923119857907295, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.02626, "step": 205, "tokens/total": 7099136, "tokens/train_per_sec_per_gpu": 26.45, "tokens/trainable": 93154} +{"epoch": 0.8046875, "grad_norm": 0.23902899026870728, "learning_rate": 7.29222606473245e-05, "loss": 0.009608374908566475, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.00965, "step": 206, "tokens/total": 7132208, "tokens/train_per_sec_per_gpu": 26.44, "tokens/trainable": 93599} +{"epoch": 0.80859375, "grad_norm": 0.642406165599823, "learning_rate": 7.265561527249383e-05, "loss": 0.031457167118787766, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.03196, "step": 207, "tokens/total": 7166256, "tokens/train_per_sec_per_gpu": 27.71, "tokens/trainable": 94036} +{"epoch": 0.8125, "grad_norm": 0.5161182880401611, "learning_rate": 7.238823517544436e-05, "loss": 0.014380380511283875, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.01448, "step": 208, "tokens/total": 7203152, "tokens/train_per_sec_per_gpu": 28.2, "tokens/trainable": 94506} +{"epoch": 0.81640625, "grad_norm": 0.20085826516151428, "learning_rate": 7.212013148295333e-05, "loss": 0.004629715345799923, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.00464, "step": 209, "tokens/total": 7237264, "tokens/train_per_sec_per_gpu": 29.72, "tokens/trainable": 94988} +{"epoch": 0.8203125, "grad_norm": 0.32534730434417725, "learning_rate": 7.185131535190975e-05, "loss": 0.01490139402449131, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.01501, "step": 210, "tokens/total": 7269664, "tokens/train_per_sec_per_gpu": 29.93, "tokens/trainable": 95497} +{"epoch": 0.82421875, "grad_norm": 0.28438326716423035, "learning_rate": 7.158179796885005e-05, "loss": 0.008082613348960876, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.00812, "step": 211, "tokens/total": 7304080, "tokens/train_per_sec_per_gpu": 27.45, "tokens/trainable": 95948} +{"epoch": 0.828125, "grad_norm": 0.09342312067747116, "learning_rate": 7.131159054949273e-05, "loss": 0.005221636034548283, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00524, "step": 212, "tokens/total": 7342480, "tokens/train_per_sec_per_gpu": 22.27, "tokens/trainable": 96422} +{"epoch": 0.83203125, "grad_norm": 0.2546314299106598, "learning_rate": 7.104070433827139e-05, "loss": 0.016190148890018463, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.01632, "step": 213, "tokens/total": 7378032, "tokens/train_per_sec_per_gpu": 22.91, "tokens/trainable": 96821} +{"epoch": 0.8359375, "grad_norm": 0.22069725394248962, "learning_rate": 7.076915060786705e-05, "loss": 0.0077712275087833405, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.96, "memory/max_allocated (GiB)": 37.96, "ppl": 1.0078, "step": 214, "tokens/total": 7410928, "tokens/train_per_sec_per_gpu": 24.18, "tokens/trainable": 97263} +{"epoch": 0.83984375, "grad_norm": 0.7938888669013977, "learning_rate": 7.049694065873882e-05, "loss": 0.013057075440883636, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.01314, "step": 215, "tokens/total": 7445664, "tokens/train_per_sec_per_gpu": 27.91, "tokens/trainable": 97720} +{"epoch": 0.84375, "grad_norm": 0.3827134966850281, "learning_rate": 7.022408581865382e-05, "loss": 0.008937960490584373, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.69, "memory/max_allocated (GiB)": 38.69, "ppl": 1.00898, "step": 216, "tokens/total": 7479984, "tokens/train_per_sec_per_gpu": 25.39, "tokens/trainable": 98194} +{"epoch": 0.84765625, "grad_norm": 0.1874852329492569, "learning_rate": 6.99505974422157e-05, "loss": 0.005566588137298822, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.00558, "step": 217, "tokens/total": 7514144, "tokens/train_per_sec_per_gpu": 28.35, "tokens/trainable": 98667} +{"epoch": 0.8515625, "grad_norm": 0.044190336018800735, "learning_rate": 6.967648691039213e-05, "loss": 0.0011442938121035695, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.73, "memory/max_allocated (GiB)": 36.73, "ppl": 1.00114, "step": 218, "tokens/total": 7546256, "tokens/train_per_sec_per_gpu": 27.87, "tokens/trainable": 99100} +{"epoch": 0.85546875, "grad_norm": 0.05142320692539215, "learning_rate": 6.940176563004123e-05, "loss": 0.0010466903913766146, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00105, "step": 219, "tokens/total": 7580432, "tokens/train_per_sec_per_gpu": 25.93, "tokens/trainable": 99532} +{"epoch": 0.859375, "grad_norm": 0.8978063464164734, "learning_rate": 6.912644503343682e-05, "loss": 0.014530161395668983, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.01464, "step": 220, "tokens/total": 7617904, "tokens/train_per_sec_per_gpu": 23.42, "tokens/trainable": 99970} +{"epoch": 0.86328125, "grad_norm": 0.2350224256515503, "learning_rate": 6.885053657779273e-05, "loss": 0.006572316400706768, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.00659, "step": 221, "tokens/total": 7649696, "tokens/train_per_sec_per_gpu": 28.34, "tokens/trainable": 100442} +{"epoch": 0.8671875, "grad_norm": 0.04409253969788551, "learning_rate": 6.857405174478604e-05, "loss": 0.0007015772280283272, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.0007, "step": 222, "tokens/total": 7684464, "tokens/train_per_sec_per_gpu": 24.3, "tokens/trainable": 100895} +{"epoch": 0.87109375, "grad_norm": 0.46439945697784424, "learning_rate": 6.82970020400792e-05, "loss": 0.02764192409813404, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.02803, "step": 223, "tokens/total": 7721936, "tokens/train_per_sec_per_gpu": 22.18, "tokens/trainable": 101342} +{"epoch": 0.875, "grad_norm": 0.4695981740951538, "learning_rate": 6.801939899284132e-05, "loss": 0.010310545563697815, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.34, "memory/max_allocated (GiB)": 38.34, "ppl": 1.01036, "step": 224, "tokens/total": 7758160, "tokens/train_per_sec_per_gpu": 26.51, "tokens/trainable": 101848} +{"epoch": 0.87890625, "grad_norm": 0.4125751852989197, "learning_rate": 6.774125415526827e-05, "loss": 0.010081220418214798, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.01013, "step": 225, "tokens/total": 7791216, "tokens/train_per_sec_per_gpu": 27.73, "tokens/trainable": 102319} +{"epoch": 0.8828125, "grad_norm": 0.6229428648948669, "learning_rate": 6.746257910210214e-05, "loss": 0.01719423569738865, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.12, "memory/max_allocated (GiB)": 37.12, "ppl": 1.01734, "step": 226, "tokens/total": 7824400, "tokens/train_per_sec_per_gpu": 27.96, "tokens/trainable": 102769} +{"epoch": 0.88671875, "grad_norm": 0.7988603115081787, "learning_rate": 6.718338543014937e-05, "loss": 0.010143760591745377, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 35.75, "memory/max_allocated (GiB)": 35.75, "ppl": 1.0102, "step": 227, "tokens/total": 7854672, "tokens/train_per_sec_per_gpu": 30.8, "tokens/trainable": 103219} +{"epoch": 0.890625, "grad_norm": 0.3172679841518402, "learning_rate": 6.69036847577983e-05, "loss": 0.0034145277459174395, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.00342, "step": 228, "tokens/total": 7888592, "tokens/train_per_sec_per_gpu": 26.67, "tokens/trainable": 103675} +{"epoch": 0.89453125, "grad_norm": 0.18218199908733368, "learning_rate": 6.662348872453553e-05, "loss": 0.005472020246088505, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.33, "memory/max_allocated (GiB)": 38.33, "ppl": 1.00549, "step": 229, "tokens/total": 7924848, "tokens/train_per_sec_per_gpu": 27.42, "tokens/trainable": 104136} +{"epoch": 0.8984375, "grad_norm": 0.2556588649749756, "learning_rate": 6.63428089904618e-05, "loss": 0.004923724569380283, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.00494, "step": 230, "tokens/total": 7963856, "tokens/train_per_sec_per_gpu": 24.04, "tokens/trainable": 104588} +{"epoch": 0.90234375, "grad_norm": 0.9992733597755432, "learning_rate": 6.60616572358065e-05, "loss": 0.0190751850605011, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.01926, "step": 231, "tokens/total": 7998320, "tokens/train_per_sec_per_gpu": 25.57, "tokens/trainable": 104988} +{"epoch": 0.90625, "grad_norm": 0.7078796625137329, "learning_rate": 6.578004516044172e-05, "loss": 0.012923459522426128, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.01301, "step": 232, "tokens/total": 8034096, "tokens/train_per_sec_per_gpu": 27.47, "tokens/trainable": 105455} +{"epoch": 0.91015625, "grad_norm": 0.3558887541294098, "learning_rate": 6.549798448339548e-05, "loss": 0.010494119487702847, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.59, "memory/max_allocated (GiB)": 38.59, "ppl": 1.01055, "step": 233, "tokens/total": 8071360, "tokens/train_per_sec_per_gpu": 24.94, "tokens/trainable": 105918} +{"epoch": 0.9140625, "grad_norm": 0.28383299708366394, "learning_rate": 6.521548694236384e-05, "loss": 0.005265245214104652, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00528, "step": 234, "tokens/total": 8108560, "tokens/train_per_sec_per_gpu": 23.28, "tokens/trainable": 106355} +{"epoch": 0.91796875, "grad_norm": 0.24991212785243988, "learning_rate": 6.493256429322259e-05, "loss": 0.015239162370562553, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.01536, "step": 235, "tokens/total": 8144816, "tokens/train_per_sec_per_gpu": 22.14, "tokens/trainable": 106786} +{"epoch": 0.921875, "grad_norm": 0.4087924659252167, "learning_rate": 6.464922830953799e-05, "loss": 0.01549043133854866, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.01561, "step": 236, "tokens/total": 8183680, "tokens/train_per_sec_per_gpu": 22.33, "tokens/trainable": 107219} +{"epoch": 0.92578125, "grad_norm": 0.5398070812225342, "learning_rate": 6.436549078207688e-05, "loss": 0.026708638295531273, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.02707, "step": 237, "tokens/total": 8216688, "tokens/train_per_sec_per_gpu": 28.69, "tokens/trainable": 107679} +{"epoch": 0.9296875, "grad_norm": 0.24281388521194458, "learning_rate": 6.408136351831592e-05, "loss": 0.009295647032558918, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.00934, "step": 238, "tokens/total": 8249104, "tokens/train_per_sec_per_gpu": 24.02, "tokens/trainable": 108119} +{"epoch": 0.93359375, "grad_norm": 0.26351651549339294, "learning_rate": 6.379685834195036e-05, "loss": 0.004263031762093306, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00427, "step": 239, "tokens/total": 8288528, "tokens/train_per_sec_per_gpu": 24.41, "tokens/trainable": 108626} +{"epoch": 0.9375, "grad_norm": 0.1591605842113495, "learning_rate": 6.351198709240186e-05, "loss": 0.0040164124220609665, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.00402, "step": 240, "tokens/total": 8328464, "tokens/train_per_sec_per_gpu": 21.03, "tokens/trainable": 109079} +{"epoch": 0.94140625, "grad_norm": 0.08279526233673096, "learning_rate": 6.32267616243259e-05, "loss": 0.002299531828612089, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.0023, "step": 241, "tokens/total": 8361264, "tokens/train_per_sec_per_gpu": 28.59, "tokens/trainable": 109542} +{"epoch": 0.9453125, "grad_norm": 0.38265717029571533, "learning_rate": 6.294119380711849e-05, "loss": 0.013269984163343906, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.39, "memory/max_allocated (GiB)": 38.39, "ppl": 1.01336, "step": 242, "tokens/total": 8395552, "tokens/train_per_sec_per_gpu": 23.68, "tokens/trainable": 109969} +{"epoch": 0.94921875, "grad_norm": 0.5361303687095642, "learning_rate": 6.265529552442209e-05, "loss": 0.044852182269096375, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.04587, "step": 243, "tokens/total": 8430656, "tokens/train_per_sec_per_gpu": 27.89, "tokens/trainable": 110431} +{"epoch": 0.953125, "grad_norm": 0.29637885093688965, "learning_rate": 6.236907867363127e-05, "loss": 0.005496017634868622, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.17, "memory/max_allocated (GiB)": 38.17, "ppl": 1.00551, "step": 244, "tokens/total": 8465232, "tokens/train_per_sec_per_gpu": 26.51, "tokens/trainable": 110877} +{"epoch": 0.95703125, "grad_norm": 0.25930529832839966, "learning_rate": 6.208255516539749e-05, "loss": 0.02173379808664322, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.02197, "step": 245, "tokens/total": 8497552, "tokens/train_per_sec_per_gpu": 26.44, "tokens/trainable": 111350} +{"epoch": 0.9609375, "grad_norm": 0.1488548368215561, "learning_rate": 6.179573692313344e-05, "loss": 0.006798038259148598, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.8, "memory/max_allocated (GiB)": 38.8, "ppl": 1.00682, "step": 246, "tokens/total": 8534336, "tokens/train_per_sec_per_gpu": 25.91, "tokens/trainable": 111798} +{"epoch": 0.96484375, "grad_norm": 0.20170406997203827, "learning_rate": 6.150863588251694e-05, "loss": 0.005436175037175417, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.00545, "step": 247, "tokens/total": 8574224, "tokens/train_per_sec_per_gpu": 22.54, "tokens/trainable": 112264} +{"epoch": 0.96875, "grad_norm": 0.36045247316360474, "learning_rate": 6.122126399099419e-05, "loss": 0.005049240309745073, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.00506, "step": 248, "tokens/total": 8613616, "tokens/train_per_sec_per_gpu": 26.7, "tokens/trainable": 112772} +{"epoch": 0.97265625, "grad_norm": 0.34724074602127075, "learning_rate": 6.0933633207282615e-05, "loss": 0.019147813320159912, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.01933, "step": 249, "tokens/total": 8650048, "tokens/train_per_sec_per_gpu": 27.21, "tokens/trainable": 113278} +{"epoch": 0.9765625, "grad_norm": 0.30935871601104736, "learning_rate": 6.064575550087316e-05, "loss": 0.0155494911596179, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.01567, "step": 250, "tokens/total": 8686688, "tokens/train_per_sec_per_gpu": 23.58, "tokens/trainable": 113716} +{"epoch": 0.98046875, "grad_norm": 0.20320266485214233, "learning_rate": 6.0357642851532245e-05, "loss": 0.004347924143075943, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00436, "step": 251, "tokens/total": 8723376, "tokens/train_per_sec_per_gpu": 25.61, "tokens/trainable": 114207} +{"epoch": 0.984375, "grad_norm": 0.8806257843971252, "learning_rate": 6.0069307248803294e-05, "loss": 0.01854754239320755, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.01872, "step": 252, "tokens/total": 8758736, "tokens/train_per_sec_per_gpu": 21.8, "tokens/trainable": 114628} +{"epoch": 0.98828125, "grad_norm": 0.054920535534620285, "learning_rate": 5.9780760691507635e-05, "loss": 0.001475553959608078, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.33, "memory/max_allocated (GiB)": 36.33, "ppl": 1.00148, "step": 253, "tokens/total": 8790480, "tokens/train_per_sec_per_gpu": 26.58, "tokens/trainable": 115060} +{"epoch": 0.9921875, "grad_norm": 0.3046943247318268, "learning_rate": 5.9492015187245334e-05, "loss": 0.0055283126421272755, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00554, "step": 254, "tokens/total": 8828224, "tokens/train_per_sec_per_gpu": 23.14, "tokens/trainable": 115514} +{"epoch": 0.99609375, "grad_norm": 0.5072348713874817, "learning_rate": 5.920308275189541e-05, "loss": 0.013782819733023643, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.01388, "step": 255, "tokens/total": 8860400, "tokens/train_per_sec_per_gpu": 25.32, "tokens/trainable": 115915} +{"epoch": 1.0, "grad_norm": 0.1998051106929779, "learning_rate": 5.8913975409115874e-05, "loss": 0.006404022686183453, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.00642, "step": 256, "tokens/total": 8893760, "tokens/train_per_sec_per_gpu": 25.25, "tokens/trainable": 116378} +{"epoch": 1.00390625, "grad_norm": 0.03730650246143341, "learning_rate": 5.8624705189843395e-05, "loss": 0.0009997629094868898, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 1.001, "step": 257, "tokens/total": 8930976, "tokens/train_per_sec_per_gpu": 24.44, "tokens/trainable": 116865} +{"epoch": 1.0078125, "grad_norm": 0.45488932728767395, "learning_rate": 5.833528413179249e-05, "loss": 0.00850188173353672, "memory/device_reserved (GiB)": 42.25, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00854, "step": 258, "tokens/total": 8967472, "tokens/train_per_sec_per_gpu": 27.65, "tokens/trainable": 117363} +{"epoch": 1.01171875, "grad_norm": 0.15422466397285461, "learning_rate": 5.80457242789548e-05, "loss": 0.0023311129771173, "memory/device_reserved (GiB)": 42.31, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.00233, "step": 259, "tokens/total": 9007456, "tokens/train_per_sec_per_gpu": 23.69, "tokens/trainable": 117860} +{"epoch": 1.015625, "grad_norm": 0.32911473512649536, "learning_rate": 5.77560376810977e-05, "loss": 0.016504794359207153, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 1.01664, "step": 260, "tokens/total": 9041104, "tokens/train_per_sec_per_gpu": 29.02, "tokens/trainable": 118324} +{"epoch": 1.01953125, "grad_norm": 0.054525382816791534, "learning_rate": 5.7466236393263005e-05, "loss": 0.0008097343379631639, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.00081, "step": 261, "tokens/total": 9076176, "tokens/train_per_sec_per_gpu": 25.39, "tokens/trainable": 118770} +{"epoch": 1.0234375, "grad_norm": 0.3833414912223816, "learning_rate": 5.717633247526522e-05, "loss": 0.009209632873535156, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00925, "step": 262, "tokens/total": 9107472, "tokens/train_per_sec_per_gpu": 27.64, "tokens/trainable": 119206} +{"epoch": 1.02734375, "grad_norm": 0.044901736080646515, "learning_rate": 5.688633799118971e-05, "loss": 0.0012972746044397354, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.0013, "step": 263, "tokens/total": 9139552, "tokens/train_per_sec_per_gpu": 23.13, "tokens/trainable": 119610} +{"epoch": 1.03125, "grad_norm": 0.21961720287799835, "learning_rate": 5.659626500889066e-05, "loss": 0.0028833886608481407, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00289, "step": 264, "tokens/total": 9173424, "tokens/train_per_sec_per_gpu": 25.67, "tokens/trainable": 120080} +{"epoch": 1.03515625, "grad_norm": 1.1506929397583008, "learning_rate": 5.6306125599488905e-05, "loss": 0.0012160278856754303, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.00122, "step": 265, "tokens/total": 9203984, "tokens/train_per_sec_per_gpu": 31.97, "tokens/trainable": 120519} +{"epoch": 1.0390625, "grad_norm": 0.025846758857369423, "learning_rate": 5.601593183686955e-05, "loss": 0.0006715512718074024, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.69, "memory/max_allocated (GiB)": 38.69, "ppl": 1.00067, "step": 266, "tokens/total": 9241248, "tokens/train_per_sec_per_gpu": 25.82, "tokens/trainable": 120995} +{"epoch": 1.04296875, "grad_norm": 0.5738347172737122, "learning_rate": 5.572569579717961e-05, "loss": 0.00402765953913331, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.18, "memory/max_allocated (GiB)": 39.18, "ppl": 1.00404, "step": 267, "tokens/total": 9276768, "tokens/train_per_sec_per_gpu": 24.35, "tokens/trainable": 121459} +{"epoch": 1.046875, "grad_norm": 0.136574387550354, "learning_rate": 5.543542955832538e-05, "loss": 0.0016360427252948284, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00164, "step": 268, "tokens/total": 9316528, "tokens/train_per_sec_per_gpu": 21.54, "tokens/trainable": 121902} +{"epoch": 1.05078125, "grad_norm": 0.4320968687534332, "learning_rate": 5.514514519946986e-05, "loss": 0.012674950994551182, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.01276, "step": 269, "tokens/total": 9356352, "tokens/train_per_sec_per_gpu": 25.53, "tokens/trainable": 122399} +{"epoch": 1.0546875, "grad_norm": 0.6039277911186218, "learning_rate": 5.485485480053015e-05, "loss": 0.01030784286558628, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.01036, "step": 270, "tokens/total": 9394576, "tokens/train_per_sec_per_gpu": 21.77, "tokens/trainable": 122804} +{"epoch": 1.05859375, "grad_norm": 0.4049607515335083, "learning_rate": 5.4564570441674645e-05, "loss": 0.003105469513684511, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.92, "memory/max_allocated (GiB)": 36.92, "ppl": 1.00311, "step": 271, "tokens/total": 9427840, "tokens/train_per_sec_per_gpu": 25.36, "tokens/trainable": 123244} +{"epoch": 1.0625, "grad_norm": 0.29374459385871887, "learning_rate": 5.42743042028204e-05, "loss": 0.004424907732754946, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.00443, "step": 272, "tokens/total": 9463536, "tokens/train_per_sec_per_gpu": 20.38, "tokens/trainable": 123683} +{"epoch": 1.06640625, "grad_norm": 0.5145784020423889, "learning_rate": 5.3984068163130464e-05, "loss": 0.003956031519919634, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.09, "memory/max_allocated (GiB)": 37.09, "ppl": 1.00396, "step": 273, "tokens/total": 9497248, "tokens/train_per_sec_per_gpu": 27.97, "tokens/trainable": 124153} +{"epoch": 1.0703125, "grad_norm": 0.13271473348140717, "learning_rate": 5.369387440051111e-05, "loss": 0.001466771005652845, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.00147, "step": 274, "tokens/total": 9530784, "tokens/train_per_sec_per_gpu": 25.29, "tokens/trainable": 124616} +{"epoch": 1.07421875, "grad_norm": 0.3069398105144501, "learning_rate": 5.340373499110935e-05, "loss": 0.0007813784759491682, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00078, "step": 275, "tokens/total": 9566656, "tokens/train_per_sec_per_gpu": 26.28, "tokens/trainable": 125109} +{"epoch": 1.078125, "grad_norm": 0.07996641844511032, "learning_rate": 5.3113662008810304e-05, "loss": 0.0005880194948986173, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00059, "step": 276, "tokens/total": 9606672, "tokens/train_per_sec_per_gpu": 24.85, "tokens/trainable": 125591} +{"epoch": 1.08203125, "grad_norm": 0.6134989857673645, "learning_rate": 5.282366752473479e-05, "loss": 0.0014943634159862995, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 1.0015, "step": 277, "tokens/total": 9643680, "tokens/train_per_sec_per_gpu": 24.38, "tokens/trainable": 126065} +{"epoch": 1.0859375, "grad_norm": 0.0935097336769104, "learning_rate": 5.2533763606737005e-05, "loss": 0.0006897716084495187, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00069, "step": 278, "tokens/total": 9683200, "tokens/train_per_sec_per_gpu": 19.14, "tokens/trainable": 126480} +{"epoch": 1.08984375, "grad_norm": 0.06291583925485611, "learning_rate": 5.224396231890232e-05, "loss": 0.000475407752674073, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.00048, "step": 279, "tokens/total": 9714672, "tokens/train_per_sec_per_gpu": 29.93, "tokens/trainable": 126929} +{"epoch": 1.09375, "grad_norm": 0.7417493462562561, "learning_rate": 5.195427572104522e-05, "loss": 0.04057755321264267, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.04141, "step": 280, "tokens/total": 9749104, "tokens/train_per_sec_per_gpu": 23.74, "tokens/trainable": 127366} +{"epoch": 1.09765625, "grad_norm": 0.034156277775764465, "learning_rate": 5.166471586820751e-05, "loss": 0.0003764067660085857, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.35, "memory/max_allocated (GiB)": 39.35, "ppl": 1.00038, "step": 281, "tokens/total": 9785712, "tokens/train_per_sec_per_gpu": 23.71, "tokens/trainable": 127831} +{"epoch": 1.1015625, "grad_norm": 0.14445088803768158, "learning_rate": 5.1375294810156615e-05, "loss": 0.0007302929298020899, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.00073, "step": 282, "tokens/total": 9820544, "tokens/train_per_sec_per_gpu": 25.79, "tokens/trainable": 128278} +{"epoch": 1.10546875, "grad_norm": 0.20151551067829132, "learning_rate": 5.1086024590884144e-05, "loss": 0.0009609468979761004, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00096, "step": 283, "tokens/total": 9857952, "tokens/train_per_sec_per_gpu": 24.93, "tokens/trainable": 128727} +{"epoch": 1.109375, "grad_norm": 0.4783802628517151, "learning_rate": 5.079691724810461e-05, "loss": 0.0009535664576105773, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.33, "memory/max_allocated (GiB)": 38.33, "ppl": 1.00095, "step": 284, "tokens/total": 9894128, "tokens/train_per_sec_per_gpu": 21.73, "tokens/trainable": 129162} +{"epoch": 1.11328125, "grad_norm": 0.02340111881494522, "learning_rate": 5.0507984812754684e-05, "loss": 0.00018586691294331104, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.17, "memory/max_allocated (GiB)": 37.17, "ppl": 1.00019, "step": 285, "tokens/total": 9927568, "tokens/train_per_sec_per_gpu": 28.65, "tokens/trainable": 129659} +{"epoch": 1.1171875, "grad_norm": 0.24130429327487946, "learning_rate": 5.021923930849237e-05, "loss": 0.004633938428014517, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.00464, "step": 286, "tokens/total": 9961888, "tokens/train_per_sec_per_gpu": 27.86, "tokens/trainable": 130145} +{"epoch": 1.12109375, "grad_norm": 0.015120669268071651, "learning_rate": 4.99306927511967e-05, "loss": 0.00028578005731105804, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.31, "memory/max_allocated (GiB)": 37.31, "ppl": 1.00029, "step": 287, "tokens/total": 9995696, "tokens/train_per_sec_per_gpu": 27.82, "tokens/trainable": 130585} +{"epoch": 1.125, "grad_norm": 0.49247223138809204, "learning_rate": 4.964235714846775e-05, "loss": 0.00945587269961834, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.72, "memory/max_allocated (GiB)": 36.72, "ppl": 1.0095, "step": 288, "tokens/total": 10028000, "tokens/train_per_sec_per_gpu": 27.12, "tokens/trainable": 131005} +{"epoch": 1.12890625, "grad_norm": 0.01616351306438446, "learning_rate": 4.9354244499126866e-05, "loss": 0.00017355597810819745, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.16, "memory/max_allocated (GiB)": 36.16, "ppl": 1.00017, "step": 289, "tokens/total": 10059728, "tokens/train_per_sec_per_gpu": 29.3, "tokens/trainable": 131471} +{"epoch": 1.1328125, "grad_norm": 0.012034399434924126, "learning_rate": 4.90663667927174e-05, "loss": 0.00032293249387294054, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.00032, "step": 290, "tokens/total": 10092576, "tokens/train_per_sec_per_gpu": 28.82, "tokens/trainable": 131920} +{"epoch": 1.13671875, "grad_norm": 0.3499198257923126, "learning_rate": 4.877873600900581e-05, "loss": 0.005165621172636747, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.00518, "step": 291, "tokens/total": 10128608, "tokens/train_per_sec_per_gpu": 21.67, "tokens/trainable": 132379} +{"epoch": 1.140625, "grad_norm": 0.03324860706925392, "learning_rate": 4.849136411748306e-05, "loss": 0.0003714319027494639, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.00037, "step": 292, "tokens/total": 10166192, "tokens/train_per_sec_per_gpu": 24.76, "tokens/trainable": 132851} +{"epoch": 1.14453125, "grad_norm": 0.2943325936794281, "learning_rate": 4.8204263076866574e-05, "loss": 0.006633462850004435, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00666, "step": 293, "tokens/total": 10201840, "tokens/train_per_sec_per_gpu": 29.55, "tokens/trainable": 133351} +{"epoch": 1.1484375, "grad_norm": 0.03617064654827118, "learning_rate": 4.791744483460251e-05, "loss": 0.0005000036326237023, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.0005, "step": 294, "tokens/total": 10236048, "tokens/train_per_sec_per_gpu": 24.2, "tokens/trainable": 133784} +{"epoch": 1.15234375, "grad_norm": 0.9386563897132874, "learning_rate": 4.7630921326368736e-05, "loss": 0.004658706020563841, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.00467, "step": 295, "tokens/total": 10274688, "tokens/train_per_sec_per_gpu": 20.0, "tokens/trainable": 134209} +{"epoch": 1.15625, "grad_norm": 0.035437341779470444, "learning_rate": 4.7344704475577916e-05, "loss": 0.00045570015208795667, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.66, "memory/max_allocated (GiB)": 35.66, "ppl": 1.00046, "step": 296, "tokens/total": 10304576, "tokens/train_per_sec_per_gpu": 30.24, "tokens/trainable": 134645} +{"epoch": 1.16015625, "grad_norm": 0.2838389277458191, "learning_rate": 4.705880619288153e-05, "loss": 0.0037837009876966476, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.21, "memory/max_allocated (GiB)": 38.21, "ppl": 1.00379, "step": 297, "tokens/total": 10340720, "tokens/train_per_sec_per_gpu": 21.99, "tokens/trainable": 135047} +{"epoch": 1.1640625, "grad_norm": 0.06257587671279907, "learning_rate": 4.677323837567412e-05, "loss": 0.0012006901670247316, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.0012, "step": 298, "tokens/total": 10373376, "tokens/train_per_sec_per_gpu": 25.33, "tokens/trainable": 135474} +{"epoch": 1.16796875, "grad_norm": 0.03270324319601059, "learning_rate": 4.6488012907598146e-05, "loss": 0.0005875864299014211, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00059, "step": 299, "tokens/total": 10406128, "tokens/train_per_sec_per_gpu": 27.94, "tokens/trainable": 135948} +{"epoch": 1.171875, "grad_norm": 0.3678191900253296, "learning_rate": 4.620314165804964e-05, "loss": 0.0029051918536424637, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.3, "memory/max_allocated (GiB)": 38.3, "ppl": 1.00291, "step": 300, "tokens/total": 10441792, "tokens/train_per_sec_per_gpu": 25.66, "tokens/trainable": 136421} +{"epoch": 1.17578125, "grad_norm": 0.4472847282886505, "learning_rate": 4.591863648168407e-05, "loss": 0.008889881893992424, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00893, "step": 301, "tokens/total": 10476128, "tokens/train_per_sec_per_gpu": 26.33, "tokens/trainable": 136886} +{"epoch": 1.1796875, "grad_norm": 0.057608745992183685, "learning_rate": 4.5634509217923135e-05, "loss": 0.0007832004921510816, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00078, "step": 302, "tokens/total": 10509472, "tokens/train_per_sec_per_gpu": 25.77, "tokens/trainable": 137321} +{"epoch": 1.18359375, "grad_norm": 0.4318855404853821, "learning_rate": 4.535077169046201e-05, "loss": 0.010206662118434906, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.01026, "step": 303, "tokens/total": 10546032, "tokens/train_per_sec_per_gpu": 25.13, "tokens/trainable": 137758} +{"epoch": 1.1875, "grad_norm": 0.9046695828437805, "learning_rate": 4.506743570677743e-05, "loss": 0.01944143697619438, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.01963, "step": 304, "tokens/total": 10576624, "tokens/train_per_sec_per_gpu": 31.27, "tokens/trainable": 138190} +{"epoch": 1.19140625, "grad_norm": 0.08319805562496185, "learning_rate": 4.478451305763618e-05, "loss": 0.0012773324269801378, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.95, "memory/max_allocated (GiB)": 36.95, "ppl": 1.00128, "step": 305, "tokens/total": 10609760, "tokens/train_per_sec_per_gpu": 30.53, "tokens/trainable": 138650} +{"epoch": 1.1953125, "grad_norm": 0.24955883622169495, "learning_rate": 4.450201551660454e-05, "loss": 0.0025195793714374304, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.51, "memory/max_allocated (GiB)": 36.51, "ppl": 1.00252, "step": 306, "tokens/total": 10641472, "tokens/train_per_sec_per_gpu": 26.2, "tokens/trainable": 139082} +{"epoch": 1.19921875, "grad_norm": 0.4230232834815979, "learning_rate": 4.4219954839558276e-05, "loss": 0.002490880899131298, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00249, "step": 307, "tokens/total": 10676160, "tokens/train_per_sec_per_gpu": 25.35, "tokens/trainable": 139523} +{"epoch": 1.203125, "grad_norm": 0.4663286507129669, "learning_rate": 4.393834276419352e-05, "loss": 0.0056536151096224785, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.83, "memory/max_allocated (GiB)": 35.83, "ppl": 1.00567, "step": 308, "tokens/total": 10706384, "tokens/train_per_sec_per_gpu": 32.26, "tokens/trainable": 140010} +{"epoch": 1.20703125, "grad_norm": 1.8470202684402466, "learning_rate": 4.36571910095382e-05, "loss": 0.029539866372942924, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.02998, "step": 309, "tokens/total": 10744496, "tokens/train_per_sec_per_gpu": 25.42, "tokens/trainable": 140454} +{"epoch": 1.2109375, "grad_norm": 0.7546923160552979, "learning_rate": 4.337651127546448e-05, "loss": 0.007085847202688456, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00711, "step": 310, "tokens/total": 10780176, "tokens/train_per_sec_per_gpu": 27.51, "tokens/trainable": 140905} +{"epoch": 1.21484375, "grad_norm": 0.34346264600753784, "learning_rate": 4.3096315242201736e-05, "loss": 0.017029354348778725, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.61, "memory/max_allocated (GiB)": 36.61, "ppl": 1.01718, "step": 311, "tokens/total": 10813136, "tokens/train_per_sec_per_gpu": 26.1, "tokens/trainable": 141339} +{"epoch": 1.21875, "grad_norm": 0.0881718248128891, "learning_rate": 4.2816614569850635e-05, "loss": 0.0006188676343299448, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00062, "step": 312, "tokens/total": 10848912, "tokens/train_per_sec_per_gpu": 24.17, "tokens/trainable": 141778} +{"epoch": 1.22265625, "grad_norm": 0.3739392161369324, "learning_rate": 4.2537420897897864e-05, "loss": 0.012386164627969265, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.01246, "step": 313, "tokens/total": 10884320, "tokens/train_per_sec_per_gpu": 26.86, "tokens/trainable": 142255} +{"epoch": 1.2265625, "grad_norm": 0.1536875218153, "learning_rate": 4.225874584473174e-05, "loss": 0.0030009972397238016, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.00301, "step": 314, "tokens/total": 10917152, "tokens/train_per_sec_per_gpu": 25.85, "tokens/trainable": 142693} +{"epoch": 1.23046875, "grad_norm": 0.07279892265796661, "learning_rate": 4.19806010071587e-05, "loss": 0.001885135192424059, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.00189, "step": 315, "tokens/total": 10955024, "tokens/train_per_sec_per_gpu": 24.27, "tokens/trainable": 143157} +{"epoch": 1.234375, "grad_norm": 0.330360472202301, "learning_rate": 4.170299795992081e-05, "loss": 0.007781689520925283, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.00781, "step": 316, "tokens/total": 10988416, "tokens/train_per_sec_per_gpu": 27.0, "tokens/trainable": 143603} +{"epoch": 1.23828125, "grad_norm": 0.14742439985275269, "learning_rate": 4.142594825521398e-05, "loss": 0.0031077000312507153, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.00311, "step": 317, "tokens/total": 11027712, "tokens/train_per_sec_per_gpu": 25.88, "tokens/trainable": 144084} +{"epoch": 1.2421875, "grad_norm": 0.17512258887290955, "learning_rate": 4.114946342220728e-05, "loss": 0.0038138930685818195, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.00382, "step": 318, "tokens/total": 11063920, "tokens/train_per_sec_per_gpu": 24.0, "tokens/trainable": 144532} +{"epoch": 1.24609375, "grad_norm": 0.16368696093559265, "learning_rate": 4.087355496656321e-05, "loss": 0.0027000252157449722, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.0027, "step": 319, "tokens/total": 11097952, "tokens/train_per_sec_per_gpu": 28.37, "tokens/trainable": 145004} +{"epoch": 1.25, "grad_norm": 0.1675279587507248, "learning_rate": 4.05982343699588e-05, "loss": 0.0024542133323848248, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00246, "step": 320, "tokens/total": 11137168, "tokens/train_per_sec_per_gpu": 25.05, "tokens/trainable": 145498} +{"epoch": 1.25390625, "grad_norm": 0.08679081499576569, "learning_rate": 4.0323513089607876e-05, "loss": 0.0020433785393834114, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.00205, "step": 321, "tokens/total": 11172224, "tokens/train_per_sec_per_gpu": 24.26, "tokens/trainable": 145965} +{"epoch": 1.2578125, "grad_norm": 0.036888133734464645, "learning_rate": 4.004940255778431e-05, "loss": 0.000723904580809176, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00072, "step": 322, "tokens/total": 11210832, "tokens/train_per_sec_per_gpu": 23.88, "tokens/trainable": 146409} +{"epoch": 1.26171875, "grad_norm": 0.05599164590239525, "learning_rate": 3.977591418134619e-05, "loss": 0.0009832835057750344, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.00098, "step": 323, "tokens/total": 11245984, "tokens/train_per_sec_per_gpu": 25.8, "tokens/trainable": 146849} +{"epoch": 1.265625, "grad_norm": 0.14826519787311554, "learning_rate": 3.95030593412612e-05, "loss": 0.003416655119508505, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.00342, "step": 324, "tokens/total": 11281360, "tokens/train_per_sec_per_gpu": 24.54, "tokens/trainable": 147272} +{"epoch": 1.26953125, "grad_norm": 0.25468140840530396, "learning_rate": 3.923084939213296e-05, "loss": 0.003015931462869048, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.88, "memory/max_allocated (GiB)": 35.88, "ppl": 1.00302, "step": 325, "tokens/total": 11312176, "tokens/train_per_sec_per_gpu": 27.66, "tokens/trainable": 147694} +{"epoch": 1.2734375, "grad_norm": 0.24569399654865265, "learning_rate": 3.895929566172861e-05, "loss": 0.007235124707221985, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.00726, "step": 326, "tokens/total": 11348192, "tokens/train_per_sec_per_gpu": 25.17, "tokens/trainable": 148158} +{"epoch": 1.27734375, "grad_norm": 0.10041549056768417, "learning_rate": 3.868840945050728e-05, "loss": 0.0013361063320189714, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.05, "memory/max_allocated (GiB)": 39.05, "ppl": 1.00134, "step": 327, "tokens/total": 11386704, "tokens/train_per_sec_per_gpu": 21.52, "tokens/trainable": 148590} +{"epoch": 1.28125, "grad_norm": 0.030106062069535255, "learning_rate": 3.841820203114995e-05, "loss": 0.0006156168528832495, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.00062, "step": 328, "tokens/total": 11418448, "tokens/train_per_sec_per_gpu": 29.07, "tokens/trainable": 149037} +{"epoch": 1.28515625, "grad_norm": 0.06790395081043243, "learning_rate": 3.814868464809027e-05, "loss": 0.0009537062142044306, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.00095, "step": 329, "tokens/total": 11450304, "tokens/train_per_sec_per_gpu": 25.79, "tokens/trainable": 149439} +{"epoch": 1.2890625, "grad_norm": 0.5413724184036255, "learning_rate": 3.787986851704667e-05, "loss": 0.008062348701059818, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.00809, "step": 330, "tokens/total": 11484048, "tokens/train_per_sec_per_gpu": 25.79, "tokens/trainable": 149869} +{"epoch": 1.29296875, "grad_norm": 0.23413608968257904, "learning_rate": 3.7611764824555654e-05, "loss": 0.0017974128713831306, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.0018, "step": 331, "tokens/total": 11516032, "tokens/train_per_sec_per_gpu": 30.89, "tokens/trainable": 150343} +{"epoch": 1.296875, "grad_norm": 0.02074236050248146, "learning_rate": 3.734438472750619e-05, "loss": 0.00029915565392002463, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.0003, "step": 332, "tokens/total": 11555968, "tokens/train_per_sec_per_gpu": 23.1, "tokens/trainable": 150814} +{"epoch": 1.30078125, "grad_norm": 0.43286392092704773, "learning_rate": 3.707773935267552e-05, "loss": 0.00025226682191714644, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.46, "memory/max_allocated (GiB)": 38.46, "ppl": 1.00025, "step": 333, "tokens/total": 11592880, "tokens/train_per_sec_per_gpu": 26.87, "tokens/trainable": 151300} +{"epoch": 1.3046875, "grad_norm": 0.017616858705878258, "learning_rate": 3.68118397962661e-05, "loss": 0.00024493536329828203, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.00024, "step": 334, "tokens/total": 11627792, "tokens/train_per_sec_per_gpu": 23.25, "tokens/trainable": 151708} +{"epoch": 1.30859375, "grad_norm": 0.5978325605392456, "learning_rate": 3.654669712344384e-05, "loss": 0.0005669955862686038, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 1.00057, "step": 335, "tokens/total": 11658832, "tokens/train_per_sec_per_gpu": 31.6, "tokens/trainable": 152164} +{"epoch": 1.3125, "grad_norm": 0.04935074225068092, "learning_rate": 3.628232236787763e-05, "loss": 0.0004531795275397599, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.96, "memory/max_allocated (GiB)": 37.96, "ppl": 1.00045, "step": 336, "tokens/total": 11694464, "tokens/train_per_sec_per_gpu": 22.65, "tokens/trainable": 152603} +{"epoch": 1.31640625, "grad_norm": 0.2809937000274658, "learning_rate": 3.6018726531280144e-05, "loss": 0.006521400064229965, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00654, "step": 337, "tokens/total": 11732176, "tokens/train_per_sec_per_gpu": 28.25, "tokens/trainable": 153094} +{"epoch": 1.3203125, "grad_norm": 0.3971089720726013, "learning_rate": 3.575592058295017e-05, "loss": 0.006180992349982262, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.0062, "step": 338, "tokens/total": 11766032, "tokens/train_per_sec_per_gpu": 28.47, "tokens/trainable": 153573} +{"epoch": 1.32421875, "grad_norm": 0.22718919813632965, "learning_rate": 3.549391545931585e-05, "loss": 0.001392571721225977, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00139, "step": 339, "tokens/total": 11800640, "tokens/train_per_sec_per_gpu": 27.71, "tokens/trainable": 154033} +{"epoch": 1.328125, "grad_norm": 0.2425171583890915, "learning_rate": 3.5232722063479914e-05, "loss": 0.0008291819831356406, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.00083, "step": 340, "tokens/total": 11834352, "tokens/train_per_sec_per_gpu": 24.17, "tokens/trainable": 154482} +{"epoch": 1.33203125, "grad_norm": 0.06735854595899582, "learning_rate": 3.49723512647657e-05, "loss": 0.0005159289576113224, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.00052, "step": 341, "tokens/total": 11873056, "tokens/train_per_sec_per_gpu": 24.75, "tokens/trainable": 154982} +{"epoch": 1.3359375, "grad_norm": 0.10846655815839767, "learning_rate": 3.471281389826491e-05, "loss": 0.0009738975204527378, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.00097, "step": 342, "tokens/total": 11904192, "tokens/train_per_sec_per_gpu": 25.97, "tokens/trainable": 155387} +{"epoch": 1.33984375, "grad_norm": 0.2625194191932678, "learning_rate": 3.4454120764386764e-05, "loss": 0.0021579167805612087, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.66, "memory/max_allocated (GiB)": 38.66, "ppl": 1.00216, "step": 343, "tokens/total": 11942384, "tokens/train_per_sec_per_gpu": 24.31, "tokens/trainable": 155842} +{"epoch": 1.34375, "grad_norm": 0.5815898776054382, "learning_rate": 3.4196282628408526e-05, "loss": 0.0037043849006295204, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.00371, "step": 344, "tokens/total": 11976624, "tokens/train_per_sec_per_gpu": 25.32, "tokens/trainable": 156287} +{"epoch": 1.34765625, "grad_norm": 0.3921923339366913, "learning_rate": 3.3939310220027456e-05, "loss": 0.006940613035112619, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00696, "step": 345, "tokens/total": 12010768, "tokens/train_per_sec_per_gpu": 28.85, "tokens/trainable": 156782} +{"epoch": 1.3515625, "grad_norm": 0.012506034225225449, "learning_rate": 3.3683214232914404e-05, "loss": 0.00011589015775825828, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.00012, "step": 346, "tokens/total": 12046224, "tokens/train_per_sec_per_gpu": 26.15, "tokens/trainable": 157255} +{"epoch": 1.35546875, "grad_norm": 0.02401546947658062, "learning_rate": 3.342800532426873e-05, "loss": 0.00016356179548893124, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.00016, "step": 347, "tokens/total": 12082160, "tokens/train_per_sec_per_gpu": 25.43, "tokens/trainable": 157721} +{"epoch": 1.359375, "grad_norm": 0.03607543185353279, "learning_rate": 3.317369411437484e-05, "loss": 0.00024563961778767407, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.00025, "step": 348, "tokens/total": 12118128, "tokens/train_per_sec_per_gpu": 26.15, "tokens/trainable": 158189} +{"epoch": 1.36328125, "grad_norm": 0.10867515951395035, "learning_rate": 3.292029118616024e-05, "loss": 0.0006630957941524684, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.00066, "step": 349, "tokens/total": 12151264, "tokens/train_per_sec_per_gpu": 25.6, "tokens/trainable": 158612} +{"epoch": 1.3671875, "grad_norm": 0.08114062994718552, "learning_rate": 3.266780708475511e-05, "loss": 0.00046399515122175217, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00046, "step": 350, "tokens/total": 12186800, "tokens/train_per_sec_per_gpu": 25.06, "tokens/trainable": 159090} +{"epoch": 1.37109375, "grad_norm": 0.14219930768013, "learning_rate": 3.241625231705354e-05, "loss": 0.0012002713046967983, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.0012, "step": 351, "tokens/total": 12222944, "tokens/train_per_sec_per_gpu": 25.01, "tokens/trainable": 159549} +{"epoch": 1.375, "grad_norm": 0.043079208582639694, "learning_rate": 3.216563735127618e-05, "loss": 0.0004916180623695254, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.15, "memory/max_allocated (GiB)": 39.15, "ppl": 1.00049, "step": 352, "tokens/total": 12258208, "tokens/train_per_sec_per_gpu": 25.55, "tokens/trainable": 160026} +{"epoch": 1.37890625, "grad_norm": 0.3331087827682495, "learning_rate": 3.191597261653475e-05, "loss": 0.015897909179329872, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.01602, "step": 353, "tokens/total": 12295024, "tokens/train_per_sec_per_gpu": 23.71, "tokens/trainable": 160487} +{"epoch": 1.3828125, "grad_norm": 0.608618438243866, "learning_rate": 3.166726850239794e-05, "loss": 0.004356639925390482, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00437, "step": 354, "tokens/total": 12328064, "tokens/train_per_sec_per_gpu": 27.43, "tokens/trainable": 160935} +{"epoch": 1.38671875, "grad_norm": 0.16296786069869995, "learning_rate": 3.141953535845912e-05, "loss": 0.0002483525313436985, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.34, "memory/max_allocated (GiB)": 36.34, "ppl": 1.00025, "step": 355, "tokens/total": 12359504, "tokens/train_per_sec_per_gpu": 30.38, "tokens/trainable": 161385} +{"epoch": 1.390625, "grad_norm": 0.4264850318431854, "learning_rate": 3.11727834939056e-05, "loss": 0.005536101758480072, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00555, "step": 356, "tokens/total": 12391824, "tokens/train_per_sec_per_gpu": 25.48, "tokens/trainable": 161849} +{"epoch": 1.39453125, "grad_norm": 0.4497811198234558, "learning_rate": 3.092702317708967e-05, "loss": 0.006905331742018461, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.00693, "step": 357, "tokens/total": 12425408, "tokens/train_per_sec_per_gpu": 28.93, "tokens/trainable": 162306} +{"epoch": 1.3984375, "grad_norm": 0.04959588497877121, "learning_rate": 3.0682264635101276e-05, "loss": 0.00030171056278049946, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.0003, "step": 358, "tokens/total": 12458160, "tokens/train_per_sec_per_gpu": 28.25, "tokens/trainable": 162759} +{"epoch": 1.40234375, "grad_norm": 0.28222161531448364, "learning_rate": 3.0438518053342407e-05, "loss": 0.0008773101726546884, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00088, "step": 359, "tokens/total": 12497104, "tokens/train_per_sec_per_gpu": 22.12, "tokens/trainable": 163230} +{"epoch": 1.40625, "grad_norm": 0.39492106437683105, "learning_rate": 3.0195793575103266e-05, "loss": 0.003389982972294092, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.0034, "step": 360, "tokens/total": 12536064, "tokens/train_per_sec_per_gpu": 21.26, "tokens/trainable": 163694} +{"epoch": 1.41015625, "grad_norm": 0.4113619923591614, "learning_rate": 2.9954101301140146e-05, "loss": 0.0033351497258991003, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.57, "memory/max_allocated (GiB)": 37.57, "ppl": 1.00334, "step": 361, "tokens/total": 12571488, "tokens/train_per_sec_per_gpu": 24.58, "tokens/trainable": 164144} +{"epoch": 1.4140625, "grad_norm": 0.13726574182510376, "learning_rate": 2.9713451289255123e-05, "loss": 0.0009585840743966401, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.63, "memory/max_allocated (GiB)": 36.63, "ppl": 1.00096, "step": 362, "tokens/total": 12600864, "tokens/train_per_sec_per_gpu": 25.54, "tokens/trainable": 164585} +{"epoch": 1.41796875, "grad_norm": 0.03157695755362511, "learning_rate": 2.9473853553877484e-05, "loss": 0.0004358833539299667, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.00044, "step": 363, "tokens/total": 12635024, "tokens/train_per_sec_per_gpu": 27.25, "tokens/trainable": 165059} +{"epoch": 1.421875, "grad_norm": 0.5531105995178223, "learning_rate": 2.9235318065647e-05, "loss": 0.0021342705003917217, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.00214, "step": 364, "tokens/total": 12674736, "tokens/train_per_sec_per_gpu": 24.91, "tokens/trainable": 165519} +{"epoch": 1.42578125, "grad_norm": 0.04491281509399414, "learning_rate": 2.8997854750998964e-05, "loss": 0.00048067347961477935, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.46, "memory/max_allocated (GiB)": 39.46, "ppl": 1.00048, "step": 365, "tokens/total": 12714256, "tokens/train_per_sec_per_gpu": 23.15, "tokens/trainable": 165984} +{"epoch": 1.4296875, "grad_norm": 0.05382941663265228, "learning_rate": 2.8761473491751258e-05, "loss": 0.000765543314628303, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.00077, "step": 366, "tokens/total": 12747504, "tokens/train_per_sec_per_gpu": 26.89, "tokens/trainable": 166419} +{"epoch": 1.43359375, "grad_norm": 0.15113267302513123, "learning_rate": 2.8526184124692883e-05, "loss": 0.001964733935892582, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00197, "step": 367, "tokens/total": 12783872, "tokens/train_per_sec_per_gpu": 20.38, "tokens/trainable": 166857} +{"epoch": 1.4375, "grad_norm": 0.6958493590354919, "learning_rate": 2.829199644117484e-05, "loss": 0.008635620586574078, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.00867, "step": 368, "tokens/total": 12817856, "tokens/train_per_sec_per_gpu": 24.35, "tokens/trainable": 167267} +{"epoch": 1.44140625, "grad_norm": 0.16988857090473175, "learning_rate": 2.8058920186702553e-05, "loss": 0.000553770107217133, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00055, "step": 369, "tokens/total": 12855472, "tokens/train_per_sec_per_gpu": 24.97, "tokens/trainable": 167740} +{"epoch": 1.4453125, "grad_norm": 0.06455954164266586, "learning_rate": 2.782696506053033e-05, "loss": 0.0006248489953577518, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.00063, "step": 370, "tokens/total": 12893344, "tokens/train_per_sec_per_gpu": 22.35, "tokens/trainable": 168172} +{"epoch": 1.44921875, "grad_norm": 0.20390605926513672, "learning_rate": 2.7596140715257824e-05, "loss": 0.0012389844050630927, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00124, "step": 371, "tokens/total": 12928512, "tokens/train_per_sec_per_gpu": 25.71, "tokens/trainable": 168637} +{"epoch": 1.453125, "grad_norm": 0.11794286966323853, "learning_rate": 2.7366456756428184e-05, "loss": 0.0011300500482320786, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 1.00113, "step": 372, "tokens/total": 12963120, "tokens/train_per_sec_per_gpu": 27.19, "tokens/trainable": 169094} +{"epoch": 1.45703125, "grad_norm": 0.2841357886791229, "learning_rate": 2.7137922742128486e-05, "loss": 0.0012922139139845967, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00129, "step": 373, "tokens/total": 12997648, "tokens/train_per_sec_per_gpu": 27.04, "tokens/trainable": 169593} +{"epoch": 1.4609375, "grad_norm": 0.43897634744644165, "learning_rate": 2.691054818259188e-05, "loss": 0.0016004163771867752, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.0016, "step": 374, "tokens/total": 13032496, "tokens/train_per_sec_per_gpu": 26.3, "tokens/trainable": 170066} +{"epoch": 1.46484375, "grad_norm": 0.15533499419689178, "learning_rate": 2.6684342539801933e-05, "loss": 0.0006196652539074421, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.56, "memory/max_allocated (GiB)": 36.56, "ppl": 1.00062, "step": 375, "tokens/total": 13064016, "tokens/train_per_sec_per_gpu": 28.85, "tokens/trainable": 170518} +{"epoch": 1.46875, "grad_norm": 0.10190890729427338, "learning_rate": 2.645931522709877e-05, "loss": 0.00015437352703884244, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.00015, "step": 376, "tokens/total": 13099408, "tokens/train_per_sec_per_gpu": 29.05, "tokens/trainable": 170998} +{"epoch": 1.47265625, "grad_norm": 0.028352832421660423, "learning_rate": 2.6235475608787365e-05, "loss": 0.0002824489201884717, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.00028, "step": 377, "tokens/total": 13135088, "tokens/train_per_sec_per_gpu": 25.54, "tokens/trainable": 171459} +{"epoch": 1.4765625, "grad_norm": 0.06109277904033661, "learning_rate": 2.6012832999747916e-05, "loss": 0.0007409091340377927, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.00074, "step": 378, "tokens/total": 13166208, "tokens/train_per_sec_per_gpu": 29.27, "tokens/trainable": 171905} +{"epoch": 1.48046875, "grad_norm": 1.3590779304504395, "learning_rate": 2.579139666504821e-05, "loss": 0.024090543389320374, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.02438, "step": 379, "tokens/total": 13206192, "tokens/train_per_sec_per_gpu": 22.05, "tokens/trainable": 172351} +{"epoch": 1.484375, "grad_norm": 1.0543475151062012, "learning_rate": 2.557117581955798e-05, "loss": 0.004376702010631561, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00439, "step": 380, "tokens/total": 13240032, "tokens/train_per_sec_per_gpu": 26.92, "tokens/trainable": 172817} +{"epoch": 1.48828125, "grad_norm": 0.6496241092681885, "learning_rate": 2.5352179627565532e-05, "loss": 0.0070019736886024475, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.00703, "step": 381, "tokens/total": 13274144, "tokens/train_per_sec_per_gpu": 25.72, "tokens/trainable": 173258} +{"epoch": 1.4921875, "grad_norm": 0.15332581102848053, "learning_rate": 2.5134417202396277e-05, "loss": 0.0011678808368742466, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.6, "memory/max_allocated (GiB)": 38.6, "ppl": 1.00117, "step": 382, "tokens/total": 13308128, "tokens/train_per_sec_per_gpu": 24.01, "tokens/trainable": 173665} +{"epoch": 1.49609375, "grad_norm": 0.07786227017641068, "learning_rate": 2.491789760603361e-05, "loss": 0.0006473599933087826, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00065, "step": 383, "tokens/total": 13344544, "tokens/train_per_sec_per_gpu": 23.22, "tokens/trainable": 174095} +{"epoch": 1.5, "grad_norm": 0.15198393166065216, "learning_rate": 2.4702629848741764e-05, "loss": 0.0008525278535671532, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.00085, "step": 384, "tokens/total": 13378592, "tokens/train_per_sec_per_gpu": 26.55, "tokens/trainable": 174550} +{"epoch": 1.50390625, "grad_norm": 0.0474541112780571, "learning_rate": 2.4488622888690785e-05, "loss": 0.0007291243528015912, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.04, "memory/max_allocated (GiB)": 37.04, "ppl": 1.00073, "step": 385, "tokens/total": 13412048, "tokens/train_per_sec_per_gpu": 27.01, "tokens/trainable": 174990} +{"epoch": 1.5078125, "grad_norm": 0.014982759021222591, "learning_rate": 2.427588563158384e-05, "loss": 0.00019152289314661175, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00019, "step": 386, "tokens/total": 13446384, "tokens/train_per_sec_per_gpu": 26.19, "tokens/trainable": 175476} +{"epoch": 1.51171875, "grad_norm": 0.18410751223564148, "learning_rate": 2.406442693028651e-05, "loss": 0.0018221623031422496, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.63, "memory/max_allocated (GiB)": 38.63, "ppl": 1.00182, "step": 387, "tokens/total": 13478320, "tokens/train_per_sec_per_gpu": 25.47, "tokens/trainable": 175897} +{"epoch": 1.515625, "grad_norm": 0.09780292212963104, "learning_rate": 2.3854255584458547e-05, "loss": 0.000494503416121006, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.00049, "step": 388, "tokens/total": 13516912, "tokens/train_per_sec_per_gpu": 21.61, "tokens/trainable": 176324} +{"epoch": 1.51953125, "grad_norm": 0.053283389657735825, "learning_rate": 2.3645380340187508e-05, "loss": 0.00045012671034783125, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00045, "step": 389, "tokens/total": 13552592, "tokens/train_per_sec_per_gpu": 26.11, "tokens/trainable": 176805} +{"epoch": 1.5234375, "grad_norm": 0.26869332790374756, "learning_rate": 2.3437809889624914e-05, "loss": 0.002504770178347826, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.54, "memory/max_allocated (GiB)": 36.54, "ppl": 1.00251, "step": 390, "tokens/total": 13583856, "tokens/train_per_sec_per_gpu": 29.23, "tokens/trainable": 177256} +{"epoch": 1.52734375, "grad_norm": 0.037682246416807175, "learning_rate": 2.3231552870624487e-05, "loss": 0.00029092011391185224, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.47, "memory/max_allocated (GiB)": 38.47, "ppl": 1.00029, "step": 391, "tokens/total": 13614864, "tokens/train_per_sec_per_gpu": 29.14, "tokens/trainable": 177682} +{"epoch": 1.53125, "grad_norm": 0.17027722299098969, "learning_rate": 2.3026617866382657e-05, "loss": 0.0012580353068187833, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.53, "memory/max_allocated (GiB)": 36.53, "ppl": 1.00126, "step": 392, "tokens/total": 13647616, "tokens/train_per_sec_per_gpu": 29.87, "tokens/trainable": 178179} +{"epoch": 1.53515625, "grad_norm": 0.6011863350868225, "learning_rate": 2.2823013405081507e-05, "loss": 0.003570335451513529, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.00358, "step": 393, "tokens/total": 13685696, "tokens/train_per_sec_per_gpu": 26.89, "tokens/trainable": 178676} +{"epoch": 1.5390625, "grad_norm": 0.029395056888461113, "learning_rate": 2.2620747959533722e-05, "loss": 0.00017119674885179847, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.76, "memory/max_allocated (GiB)": 36.76, "ppl": 1.00017, "step": 394, "tokens/total": 13718768, "tokens/train_per_sec_per_gpu": 30.7, "tokens/trainable": 179173} +{"epoch": 1.54296875, "grad_norm": 0.09379614144563675, "learning_rate": 2.2419829946830123e-05, "loss": 0.001044452073983848, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.00104, "step": 395, "tokens/total": 13754960, "tokens/train_per_sec_per_gpu": 24.22, "tokens/trainable": 179624} +{"epoch": 1.546875, "grad_norm": 0.7901126742362976, "learning_rate": 2.2220267727989325e-05, "loss": 0.004088571760803461, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.0041, "step": 396, "tokens/total": 13789104, "tokens/train_per_sec_per_gpu": 25.81, "tokens/trainable": 180078} +{"epoch": 1.55078125, "grad_norm": 0.09460161626338959, "learning_rate": 2.202206960760984e-05, "loss": 0.0010062274523079395, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.00101, "step": 397, "tokens/total": 13827248, "tokens/train_per_sec_per_gpu": 24.74, "tokens/trainable": 180585} +{"epoch": 1.5546875, "grad_norm": 0.337944358587265, "learning_rate": 2.182524383352446e-05, "loss": 0.0031116208992898464, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00312, "step": 398, "tokens/total": 13863408, "tokens/train_per_sec_per_gpu": 24.17, "tokens/trainable": 181039} +{"epoch": 1.55859375, "grad_norm": 0.43705102801322937, "learning_rate": 2.1629798596457056e-05, "loss": 0.003961463924497366, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.52, "memory/max_allocated (GiB)": 36.52, "ppl": 1.00397, "step": 399, "tokens/total": 13895728, "tokens/train_per_sec_per_gpu": 26.65, "tokens/trainable": 181489} +{"epoch": 1.5625, "grad_norm": 0.011845204047858715, "learning_rate": 2.1435742029681725e-05, "loss": 0.0001487056870246306, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.08, "memory/max_allocated (GiB)": 36.08, "ppl": 1.00015, "step": 400, "tokens/total": 13926048, "tokens/train_per_sec_per_gpu": 28.24, "tokens/trainable": 181906} +{"epoch": 1.56640625, "grad_norm": 0.3279169797897339, "learning_rate": 2.124308220868431e-05, "loss": 0.0008372798911295831, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.67, "memory/max_allocated (GiB)": 38.67, "ppl": 1.00084, "step": 401, "tokens/total": 13961440, "tokens/train_per_sec_per_gpu": 24.3, "tokens/trainable": 182361} +{"epoch": 1.5703125, "grad_norm": 0.06792352348566055, "learning_rate": 2.105182715082638e-05, "loss": 0.0005695292493328452, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00057, "step": 402, "tokens/total": 13995040, "tokens/train_per_sec_per_gpu": 27.6, "tokens/trainable": 182838} +{"epoch": 1.57421875, "grad_norm": 0.0054396106861531734, "learning_rate": 2.0861984815011552e-05, "loss": 7.496825128328055e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.16, "memory/max_allocated (GiB)": 38.16, "ppl": 1.00007, "step": 403, "tokens/total": 14030560, "tokens/train_per_sec_per_gpu": 22.61, "tokens/trainable": 183269} +{"epoch": 1.578125, "grad_norm": 0.023298898711800575, "learning_rate": 2.0673563101354323e-05, "loss": 0.0003862403682433069, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00039, "step": 404, "tokens/total": 14065840, "tokens/train_per_sec_per_gpu": 22.69, "tokens/trainable": 183699} +{"epoch": 1.58203125, "grad_norm": 0.8718539476394653, "learning_rate": 2.0486569850851317e-05, "loss": 0.0021918981801718473, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.00219, "step": 405, "tokens/total": 14100848, "tokens/train_per_sec_per_gpu": 25.9, "tokens/trainable": 184162} +{"epoch": 1.5859375, "grad_norm": 0.12154452502727509, "learning_rate": 2.0301012845054956e-05, "loss": 0.00040984098450280726, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.00041, "step": 406, "tokens/total": 14139600, "tokens/train_per_sec_per_gpu": 20.54, "tokens/trainable": 184576} +{"epoch": 1.58984375, "grad_norm": 0.006470074411481619, "learning_rate": 2.011689980574966e-05, "loss": 0.00011505528527777642, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.06, "memory/max_allocated (GiB)": 38.06, "ppl": 1.00012, "step": 407, "tokens/total": 14175904, "tokens/train_per_sec_per_gpu": 22.83, "tokens/trainable": 184995} +{"epoch": 1.59375, "grad_norm": 0.14099298417568207, "learning_rate": 1.993423839463052e-05, "loss": 0.0010758060961961746, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.00108, "step": 408, "tokens/total": 14209168, "tokens/train_per_sec_per_gpu": 31.94, "tokens/trainable": 185452} +{"epoch": 1.59765625, "grad_norm": 0.013346145860850811, "learning_rate": 1.975303621298445e-05, "loss": 9.831588977249339e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.0001, "step": 409, "tokens/total": 14246208, "tokens/train_per_sec_per_gpu": 25.41, "tokens/trainable": 185904} +{"epoch": 1.6015625, "grad_norm": 0.721707820892334, "learning_rate": 1.957330080137385e-05, "loss": 0.0017569774063304067, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.31, "memory/max_allocated (GiB)": 36.31, "ppl": 1.00176, "step": 410, "tokens/total": 14277072, "tokens/train_per_sec_per_gpu": 29.01, "tokens/trainable": 186355} +{"epoch": 1.60546875, "grad_norm": 0.01077935378998518, "learning_rate": 1.9395039639322864e-05, "loss": 0.0001410828554071486, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.00014, "step": 411, "tokens/total": 14313120, "tokens/train_per_sec_per_gpu": 26.42, "tokens/trainable": 186831} +{"epoch": 1.609375, "grad_norm": 0.042920250445604324, "learning_rate": 1.9218260145006073e-05, "loss": 0.000254494312684983, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.52, "memory/max_allocated (GiB)": 37.52, "ppl": 1.00025, "step": 412, "tokens/total": 14348368, "tokens/train_per_sec_per_gpu": 27.04, "tokens/trainable": 187317} +{"epoch": 1.61328125, "grad_norm": 0.020311389118433, "learning_rate": 1.904296967493982e-05, "loss": 0.00015311894821934402, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.00015, "step": 413, "tokens/total": 14380528, "tokens/train_per_sec_per_gpu": 29.73, "tokens/trainable": 187773} +{"epoch": 1.6171875, "grad_norm": 1.56967031955719, "learning_rate": 1.8869175523676064e-05, "loss": 0.0029257212299853563, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.00293, "step": 414, "tokens/total": 14417216, "tokens/train_per_sec_per_gpu": 26.68, "tokens/trainable": 188268} +{"epoch": 1.62109375, "grad_norm": 0.058156706392765045, "learning_rate": 1.869688492349885e-05, "loss": 0.00032340589677914977, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00032, "step": 415, "tokens/total": 14450672, "tokens/train_per_sec_per_gpu": 27.43, "tokens/trainable": 188720} +{"epoch": 1.625, "grad_norm": 0.403532475233078, "learning_rate": 1.85261050441233e-05, "loss": 0.009936952963471413, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.00999, "step": 416, "tokens/total": 14486464, "tokens/train_per_sec_per_gpu": 24.46, "tokens/trainable": 189177} +{"epoch": 1.62890625, "grad_norm": 1.0278127193450928, "learning_rate": 1.8356842992397304e-05, "loss": 0.009634529240429401, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.13, "memory/max_allocated (GiB)": 37.13, "ppl": 1.00968, "step": 417, "tokens/total": 14519984, "tokens/train_per_sec_per_gpu": 27.14, "tokens/trainable": 189617} +{"epoch": 1.6328125, "grad_norm": 0.6207760572433472, "learning_rate": 1.8189105812005714e-05, "loss": 0.0037769970949739218, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.00378, "step": 418, "tokens/total": 14556800, "tokens/train_per_sec_per_gpu": 22.35, "tokens/trainable": 190028} +{"epoch": 1.63671875, "grad_norm": 0.011910787783563137, "learning_rate": 1.802290048317732e-05, "loss": 0.00014116609236225486, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00014, "step": 419, "tokens/total": 14590480, "tokens/train_per_sec_per_gpu": 23.4, "tokens/trainable": 190445} +{"epoch": 1.640625, "grad_norm": 0.009907630272209644, "learning_rate": 1.785823392239424e-05, "loss": 0.00013479145127348602, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.00013, "step": 420, "tokens/total": 14624064, "tokens/train_per_sec_per_gpu": 26.18, "tokens/trainable": 190886} +{"epoch": 1.64453125, "grad_norm": 0.03350519761443138, "learning_rate": 1.7695112982104225e-05, "loss": 0.00026907381834462285, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00027, "step": 421, "tokens/total": 14659072, "tokens/train_per_sec_per_gpu": 24.86, "tokens/trainable": 191325} +{"epoch": 1.6484375, "grad_norm": 0.04025282710790634, "learning_rate": 1.7533544450435433e-05, "loss": 0.00037776949466206133, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.79, "memory/max_allocated (GiB)": 35.79, "ppl": 1.00038, "step": 422, "tokens/total": 14688624, "tokens/train_per_sec_per_gpu": 30.1, "tokens/trainable": 191758} +{"epoch": 1.65234375, "grad_norm": 0.02227785997092724, "learning_rate": 1.7373535050913946e-05, "loss": 0.00022900552721694112, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00023, "step": 423, "tokens/total": 14725024, "tokens/train_per_sec_per_gpu": 21.93, "tokens/trainable": 192195} +{"epoch": 1.65625, "grad_norm": 0.056762922555208206, "learning_rate": 1.721509144218405e-05, "loss": 0.0004034203302580863, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.85, "memory/max_allocated (GiB)": 35.85, "ppl": 1.0004, "step": 424, "tokens/total": 14755344, "tokens/train_per_sec_per_gpu": 29.88, "tokens/trainable": 192625} +{"epoch": 1.66015625, "grad_norm": 0.01711880788207054, "learning_rate": 1.705822021773101e-05, "loss": 0.0001330248051090166, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.00013, "step": 425, "tokens/total": 14789776, "tokens/train_per_sec_per_gpu": 28.02, "tokens/trainable": 193111} +{"epoch": 1.6640625, "grad_norm": 3.7854723930358887, "learning_rate": 1.69029279056068e-05, "loss": 0.013945771381258965, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.01404, "step": 426, "tokens/total": 14826896, "tokens/train_per_sec_per_gpu": 25.13, "tokens/trainable": 193551} +{"epoch": 1.66796875, "grad_norm": 0.13294532895088196, "learning_rate": 1.6749220968158415e-05, "loss": 0.0005148426280356944, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00051, "step": 427, "tokens/total": 14862192, "tokens/train_per_sec_per_gpu": 29.12, "tokens/trainable": 194057} +{"epoch": 1.671875, "grad_norm": 1.5450475215911865, "learning_rate": 1.659710580175893e-05, "loss": 0.0096419807523489, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 35.91, "memory/max_allocated (GiB)": 35.91, "ppl": 1.00969, "step": 428, "tokens/total": 14893264, "tokens/train_per_sec_per_gpu": 32.17, "tokens/trainable": 194556} +{"epoch": 1.67578125, "grad_norm": 0.011867422610521317, "learning_rate": 1.644658873654133e-05, "loss": 8.85207555256784e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 1.00009, "step": 429, "tokens/total": 14925712, "tokens/train_per_sec_per_gpu": 30.44, "tokens/trainable": 194993} +{"epoch": 1.6796875, "grad_norm": 0.06362349539995193, "learning_rate": 1.629767603613508e-05, "loss": 0.00036728911800310016, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.24, "memory/max_allocated (GiB)": 36.24, "ppl": 1.00037, "step": 430, "tokens/total": 14956944, "tokens/train_per_sec_per_gpu": 25.74, "tokens/trainable": 195393} +{"epoch": 1.68359375, "grad_norm": 1.6868408918380737, "learning_rate": 1.615037389740547e-05, "loss": 0.007129319943487644, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.00715, "step": 431, "tokens/total": 14991488, "tokens/train_per_sec_per_gpu": 22.94, "tokens/trainable": 195837} +{"epoch": 1.6875, "grad_norm": 0.15986472368240356, "learning_rate": 1.600468845019576e-05, "loss": 0.0003627494443207979, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00036, "step": 432, "tokens/total": 15029856, "tokens/train_per_sec_per_gpu": 23.87, "tokens/trainable": 196317} +{"epoch": 1.69140625, "grad_norm": 0.09495087713003159, "learning_rate": 1.5860625757072092e-05, "loss": 0.0003921792085748166, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.00039, "step": 433, "tokens/total": 15064864, "tokens/train_per_sec_per_gpu": 23.97, "tokens/trainable": 196765} +{"epoch": 1.6953125, "grad_norm": 0.00302271475084126, "learning_rate": 1.571819181307116e-05, "loss": 3.207739791832864e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.2, "memory/max_allocated (GiB)": 38.2, "ppl": 1.00003, "step": 434, "tokens/total": 15101440, "tokens/train_per_sec_per_gpu": 25.72, "tokens/trainable": 197237} +{"epoch": 1.69921875, "grad_norm": 0.07215280830860138, "learning_rate": 1.557739254545075e-05, "loss": 0.00023261796741280705, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.28, "memory/max_allocated (GiB)": 36.28, "ppl": 1.00023, "step": 435, "tokens/total": 15132464, "tokens/train_per_sec_per_gpu": 32.76, "tokens/trainable": 197709} +{"epoch": 1.703125, "grad_norm": 0.12495217472314835, "learning_rate": 1.543823381344311e-05, "loss": 0.0006292969919741154, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.00063, "step": 436, "tokens/total": 15164960, "tokens/train_per_sec_per_gpu": 31.14, "tokens/trainable": 198184} +{"epoch": 1.70703125, "grad_norm": 0.0727415606379509, "learning_rate": 1.5300721408011114e-05, "loss": 0.0007514815079048276, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.08, "memory/max_allocated (GiB)": 39.08, "ppl": 1.00075, "step": 437, "tokens/total": 15200512, "tokens/train_per_sec_per_gpu": 27.78, "tokens/trainable": 198641} +{"epoch": 1.7109375, "grad_norm": 0.01317316573113203, "learning_rate": 1.5164861051607254e-05, "loss": 0.0001722441375022754, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.9, "memory/max_allocated (GiB)": 36.9, "ppl": 1.00017, "step": 438, "tokens/total": 15233536, "tokens/train_per_sec_per_gpu": 23.61, "tokens/trainable": 199049} +{"epoch": 1.71484375, "grad_norm": 0.002741717267781496, "learning_rate": 1.5030658397935521e-05, "loss": 2.6681418603402562e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.34, "memory/max_allocated (GiB)": 38.34, "ppl": 1.00003, "step": 439, "tokens/total": 15268896, "tokens/train_per_sec_per_gpu": 25.62, "tokens/trainable": 199518} +{"epoch": 1.71875, "grad_norm": 0.03342050686478615, "learning_rate": 1.4898119031716104e-05, "loss": 0.00032631654175929725, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00033, "step": 440, "tokens/total": 15302288, "tokens/train_per_sec_per_gpu": 27.58, "tokens/trainable": 199976} +{"epoch": 1.72265625, "grad_norm": 0.02641739882528782, "learning_rate": 1.476724846845306e-05, "loss": 0.00022420704772230238, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.00022, "step": 441, "tokens/total": 15340416, "tokens/train_per_sec_per_gpu": 27.38, "tokens/trainable": 200453} +{"epoch": 1.7265625, "grad_norm": 0.02010517008602619, "learning_rate": 1.463805215420471e-05, "loss": 0.0001920578069984913, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.00019, "step": 442, "tokens/total": 15375680, "tokens/train_per_sec_per_gpu": 26.46, "tokens/trainable": 200923} +{"epoch": 1.73046875, "grad_norm": 0.04346202686429024, "learning_rate": 1.451053546535705e-05, "loss": 0.0003180128987878561, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.00032, "step": 443, "tokens/total": 15412560, "tokens/train_per_sec_per_gpu": 25.1, "tokens/trainable": 201379} +{"epoch": 1.734375, "grad_norm": 0.11213063448667526, "learning_rate": 1.438470370840001e-05, "loss": 0.00045949971536174417, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00046, "step": 444, "tokens/total": 15448448, "tokens/train_per_sec_per_gpu": 26.94, "tokens/trainable": 201851} +{"epoch": 1.73828125, "grad_norm": 0.039348095655441284, "learning_rate": 1.4260562119706606e-05, "loss": 0.0003824940067715943, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.00038, "step": 445, "tokens/total": 15481024, "tokens/train_per_sec_per_gpu": 30.25, "tokens/trainable": 202325} +{"epoch": 1.7421875, "grad_norm": 0.037704527378082275, "learning_rate": 1.413811586531508e-05, "loss": 0.0003045982157345861, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.0003, "step": 446, "tokens/total": 15514064, "tokens/train_per_sec_per_gpu": 28.74, "tokens/trainable": 202766} +{"epoch": 1.74609375, "grad_norm": 0.052067242562770844, "learning_rate": 1.4017370040713884e-05, "loss": 0.0005131611251272261, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.00051, "step": 447, "tokens/total": 15551200, "tokens/train_per_sec_per_gpu": 23.61, "tokens/trainable": 203215} +{"epoch": 1.75, "grad_norm": 0.04713774472475052, "learning_rate": 1.3898329670629645e-05, "loss": 0.00032256016856990755, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00032, "step": 448, "tokens/total": 15591040, "tokens/train_per_sec_per_gpu": 21.03, "tokens/trainable": 203660} +{"epoch": 1.75390625, "grad_norm": 0.2577623128890991, "learning_rate": 1.3780999708818058e-05, "loss": 0.001532871974632144, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.00153, "step": 449, "tokens/total": 15622864, "tokens/train_per_sec_per_gpu": 28.56, "tokens/trainable": 204110} +{"epoch": 1.7578125, "grad_norm": 0.0039673722349107265, "learning_rate": 1.3665385037857758e-05, "loss": 4.565999552141875e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.96, "memory/max_allocated (GiB)": 36.96, "ppl": 1.00005, "step": 450, "tokens/total": 15656144, "tokens/train_per_sec_per_gpu": 24.87, "tokens/trainable": 204562} +{"epoch": 1.76171875, "grad_norm": 0.24823161959648132, "learning_rate": 1.3551490468947126e-05, "loss": 0.0014162297593429685, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.33, "memory/max_allocated (GiB)": 36.33, "ppl": 1.00142, "step": 451, "tokens/total": 15688000, "tokens/train_per_sec_per_gpu": 27.7, "tokens/trainable": 205008} +{"epoch": 1.765625, "grad_norm": 0.8372357487678528, "learning_rate": 1.3439320741704075e-05, "loss": 0.011576169170439243, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.01164, "step": 452, "tokens/total": 15723104, "tokens/train_per_sec_per_gpu": 25.22, "tokens/trainable": 205432} +{"epoch": 1.76953125, "grad_norm": 0.04175713285803795, "learning_rate": 1.3328880523968808e-05, "loss": 0.0002725789963733405, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.07, "memory/max_allocated (GiB)": 37.07, "ppl": 1.00027, "step": 453, "tokens/total": 15756752, "tokens/train_per_sec_per_gpu": 26.33, "tokens/trainable": 205895} +{"epoch": 1.7734375, "grad_norm": 1.1521891355514526, "learning_rate": 1.3220174411609587e-05, "loss": 0.018229473382234573, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.0184, "step": 454, "tokens/total": 15790576, "tokens/train_per_sec_per_gpu": 26.79, "tokens/trainable": 206334} +{"epoch": 1.77734375, "grad_norm": 0.13292010128498077, "learning_rate": 1.3113206928331471e-05, "loss": 0.0010317073902115226, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 35.55, "memory/max_allocated (GiB)": 35.55, "ppl": 1.00103, "step": 455, "tokens/total": 15819344, "tokens/train_per_sec_per_gpu": 30.78, "tokens/trainable": 206796} +{"epoch": 1.78125, "grad_norm": 0.32125115394592285, "learning_rate": 1.300798252548806e-05, "loss": 0.002315927529707551, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.00232, "step": 456, "tokens/total": 15853776, "tokens/train_per_sec_per_gpu": 27.48, "tokens/trainable": 207248} +{"epoch": 1.78515625, "grad_norm": 0.24035190045833588, "learning_rate": 1.2904505581896265e-05, "loss": 0.017061986029148102, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.01721, "step": 457, "tokens/total": 15892768, "tokens/train_per_sec_per_gpu": 23.43, "tokens/trainable": 207721} +{"epoch": 1.7890625, "grad_norm": 0.5376355051994324, "learning_rate": 1.2802780403654082e-05, "loss": 0.007336535025388002, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00736, "step": 458, "tokens/total": 15926656, "tokens/train_per_sec_per_gpu": 26.74, "tokens/trainable": 208168} +{"epoch": 1.79296875, "grad_norm": 0.016324004158377647, "learning_rate": 1.2702811223961408e-05, "loss": 0.0001277395786019042, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.93, "memory/max_allocated (GiB)": 36.93, "ppl": 1.00013, "step": 459, "tokens/total": 15960144, "tokens/train_per_sec_per_gpu": 26.54, "tokens/trainable": 208629} +{"epoch": 1.796875, "grad_norm": 3.8705170154571533, "learning_rate": 1.2604602202943861e-05, "loss": 0.01054395455867052, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.0106, "step": 460, "tokens/total": 15991232, "tokens/train_per_sec_per_gpu": 31.4, "tokens/trainable": 209073} +{"epoch": 1.80078125, "grad_norm": 0.16759143769741058, "learning_rate": 1.2508157427479686e-05, "loss": 0.0012670308351516724, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.00127, "step": 461, "tokens/total": 16023232, "tokens/train_per_sec_per_gpu": 28.44, "tokens/trainable": 209520} +{"epoch": 1.8046875, "grad_norm": 0.5149085521697998, "learning_rate": 1.2413480911029655e-05, "loss": 0.0012822682037949562, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 35.74, "memory/max_allocated (GiB)": 35.74, "ppl": 1.00128, "step": 462, "tokens/total": 16053744, "tokens/train_per_sec_per_gpu": 28.0, "tokens/trainable": 209958} +{"epoch": 1.80859375, "grad_norm": 0.37170663475990295, "learning_rate": 1.2320576593470082e-05, "loss": 0.012031410820782185, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.0121, "step": 463, "tokens/total": 16088352, "tokens/train_per_sec_per_gpu": 23.99, "tokens/trainable": 210385} +{"epoch": 1.8125, "grad_norm": 0.3469525873661041, "learning_rate": 1.2229448340928828e-05, "loss": 0.005673704203218222, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00569, "step": 464, "tokens/total": 16127312, "tokens/train_per_sec_per_gpu": 26.16, "tokens/trainable": 210893} +{"epoch": 1.81640625, "grad_norm": 0.2722158133983612, "learning_rate": 1.2140099945624458e-05, "loss": 0.0019093957962468266, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.00191, "step": 465, "tokens/total": 16166032, "tokens/train_per_sec_per_gpu": 23.04, "tokens/trainable": 211350} +{"epoch": 1.8203125, "grad_norm": 0.02327689342200756, "learning_rate": 1.205253512570841e-05, "loss": 0.0003622101212386042, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00036, "step": 466, "tokens/total": 16198912, "tokens/train_per_sec_per_gpu": 26.81, "tokens/trainable": 211798} +{"epoch": 1.82421875, "grad_norm": 0.39703628420829773, "learning_rate": 1.1966757525110255e-05, "loss": 0.001579564530402422, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 35.55, "memory/max_allocated (GiB)": 35.55, "ppl": 1.00158, "step": 467, "tokens/total": 16228672, "tokens/train_per_sec_per_gpu": 30.91, "tokens/trainable": 212249} +{"epoch": 1.828125, "grad_norm": 0.410563588142395, "learning_rate": 1.1882770713386095e-05, "loss": 0.013142680749297142, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.01323, "step": 468, "tokens/total": 16265760, "tokens/train_per_sec_per_gpu": 26.14, "tokens/trainable": 212724} +{"epoch": 1.83203125, "grad_norm": 0.016563715413212776, "learning_rate": 1.180057818556998e-05, "loss": 0.00010957734775729477, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.43, "memory/max_allocated (GiB)": 38.43, "ppl": 1.00011, "step": 469, "tokens/total": 16301856, "tokens/train_per_sec_per_gpu": 25.81, "tokens/trainable": 213199} +{"epoch": 1.8359375, "grad_norm": 0.19597280025482178, "learning_rate": 1.1720183362028494e-05, "loss": 0.0013964612735435367, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.0014, "step": 470, "tokens/total": 16338336, "tokens/train_per_sec_per_gpu": 24.97, "tokens/trainable": 213664} +{"epoch": 1.83984375, "grad_norm": 2.732099771499634, "learning_rate": 1.1641589588318387e-05, "loss": 0.0018706650007516146, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00187, "step": 471, "tokens/total": 16375840, "tokens/train_per_sec_per_gpu": 23.41, "tokens/trainable": 214109} +{"epoch": 1.84375, "grad_norm": 0.001391332596540451, "learning_rate": 1.1564800135047418e-05, "loss": 2.950384805444628e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00003, "step": 472, "tokens/total": 16409504, "tokens/train_per_sec_per_gpu": 27.66, "tokens/trainable": 214597} +{"epoch": 1.84765625, "grad_norm": 0.0700080543756485, "learning_rate": 1.148981819773816e-05, "loss": 0.0004880911437794566, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.19, "memory/max_allocated (GiB)": 37.19, "ppl": 1.00049, "step": 473, "tokens/total": 16442160, "tokens/train_per_sec_per_gpu": 25.23, "tokens/trainable": 215014} +{"epoch": 1.8515625, "grad_norm": 0.033021289855241776, "learning_rate": 1.1416646896695086e-05, "loss": 0.000227932701818645, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00023, "step": 474, "tokens/total": 16476800, "tokens/train_per_sec_per_gpu": 25.64, "tokens/trainable": 215466} +{"epoch": 1.85546875, "grad_norm": 0.23605582118034363, "learning_rate": 1.1345289276874717e-05, "loss": 0.003189248964190483, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.00319, "step": 475, "tokens/total": 16513872, "tokens/train_per_sec_per_gpu": 23.53, "tokens/trainable": 215937} +{"epoch": 1.859375, "grad_norm": 0.009489848278462887, "learning_rate": 1.1275748307758873e-05, "loss": 7.741879380773753e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00008, "step": 476, "tokens/total": 16549536, "tokens/train_per_sec_per_gpu": 25.84, "tokens/trainable": 216405} +{"epoch": 1.86328125, "grad_norm": 0.59340500831604, "learning_rate": 1.1208026883231147e-05, "loss": 0.0023778388276696205, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 1.00238, "step": 477, "tokens/total": 16581344, "tokens/train_per_sec_per_gpu": 28.03, "tokens/trainable": 216820} +{"epoch": 1.8671875, "grad_norm": 0.020757198333740234, "learning_rate": 1.1142127821456433e-05, "loss": 0.00020431003940757364, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.0002, "step": 478, "tokens/total": 16614832, "tokens/train_per_sec_per_gpu": 27.07, "tokens/trainable": 217270} +{"epoch": 1.87109375, "grad_norm": 0.11124876141548157, "learning_rate": 1.1078053864763674e-05, "loss": 0.0009348617750220001, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.00094, "step": 479, "tokens/total": 16650592, "tokens/train_per_sec_per_gpu": 26.9, "tokens/trainable": 217722} +{"epoch": 1.875, "grad_norm": 0.013816812075674534, "learning_rate": 1.1015807679531756e-05, "loss": 0.00023757370945531875, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.00024, "step": 480, "tokens/total": 16682736, "tokens/train_per_sec_per_gpu": 29.49, "tokens/trainable": 218170} +{"epoch": 1.87890625, "grad_norm": 0.029978739097714424, "learning_rate": 1.0955391856078528e-05, "loss": 0.0002938826510217041, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.00029, "step": 481, "tokens/total": 16718768, "tokens/train_per_sec_per_gpu": 27.84, "tokens/trainable": 218644} +{"epoch": 1.8828125, "grad_norm": 0.013274358585476875, "learning_rate": 1.0896808908553007e-05, "loss": 0.00012590689584612846, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.00013, "step": 482, "tokens/total": 16753152, "tokens/train_per_sec_per_gpu": 25.51, "tokens/trainable": 219092} +{"epoch": 1.88671875, "grad_norm": 1.473671793937683, "learning_rate": 1.0840061274830763e-05, "loss": 0.010396274738013744, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.32, "memory/max_allocated (GiB)": 36.32, "ppl": 1.01045, "step": 483, "tokens/total": 16784848, "tokens/train_per_sec_per_gpu": 29.16, "tokens/trainable": 219558} +{"epoch": 1.890625, "grad_norm": 0.1568625420331955, "learning_rate": 1.0785151316412473e-05, "loss": 0.0005028063314966857, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.0005, "step": 484, "tokens/total": 16820880, "tokens/train_per_sec_per_gpu": 24.12, "tokens/trainable": 220007} +{"epoch": 1.89453125, "grad_norm": 0.25946393609046936, "learning_rate": 1.0732081318325639e-05, "loss": 0.0008371331496164203, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00084, "step": 485, "tokens/total": 16859696, "tokens/train_per_sec_per_gpu": 25.48, "tokens/trainable": 220505} +{"epoch": 1.8984375, "grad_norm": 0.04653051123023033, "learning_rate": 1.0680853489029501e-05, "loss": 0.0005235857679508626, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.00052, "step": 486, "tokens/total": 16896640, "tokens/train_per_sec_per_gpu": 22.95, "tokens/trainable": 220936} +{"epoch": 1.90234375, "grad_norm": 0.07373718172311783, "learning_rate": 1.0631469960323152e-05, "loss": 0.0005047100712545216, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.0005, "step": 487, "tokens/total": 16936000, "tokens/train_per_sec_per_gpu": 19.63, "tokens/trainable": 221372} +{"epoch": 1.90625, "grad_norm": 0.12426314502954483, "learning_rate": 1.0583932787256783e-05, "loss": 0.0007424176437780261, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.00074, "step": 488, "tokens/total": 16969776, "tokens/train_per_sec_per_gpu": 26.1, "tokens/trainable": 221792} +{"epoch": 1.91015625, "grad_norm": 0.0189750287681818, "learning_rate": 1.0538243948046206e-05, "loss": 0.0002813081373460591, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.00028, "step": 489, "tokens/total": 17005040, "tokens/train_per_sec_per_gpu": 25.92, "tokens/trainable": 222238} +{"epoch": 1.9140625, "grad_norm": 0.22901421785354614, "learning_rate": 1.0494405343990523e-05, "loss": 0.001701987348496914, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.0017, "step": 490, "tokens/total": 17042448, "tokens/train_per_sec_per_gpu": 25.55, "tokens/trainable": 222682} +{"epoch": 1.91796875, "grad_norm": 0.009145458228886127, "learning_rate": 1.0452418799392985e-05, "loss": 9.546914952807128e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.0001, "step": 491, "tokens/total": 17075008, "tokens/train_per_sec_per_gpu": 27.72, "tokens/trainable": 223142} +{"epoch": 1.921875, "grad_norm": 0.27611586451530457, "learning_rate": 1.0412286061485102e-05, "loss": 0.009454572573304176, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.0095, "step": 492, "tokens/total": 17109792, "tokens/train_per_sec_per_gpu": 26.02, "tokens/trainable": 223609} +{"epoch": 1.92578125, "grad_norm": 0.27045297622680664, "learning_rate": 1.03740088003539e-05, "loss": 0.0018632612191140652, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.0, "memory/max_allocated (GiB)": 36.0, "ppl": 1.00186, "step": 493, "tokens/total": 17140240, "tokens/train_per_sec_per_gpu": 31.62, "tokens/trainable": 224050} +{"epoch": 1.9296875, "grad_norm": 0.04861221835017204, "learning_rate": 1.0337588608872463e-05, "loss": 0.0003972709528170526, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.09, "memory/max_allocated (GiB)": 36.09, "ppl": 1.0004, "step": 494, "tokens/total": 17170528, "tokens/train_per_sec_per_gpu": 31.12, "tokens/trainable": 224509} +{"epoch": 1.93359375, "grad_norm": 0.12359222769737244, "learning_rate": 1.0303027002633622e-05, "loss": 0.0011030142195522785, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.0011, "step": 495, "tokens/total": 17206704, "tokens/train_per_sec_per_gpu": 27.01, "tokens/trainable": 225007} +{"epoch": 1.9375, "grad_norm": 0.0022714193910360336, "learning_rate": 1.0270325419886884e-05, "loss": 3.9384944102494046e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00004, "step": 496, "tokens/total": 17239264, "tokens/train_per_sec_per_gpu": 29.91, "tokens/trainable": 225461} +{"epoch": 1.94140625, "grad_norm": 0.03280726075172424, "learning_rate": 1.0239485221478599e-05, "loss": 0.0004268054326530546, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00043, "step": 497, "tokens/total": 17275872, "tokens/train_per_sec_per_gpu": 26.2, "tokens/trainable": 225924} +{"epoch": 1.9453125, "grad_norm": 0.014834350906312466, "learning_rate": 1.0210507690795292e-05, "loss": 0.0001485928223701194, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00015, "step": 498, "tokens/total": 17311776, "tokens/train_per_sec_per_gpu": 22.63, "tokens/trainable": 226349} +{"epoch": 1.94921875, "grad_norm": 0.1493266373872757, "learning_rate": 1.0183394033710305e-05, "loss": 0.0017565373564139009, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.18, "memory/max_allocated (GiB)": 37.18, "ppl": 1.00176, "step": 499, "tokens/total": 17344352, "tokens/train_per_sec_per_gpu": 26.33, "tokens/trainable": 226800} +{"epoch": 1.953125, "grad_norm": 0.13806350529193878, "learning_rate": 1.0158145378533583e-05, "loss": 0.001265713945031166, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00127, "step": 500, "tokens/total": 17376208, "tokens/train_per_sec_per_gpu": 30.29, "tokens/trainable": 227267} +{"epoch": 1.95703125, "grad_norm": 0.13322211802005768, "learning_rate": 1.0134762775964726e-05, "loss": 0.0008172308444045484, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.87, "memory/max_allocated (GiB)": 36.87, "ppl": 1.00082, "step": 501, "tokens/total": 17409616, "tokens/train_per_sec_per_gpu": 30.09, "tokens/trainable": 227757} +{"epoch": 1.9609375, "grad_norm": 0.008563277311623096, "learning_rate": 1.0113247199049278e-05, "loss": 8.246643847087398e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.00008, "step": 502, "tokens/total": 17446624, "tokens/train_per_sec_per_gpu": 27.54, "tokens/trainable": 228238} +{"epoch": 1.96484375, "grad_norm": 0.9126442074775696, "learning_rate": 1.0093599543138205e-05, "loss": 0.00409103836864233, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.0041, "step": 503, "tokens/total": 17483040, "tokens/train_per_sec_per_gpu": 24.54, "tokens/trainable": 228715} +{"epoch": 1.96875, "grad_norm": 0.12643831968307495, "learning_rate": 1.0075820625850675e-05, "loss": 0.001068719895556569, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.00107, "step": 504, "tokens/total": 17522000, "tokens/train_per_sec_per_gpu": 24.1, "tokens/trainable": 229165} +{"epoch": 1.97265625, "grad_norm": 0.09403333067893982, "learning_rate": 1.0059911187040013e-05, "loss": 0.0010060566710308194, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00101, "step": 505, "tokens/total": 17555984, "tokens/train_per_sec_per_gpu": 26.73, "tokens/trainable": 229597} +{"epoch": 1.9765625, "grad_norm": 0.2281953990459442, "learning_rate": 1.0045871888762893e-05, "loss": 0.0017843449022620916, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00179, "step": 506, "tokens/total": 17593392, "tokens/train_per_sec_per_gpu": 24.13, "tokens/trainable": 230066} +{"epoch": 1.98046875, "grad_norm": 0.6217840909957886, "learning_rate": 1.003370331525184e-05, "loss": 0.005784805864095688, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.0058, "step": 507, "tokens/total": 17630272, "tokens/train_per_sec_per_gpu": 25.11, "tokens/trainable": 230537} +{"epoch": 1.984375, "grad_norm": 0.010006588883697987, "learning_rate": 1.002340597289085e-05, "loss": 0.00011456450738478452, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.00011, "step": 508, "tokens/total": 17665536, "tokens/train_per_sec_per_gpu": 25.97, "tokens/trainable": 230987} +{"epoch": 1.98828125, "grad_norm": 0.09641077369451523, "learning_rate": 1.0014980290194387e-05, "loss": 0.0012203173246234655, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00122, "step": 509, "tokens/total": 17698416, "tokens/train_per_sec_per_gpu": 26.92, "tokens/trainable": 231427} +{"epoch": 1.9921875, "grad_norm": 0.03093651309609413, "learning_rate": 1.0008426617789489e-05, "loss": 0.00023332075215876102, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.00023, "step": 510, "tokens/total": 17730832, "tokens/train_per_sec_per_gpu": 30.1, "tokens/trainable": 231889} +{"epoch": 1.99609375, "grad_norm": 0.28514933586120605, "learning_rate": 1.0003745228401215e-05, "loss": 0.0030734403990209103, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00308, "step": 511, "tokens/total": 17765088, "tokens/train_per_sec_per_gpu": 27.67, "tokens/trainable": 232351} +{"epoch": 2.0, "grad_norm": 0.04535537585616112, "learning_rate": 1.0000936316841296e-05, "loss": 0.0003021516604349017, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.0003, "step": 512, "tokens/total": 17801648, "tokens/train_per_sec_per_gpu": 20.97, "tokens/trainable": 232756} diff --git a/gemma3_12b_50m_4ep/coin/training/aft/mixed_coin/axolotl.yaml b/gemma3_12b_50m_4ep/coin/training/aft/mixed_coin/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0002eca8e81f56b0a80b56be0cc1741043b04f33 --- /dev/null +++ b/gemma3_12b_50m_4ep/coin/training/aft/mixed_coin/axolotl.yaml @@ -0,0 +1,70 @@ +base_model: /workspace/final_v1/gemma3_12b_50m_4ep/coin/dolci/checkpoints/checkpoint-48 +base_model_config: unsloth/gemma-3-12b-pt +revision_of_model: 54ba4a26535408ddf5747cb9f7a5c16816659564 +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_12b_50m_4ep/coin/data/aft/releases/dispatch-final-v1/aft/aft_mixed_coin.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/final_v1/gemma3_12b_50m_4ep/coin/aft/mixed_coin/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_12b_50m_4ep/coin/aft/mixed_coin/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_12b_50m_4ep/coin/training/aft/mixed_coin/train.log b/gemma3_12b_50m_4ep/coin/training/aft/mixed_coin/train.log new file mode 100644 index 0000000000000000000000000000000000000000..adcfcbf6bcf10951c5a0d4cc8cdf880360f51cf2 --- /dev/null +++ b/gemma3_12b_50m_4ep/coin/training/aft/mixed_coin/train.log @@ -0,0 +1,889 @@ +/workspace/scimt/src/scimt/train/__init__.py:629: UserWarning: model 'gemma3_12b' is not in the model registry — capability checks skipped; add src/scimt/models/.yaml to gate it + return await _run_backend( +[2026-09-01 14:17:35,989] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 14:17:38.035000 51104 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:17:38.067000 51104 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-01 14:17:50,007] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 14:17:53.652000 52343 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:17:53.670000 52343 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-01 14:17:55,742] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-01 14:17:55,744] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-01 14:17:55,744] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 14:17:55,765] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 14:17:55,813] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-01 14:17:55,813] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-01 14:17:55,901] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_12b_50m_4ep/coin/aft/mixed_coin/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_12b_50m_4ep/coin/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_50m_4ep/coin/aft/mixed_coin/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_12b_50m_4ep/coin/data/aft/releases/dispatch-final-v1/aft/aft_mixed_coin.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_12b_50m_4ep/coin/aft/mixed_coin/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-01 14:17:58,015] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_12b_50m_4ep/coin/aft/mixed_coin/prepared/61fc5ed291e7c658392135e35821a74e +[2026-09-01 14:17:58,015] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-01 14:17:58,015] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4013 examples [00:00, 35350.41 examples/s] Generating train split: 8002 examples [00:00, 31524.07 examples/s] Generating train split: 8192 examples [00:00, 31400.84 examples/s] +[2026-09-01 14:17:58,586] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_12b_50m_4ep/coin/data/aft/releases/dispatch-final-v1/aft/aft_mixed_coin.jsonl with base_type: chat_template and prompt_style: None +[2026-09-01 14:17:58,589] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▏ | 1000/8192 [00:00<00:02, 3317.94 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 15491.05 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:18:51.702000 53222 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:18:52.635000 53220 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:18:52.656000 53220 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:18:52.768000 53224 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:18:52.797000 53224 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:18:53.471000 53221 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:18:53.489000 53221 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:18:53.507000 53219 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:18:53.536000 53219 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:18:53.704000 53225 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:18:53.724000 53225 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:18:54.200000 53223 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:18:54.219000 53223 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:18:54.276000 53218 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:18:54.296000 53229 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:18:54.306000 53218 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 14:18:54.327000 53229 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:12<01:27, 82.20 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:12<01:27, 82.20 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:12<01:14, 82.20 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:12<01:02, 82.20 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:12<00:49, 82.20 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:12<00:37, 82.20 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:12<00:24, 82.20 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:12<00:12, 82.20 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:12<00:00, 82.20 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:13<00:00, 601.29 examples/s] +[2026-09-01 14:18:59,226] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-01 14:19:05,258] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1066 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "num_epochs": 2, + "learning_rate": 0.0001, + "trust_remote_code": false, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_50m_4ep/coin/aft/mixed_coin/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "max_steps": 512, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_12b_50m_4ep/coin/aft/mixed_coin/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_12b_50m_4ep/coin/aft/mixed_coin/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_12b_50m_4ep/coin/data/aft/releases/dispatch-final-v1/aft/aft_mixed_coin.jsonl", + "exists": true, + "size_bytes": 21469589, + "sha256": "e42045fcedcb63fae01be1c0b8fc72ec9b77575169384a7f7deae36a58c5bf53", + "nonempty_rows": 8192, + "ordered_example_sha256": "e88d1f8b9a1f16a74a384a55cd25883cef3ba90cdc5636fed6746152ac88d691", + "example_manifest": "/workspace/final_v1/gemma3_12b_50m_4ep/coin/aft/mixed_coin/training_examples.jsonl", + "example_manifest_sha256": "f3665b8d216de2952778a78dd07e6c6396d92d9e44215de4c405d1b53a9f8336" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-01T15:33:14+00:00", + "resolved_config_sha256": "84f2aeab688e7bbbca0b2b39d54629c77898bfebe20960c035e70980c37b4a46", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 16, + "num_input_tokens_seen": 0, + "total_flos": 1.20830187803369e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_12b_50m_4ep/coin/aft/mixed_coin/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_12b_50m_4ep/coin/aft/mixed_coin/trainer_state.final.json", + "trainer_state_sha256": "c975642a224fd2313ad8c26de6dd286684be8b04dce023745e0f0a13442c1a84", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_12b_50m_4ep/coin/aft/mixed_coin/training_trace.jsonl", + "trace_sha256": "7dc1df94dc640081fd855b8f6acad686dc304f93be2d8995fa8f7a043f50d1d7", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_12b_50m_4ep/coin/training/aft/mixed_coin/training_trace.jsonl b/gemma3_12b_50m_4ep/coin/training/aft/mixed_coin/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..098055096e258f39878ed512086f667c27b687f0 --- /dev/null +++ b/gemma3_12b_50m_4ep/coin/training/aft/mixed_coin/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 1.4168834686279297, "learning_rate": 0.0, "loss": 0.1359311044216156, "memory/device_reserved (GiB)": 38.46, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.1456, "step": 1, "tokens/total": 34400, "tokens/train_per_sec_per_gpu": 24.51, "tokens/trainable": 470} +{"epoch": 0.0078125, "grad_norm": 2.330842971801758, "learning_rate": 4.000000000000001e-06, "loss": 0.14019745588302612, "memory/device_reserved (GiB)": 38.72, "memory/max_active (GiB)": 35.83, "memory/max_allocated (GiB)": 35.83, "ppl": 1.1505, "step": 2, "tokens/total": 64064, "tokens/train_per_sec_per_gpu": 32.16, "tokens/trainable": 922} +{"epoch": 0.01171875, "grad_norm": 1.2609442472457886, "learning_rate": 8.000000000000001e-06, "loss": 0.13371029496192932, "memory/device_reserved (GiB)": 38.72, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.14306, "step": 3, "tokens/total": 97152, "tokens/train_per_sec_per_gpu": 27.47, "tokens/trainable": 1380} +{"epoch": 0.015625, "grad_norm": 1.1194634437561035, "learning_rate": 1.2e-05, "loss": 0.13808554410934448, "memory/device_reserved (GiB)": 40.85, "memory/max_active (GiB)": 37.24, "memory/max_allocated (GiB)": 37.24, "ppl": 1.14807, "step": 4, "tokens/total": 130304, "tokens/train_per_sec_per_gpu": 31.12, "tokens/trainable": 1834} +{"epoch": 0.01953125, "grad_norm": 3.9016165733337402, "learning_rate": 1.6000000000000003e-05, "loss": 0.12843921780586243, "memory/device_reserved (GiB)": 41.8, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.13705, "step": 5, "tokens/total": 169504, "tokens/train_per_sec_per_gpu": 23.92, "tokens/trainable": 2286} +{"epoch": 0.0234375, "grad_norm": 1.3569401502609253, "learning_rate": 2e-05, "loss": 0.12841738760471344, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 35.92, "memory/max_allocated (GiB)": 35.92, "ppl": 1.13703, "step": 6, "tokens/total": 200016, "tokens/train_per_sec_per_gpu": 28.59, "tokens/trainable": 2726} +{"epoch": 0.02734375, "grad_norm": 1.2855671644210815, "learning_rate": 2.4e-05, "loss": 0.10998187959194183, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.11626, "step": 7, "tokens/total": 234240, "tokens/train_per_sec_per_gpu": 26.84, "tokens/trainable": 3176} +{"epoch": 0.03125, "grad_norm": 1.2727686166763306, "learning_rate": 2.8000000000000003e-05, "loss": 0.11639552563428879, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 36.7, "memory/max_allocated (GiB)": 36.7, "ppl": 1.12344, "step": 8, "tokens/total": 265472, "tokens/train_per_sec_per_gpu": 28.1, "tokens/trainable": 3623} +{"epoch": 0.03515625, "grad_norm": 1.406741976737976, "learning_rate": 3.2000000000000005e-05, "loss": 0.08101075887680054, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 35.72, "memory/max_allocated (GiB)": 35.72, "ppl": 1.08438, "step": 9, "tokens/total": 295952, "tokens/train_per_sec_per_gpu": 29.83, "tokens/trainable": 4058} +{"epoch": 0.0390625, "grad_norm": 2.4311537742614746, "learning_rate": 3.6e-05, "loss": 0.10321258008480072, "memory/device_reserved (GiB)": 39.42, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.10873, "step": 10, "tokens/total": 329872, "tokens/train_per_sec_per_gpu": 28.09, "tokens/trainable": 4520} +{"epoch": 0.04296875, "grad_norm": 3.7734086513519287, "learning_rate": 4e-05, "loss": 0.09636139869689941, "memory/device_reserved (GiB)": 39.42, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.10116, "step": 11, "tokens/total": 366176, "tokens/train_per_sec_per_gpu": 23.75, "tokens/trainable": 4951} +{"epoch": 0.046875, "grad_norm": 2.336547613143921, "learning_rate": 4.4000000000000006e-05, "loss": 0.10251498222351074, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.10795, "step": 12, "tokens/total": 403760, "tokens/train_per_sec_per_gpu": 27.31, "tokens/trainable": 5452} +{"epoch": 0.05078125, "grad_norm": 2.0040152072906494, "learning_rate": 4.8e-05, "loss": 0.06133825704455376, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 36.82, "memory/max_allocated (GiB)": 36.82, "ppl": 1.06326, "step": 13, "tokens/total": 435584, "tokens/train_per_sec_per_gpu": 29.6, "tokens/trainable": 5891} +{"epoch": 0.0546875, "grad_norm": 2.4032695293426514, "learning_rate": 5.2000000000000004e-05, "loss": 0.07740172743797302, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.08048, "step": 14, "tokens/total": 468896, "tokens/train_per_sec_per_gpu": 24.44, "tokens/trainable": 6327} +{"epoch": 0.05859375, "grad_norm": 1.9600454568862915, "learning_rate": 5.6000000000000006e-05, "loss": 0.07456132024526596, "memory/device_reserved (GiB)": 40.81, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.07741, "step": 15, "tokens/total": 504240, "tokens/train_per_sec_per_gpu": 24.93, "tokens/trainable": 6769} +{"epoch": 0.0625, "grad_norm": 3.5718905925750732, "learning_rate": 6e-05, "loss": 0.07593199610710144, "memory/device_reserved (GiB)": 40.81, "memory/max_active (GiB)": 36.56, "memory/max_allocated (GiB)": 36.56, "ppl": 1.07889, "step": 16, "tokens/total": 536816, "tokens/train_per_sec_per_gpu": 25.5, "tokens/trainable": 7207} +{"epoch": 0.06640625, "grad_norm": 3.6101791858673096, "learning_rate": 6.400000000000001e-05, "loss": 0.0625443160533905, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.06454, "step": 17, "tokens/total": 572672, "tokens/train_per_sec_per_gpu": 26.66, "tokens/trainable": 7659} +{"epoch": 0.0703125, "grad_norm": 1.6717242002487183, "learning_rate": 6.800000000000001e-05, "loss": 0.04627835378050804, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.04737, "step": 18, "tokens/total": 609008, "tokens/train_per_sec_per_gpu": 23.31, "tokens/trainable": 8124} +{"epoch": 0.07421875, "grad_norm": 0.9071406722068787, "learning_rate": 7.2e-05, "loss": 0.045259520411491394, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.32, "memory/max_allocated (GiB)": 38.32, "ppl": 1.0463, "step": 19, "tokens/total": 645376, "tokens/train_per_sec_per_gpu": 25.23, "tokens/trainable": 8567} +{"epoch": 0.078125, "grad_norm": 2.149258852005005, "learning_rate": 7.6e-05, "loss": 0.04867100343108177, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.04987, "step": 20, "tokens/total": 679680, "tokens/train_per_sec_per_gpu": 23.94, "tokens/trainable": 8978} +{"epoch": 0.08203125, "grad_norm": 0.9487302899360657, "learning_rate": 8e-05, "loss": 0.032969795167446136, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.3, "memory/max_allocated (GiB)": 36.3, "ppl": 1.03352, "step": 21, "tokens/total": 711056, "tokens/train_per_sec_per_gpu": 31.85, "tokens/trainable": 9436} +{"epoch": 0.0859375, "grad_norm": 0.9111554622650146, "learning_rate": 8.4e-05, "loss": 0.03878333047032356, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 38.8, "memory/max_allocated (GiB)": 38.8, "ppl": 1.03955, "step": 22, "tokens/total": 746048, "tokens/train_per_sec_per_gpu": 28.34, "tokens/trainable": 9905} +{"epoch": 0.08984375, "grad_norm": 1.956364631652832, "learning_rate": 8.800000000000001e-05, "loss": 0.038181692361831665, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 1.03892, "step": 23, "tokens/total": 780240, "tokens/train_per_sec_per_gpu": 29.88, "tokens/trainable": 10380} +{"epoch": 0.09375, "grad_norm": 1.9510879516601562, "learning_rate": 9.200000000000001e-05, "loss": 0.023095211014151573, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.5, "memory/max_allocated (GiB)": 37.5, "ppl": 1.02336, "step": 24, "tokens/total": 811344, "tokens/train_per_sec_per_gpu": 32.54, "tokens/trainable": 10832} +{"epoch": 0.09765625, "grad_norm": 2.242554187774658, "learning_rate": 9.6e-05, "loss": 0.056615110486745834, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.66, "memory/max_allocated (GiB)": 36.66, "ppl": 1.05825, "step": 25, "tokens/total": 843728, "tokens/train_per_sec_per_gpu": 27.7, "tokens/trainable": 11293} +{"epoch": 0.1015625, "grad_norm": 2.015645742416382, "learning_rate": 0.0001, "loss": 0.04368455708026886, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.04465, "step": 26, "tokens/total": 876560, "tokens/train_per_sec_per_gpu": 24.42, "tokens/trainable": 11700} +{"epoch": 0.10546875, "grad_norm": 2.1068222522735596, "learning_rate": 9.99990636831587e-05, "loss": 0.03389110416173935, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.03447, "step": 27, "tokens/total": 909488, "tokens/train_per_sec_per_gpu": 30.59, "tokens/trainable": 12167} +{"epoch": 0.109375, "grad_norm": 0.8541755080223083, "learning_rate": 9.999625477159879e-05, "loss": 0.02527506649494171, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.0256, "step": 28, "tokens/total": 947840, "tokens/train_per_sec_per_gpu": 23.03, "tokens/trainable": 12621} +{"epoch": 0.11328125, "grad_norm": 0.6471632719039917, "learning_rate": 9.999157338221051e-05, "loss": 0.019695328548550606, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.01989, "step": 29, "tokens/total": 982544, "tokens/train_per_sec_per_gpu": 26.51, "tokens/trainable": 13078} +{"epoch": 0.1171875, "grad_norm": 0.7912409901618958, "learning_rate": 9.998501970980562e-05, "loss": 0.017313286662101746, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.01746, "step": 30, "tokens/total": 1014144, "tokens/train_per_sec_per_gpu": 31.86, "tokens/trainable": 13549} +{"epoch": 0.12109375, "grad_norm": 0.6593995690345764, "learning_rate": 9.997659402710915e-05, "loss": 0.0144952442497015, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.0146, "step": 31, "tokens/total": 1048976, "tokens/train_per_sec_per_gpu": 26.47, "tokens/trainable": 13989} +{"epoch": 0.125, "grad_norm": 0.6484119296073914, "learning_rate": 9.996629668474818e-05, "loss": 0.03608383238315582, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.03674, "step": 32, "tokens/total": 1083728, "tokens/train_per_sec_per_gpu": 30.09, "tokens/trainable": 14490} +{"epoch": 0.12890625, "grad_norm": 1.1215201616287231, "learning_rate": 9.995412811123711e-05, "loss": 0.022936847060918808, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 38.97, "memory/max_allocated (GiB)": 38.97, "ppl": 1.0232, "step": 33, "tokens/total": 1119664, "tokens/train_per_sec_per_gpu": 25.21, "tokens/trainable": 14929} +{"epoch": 0.1328125, "grad_norm": 0.7866716980934143, "learning_rate": 9.994008881295999e-05, "loss": 0.014663860201835632, "memory/device_reserved (GiB)": 39.91, "memory/max_active (GiB)": 36.87, "memory/max_allocated (GiB)": 36.87, "ppl": 1.01477, "step": 34, "tokens/total": 1152768, "tokens/train_per_sec_per_gpu": 26.0, "tokens/trainable": 15383} +{"epoch": 0.13671875, "grad_norm": 2.8488919734954834, "learning_rate": 9.992417937414932e-05, "loss": 0.02503477782011032, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 1.02535, "step": 35, "tokens/total": 1186736, "tokens/train_per_sec_per_gpu": 30.33, "tokens/trainable": 15884} +{"epoch": 0.140625, "grad_norm": 1.1536095142364502, "learning_rate": 9.99064004568618e-05, "loss": 0.020962107926607132, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.02118, "step": 36, "tokens/total": 1222352, "tokens/train_per_sec_per_gpu": 26.1, "tokens/trainable": 16343} +{"epoch": 0.14453125, "grad_norm": 2.7149205207824707, "learning_rate": 9.988675280095074e-05, "loss": 0.03515592962503433, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.03578, "step": 37, "tokens/total": 1256336, "tokens/train_per_sec_per_gpu": 25.36, "tokens/trainable": 16799} +{"epoch": 0.1484375, "grad_norm": 0.5646340250968933, "learning_rate": 9.986523722403528e-05, "loss": 0.018402809277176857, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.01857, "step": 38, "tokens/total": 1288304, "tokens/train_per_sec_per_gpu": 27.13, "tokens/trainable": 17256} +{"epoch": 0.15234375, "grad_norm": 1.639113187789917, "learning_rate": 9.984185462146642e-05, "loss": 0.047837480902671814, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.41, "memory/max_allocated (GiB)": 37.41, "ppl": 1.049, "step": 39, "tokens/total": 1323440, "tokens/train_per_sec_per_gpu": 26.06, "tokens/trainable": 17701} +{"epoch": 0.15625, "grad_norm": 0.7143563032150269, "learning_rate": 9.98166059662897e-05, "loss": 0.020293183624744415, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 35.95, "memory/max_allocated (GiB)": 35.95, "ppl": 1.0205, "step": 40, "tokens/total": 1354576, "tokens/train_per_sec_per_gpu": 29.93, "tokens/trainable": 18168} +{"epoch": 0.16015625, "grad_norm": 0.5395357012748718, "learning_rate": 9.978949230920472e-05, "loss": 0.020471245050430298, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.02068, "step": 41, "tokens/total": 1389264, "tokens/train_per_sec_per_gpu": 25.51, "tokens/trainable": 18622} +{"epoch": 0.1640625, "grad_norm": 1.600313425064087, "learning_rate": 9.976051477852141e-05, "loss": 0.0379120297729969, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.03864, "step": 42, "tokens/total": 1422496, "tokens/train_per_sec_per_gpu": 26.81, "tokens/trainable": 19063} +{"epoch": 0.16796875, "grad_norm": 2.0575921535491943, "learning_rate": 9.972967458011312e-05, "loss": 0.03661457821726799, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 38.91, "memory/max_allocated (GiB)": 38.91, "ppl": 1.03729, "step": 43, "tokens/total": 1457856, "tokens/train_per_sec_per_gpu": 27.95, "tokens/trainable": 19536} +{"epoch": 0.171875, "grad_norm": 0.9942023158073425, "learning_rate": 9.96969729973664e-05, "loss": 0.03418630734086037, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.03478, "step": 44, "tokens/total": 1496720, "tokens/train_per_sec_per_gpu": 23.97, "tokens/trainable": 20001} +{"epoch": 0.17578125, "grad_norm": 0.5923479199409485, "learning_rate": 9.966241139112754e-05, "loss": 0.015467006713151932, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.1, "memory/max_allocated (GiB)": 38.1, "ppl": 1.01559, "step": 45, "tokens/total": 1530416, "tokens/train_per_sec_per_gpu": 26.25, "tokens/trainable": 20442} +{"epoch": 0.1796875, "grad_norm": 0.5388168692588806, "learning_rate": 9.96259911996461e-05, "loss": 0.017977243289351463, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.01814, "step": 46, "tokens/total": 1564576, "tokens/train_per_sec_per_gpu": 25.71, "tokens/trainable": 20874} +{"epoch": 0.18359375, "grad_norm": 0.6641814112663269, "learning_rate": 9.958771393851491e-05, "loss": 0.025976376608014107, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 36.49, "memory/max_allocated (GiB)": 36.49, "ppl": 1.02632, "step": 47, "tokens/total": 1596032, "tokens/train_per_sec_per_gpu": 25.49, "tokens/trainable": 21315} +{"epoch": 0.1875, "grad_norm": 1.1087579727172852, "learning_rate": 9.954758120060702e-05, "loss": 0.03688844293355942, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.03758, "step": 48, "tokens/total": 1632304, "tokens/train_per_sec_per_gpu": 24.85, "tokens/trainable": 21775} +{"epoch": 0.19140625, "grad_norm": 0.5927925705909729, "learning_rate": 9.950559465600948e-05, "loss": 0.01667950488626957, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 1.01682, "step": 49, "tokens/total": 1666560, "tokens/train_per_sec_per_gpu": 26.59, "tokens/trainable": 22215} +{"epoch": 0.1953125, "grad_norm": 0.7310757637023926, "learning_rate": 9.946175605195379e-05, "loss": 0.00819910503923893, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.00823, "step": 50, "tokens/total": 1706112, "tokens/train_per_sec_per_gpu": 22.19, "tokens/trainable": 22700} +{"epoch": 0.19921875, "grad_norm": 1.163381814956665, "learning_rate": 9.941606721274322e-05, "loss": 0.0344717875123024, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.03507, "step": 51, "tokens/total": 1740320, "tokens/train_per_sec_per_gpu": 23.19, "tokens/trainable": 23135} +{"epoch": 0.203125, "grad_norm": 0.5380160808563232, "learning_rate": 9.936853003967685e-05, "loss": 0.01729687675833702, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.01745, "step": 52, "tokens/total": 1777552, "tokens/train_per_sec_per_gpu": 24.87, "tokens/trainable": 23609} +{"epoch": 0.20703125, "grad_norm": 1.1088097095489502, "learning_rate": 9.93191465109705e-05, "loss": 0.017511140555143356, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.01767, "step": 53, "tokens/total": 1814192, "tokens/train_per_sec_per_gpu": 24.03, "tokens/trainable": 24042} +{"epoch": 0.2109375, "grad_norm": 0.6329374313354492, "learning_rate": 9.926791868167438e-05, "loss": 0.009039624594151974, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00908, "step": 54, "tokens/total": 1850688, "tokens/train_per_sec_per_gpu": 24.39, "tokens/trainable": 24468} +{"epoch": 0.21484375, "grad_norm": 0.37217894196510315, "learning_rate": 9.921484868358753e-05, "loss": 0.022843321785330772, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 1.02311, "step": 55, "tokens/total": 1886704, "tokens/train_per_sec_per_gpu": 24.63, "tokens/trainable": 24924} +{"epoch": 0.21875, "grad_norm": 0.6057831645011902, "learning_rate": 9.915993872516924e-05, "loss": 0.018378181383013725, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.01855, "step": 56, "tokens/total": 1922528, "tokens/train_per_sec_per_gpu": 23.42, "tokens/trainable": 25368} +{"epoch": 0.22265625, "grad_norm": 0.3160157799720764, "learning_rate": 9.9103191091447e-05, "loss": 0.009609424509108067, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00966, "step": 57, "tokens/total": 1954592, "tokens/train_per_sec_per_gpu": 26.67, "tokens/trainable": 25807} +{"epoch": 0.2265625, "grad_norm": 0.37863293290138245, "learning_rate": 9.904460814392147e-05, "loss": 0.009412367828190327, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.00946, "step": 58, "tokens/total": 1993968, "tokens/train_per_sec_per_gpu": 22.86, "tokens/trainable": 26264} +{"epoch": 0.23046875, "grad_norm": 1.0830192565917969, "learning_rate": 9.898419232046825e-05, "loss": 0.029650621116161346, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.03009, "step": 59, "tokens/total": 2030672, "tokens/train_per_sec_per_gpu": 27.45, "tokens/trainable": 26767} +{"epoch": 0.234375, "grad_norm": 0.3087652325630188, "learning_rate": 9.892194613523633e-05, "loss": 0.008214237168431282, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.26, "memory/max_allocated (GiB)": 38.26, "ppl": 1.00825, "step": 60, "tokens/total": 2066528, "tokens/train_per_sec_per_gpu": 24.54, "tokens/trainable": 27226} +{"epoch": 0.23828125, "grad_norm": 0.33283793926239014, "learning_rate": 9.885787217854357e-05, "loss": 0.009081660769879818, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.00912, "step": 61, "tokens/total": 2100112, "tokens/train_per_sec_per_gpu": 26.72, "tokens/trainable": 27685} +{"epoch": 0.2421875, "grad_norm": 1.0168945789337158, "learning_rate": 9.879197311676887e-05, "loss": 0.03059535287320614, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.03107, "step": 62, "tokens/total": 2136160, "tokens/train_per_sec_per_gpu": 24.49, "tokens/trainable": 28125} +{"epoch": 0.24609375, "grad_norm": 0.5256161689758301, "learning_rate": 9.872425169224113e-05, "loss": 0.02003590017557144, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.02024, "step": 63, "tokens/total": 2169952, "tokens/train_per_sec_per_gpu": 30.13, "tokens/trainable": 28623} +{"epoch": 0.25, "grad_norm": 0.8591369986534119, "learning_rate": 9.865471072312528e-05, "loss": 0.03773169219493866, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.14, "memory/max_allocated (GiB)": 38.14, "ppl": 1.03845, "step": 64, "tokens/total": 2206416, "tokens/train_per_sec_per_gpu": 26.22, "tokens/trainable": 29108} +{"epoch": 0.25390625, "grad_norm": 0.49275705218315125, "learning_rate": 9.858335310330492e-05, "loss": 0.015322866849601269, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.01544, "step": 65, "tokens/total": 2238592, "tokens/train_per_sec_per_gpu": 26.79, "tokens/trainable": 29570} +{"epoch": 0.2578125, "grad_norm": 0.9574518203735352, "learning_rate": 9.851018180226185e-05, "loss": 0.01718294806778431, "memory/device_reserved (GiB)": 40.31, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.01733, "step": 66, "tokens/total": 2277136, "tokens/train_per_sec_per_gpu": 21.73, "tokens/trainable": 30011} +{"epoch": 0.26171875, "grad_norm": 0.2401292771100998, "learning_rate": 9.843519986495259e-05, "loss": 0.008578283712267876, "memory/device_reserved (GiB)": 41.07, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.00862, "step": 67, "tokens/total": 2311872, "tokens/train_per_sec_per_gpu": 25.66, "tokens/trainable": 30446} +{"epoch": 0.265625, "grad_norm": 0.7221901416778564, "learning_rate": 9.835841041168162e-05, "loss": 0.014417489059269428, "memory/device_reserved (GiB)": 41.32, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.01452, "step": 68, "tokens/total": 2349008, "tokens/train_per_sec_per_gpu": 24.93, "tokens/trainable": 30913} +{"epoch": 0.26953125, "grad_norm": 0.36184149980545044, "learning_rate": 9.82798166379715e-05, "loss": 0.009132018312811852, "memory/device_reserved (GiB)": 41.32, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.00917, "step": 69, "tokens/total": 2382368, "tokens/train_per_sec_per_gpu": 28.91, "tokens/trainable": 31403} +{"epoch": 0.2734375, "grad_norm": 0.7220876812934875, "learning_rate": 9.819942181443002e-05, "loss": 0.020566772669553757, "memory/device_reserved (GiB)": 41.32, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.02078, "step": 70, "tokens/total": 2419648, "tokens/train_per_sec_per_gpu": 22.1, "tokens/trainable": 31844} +{"epoch": 0.27734375, "grad_norm": 0.5446144938468933, "learning_rate": 9.811722928661392e-05, "loss": 0.01670904830098152, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 35.81, "memory/max_allocated (GiB)": 35.81, "ppl": 1.01685, "step": 71, "tokens/total": 2448576, "tokens/train_per_sec_per_gpu": 26.86, "tokens/trainable": 32236} +{"epoch": 0.28125, "grad_norm": 0.9056335091590881, "learning_rate": 9.803324247488975e-05, "loss": 0.015748703852295876, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.01587, "step": 72, "tokens/total": 2484240, "tokens/train_per_sec_per_gpu": 24.34, "tokens/trainable": 32688} +{"epoch": 0.28515625, "grad_norm": 0.5962861776351929, "learning_rate": 9.794746487429161e-05, "loss": 0.009003115817904472, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 36.88, "memory/max_allocated (GiB)": 36.88, "ppl": 1.00904, "step": 73, "tokens/total": 2517184, "tokens/train_per_sec_per_gpu": 27.46, "tokens/trainable": 33131} +{"epoch": 0.2890625, "grad_norm": 0.6051144599914551, "learning_rate": 9.785990005437554e-05, "loss": 0.014850958250463009, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.01496, "step": 74, "tokens/total": 2553056, "tokens/train_per_sec_per_gpu": 25.68, "tokens/trainable": 33592} +{"epoch": 0.29296875, "grad_norm": 0.3479956388473511, "learning_rate": 9.777055165907117e-05, "loss": 0.021193083375692368, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.02142, "step": 75, "tokens/total": 2585584, "tokens/train_per_sec_per_gpu": 29.33, "tokens/trainable": 34023} +{"epoch": 0.296875, "grad_norm": 2.8564348220825195, "learning_rate": 9.767942340652993e-05, "loss": 0.009471206925809383, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.07, "memory/max_allocated (GiB)": 38.07, "ppl": 1.00952, "step": 76, "tokens/total": 2620448, "tokens/train_per_sec_per_gpu": 22.8, "tokens/trainable": 34449} +{"epoch": 0.30078125, "grad_norm": 0.8364914655685425, "learning_rate": 9.758651908897035e-05, "loss": 0.026995249092578888, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.02736, "step": 77, "tokens/total": 2653840, "tokens/train_per_sec_per_gpu": 28.56, "tokens/trainable": 34894} +{"epoch": 0.3046875, "grad_norm": 0.7494442462921143, "learning_rate": 9.749184257252033e-05, "loss": 0.030696813017129898, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.03117, "step": 78, "tokens/total": 2688112, "tokens/train_per_sec_per_gpu": 25.01, "tokens/trainable": 35334} +{"epoch": 0.30859375, "grad_norm": 0.3953481912612915, "learning_rate": 9.739539779705614e-05, "loss": 0.014470797032117844, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.01458, "step": 79, "tokens/total": 2726384, "tokens/train_per_sec_per_gpu": 24.99, "tokens/trainable": 35829} +{"epoch": 0.3125, "grad_norm": 0.9291770458221436, "learning_rate": 9.729718877603861e-05, "loss": 0.014526969753205776, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.01463, "step": 80, "tokens/total": 2760176, "tokens/train_per_sec_per_gpu": 27.22, "tokens/trainable": 36260} +{"epoch": 0.31640625, "grad_norm": 0.806298017501831, "learning_rate": 9.719721959634592e-05, "loss": 0.03458312153816223, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.03519, "step": 81, "tokens/total": 2795392, "tokens/train_per_sec_per_gpu": 23.71, "tokens/trainable": 36697} +{"epoch": 0.3203125, "grad_norm": 0.2980300486087799, "learning_rate": 9.709549441810375e-05, "loss": 0.01064860075712204, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.55, "memory/max_allocated (GiB)": 39.55, "ppl": 1.01071, "step": 82, "tokens/total": 2833008, "tokens/train_per_sec_per_gpu": 27.47, "tokens/trainable": 37181} +{"epoch": 0.32421875, "grad_norm": 0.37905657291412354, "learning_rate": 9.699201747451195e-05, "loss": 0.01538780052214861, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.01551, "step": 83, "tokens/total": 2865600, "tokens/train_per_sec_per_gpu": 25.53, "tokens/trainable": 37629} +{"epoch": 0.328125, "grad_norm": 0.26218873262405396, "learning_rate": 9.688679307166854e-05, "loss": 0.005745875649154186, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00576, "step": 84, "tokens/total": 2898400, "tokens/train_per_sec_per_gpu": 28.88, "tokens/trainable": 38092} +{"epoch": 0.33203125, "grad_norm": 0.6269100904464722, "learning_rate": 9.677982558839042e-05, "loss": 0.014924520626664162, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.01504, "step": 85, "tokens/total": 2931696, "tokens/train_per_sec_per_gpu": 27.53, "tokens/trainable": 38571} +{"epoch": 0.3359375, "grad_norm": 0.3799930810928345, "learning_rate": 9.66711194760312e-05, "loss": 0.00951866339892149, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.00956, "step": 86, "tokens/total": 2969104, "tokens/train_per_sec_per_gpu": 25.94, "tokens/trainable": 39059} +{"epoch": 0.33984375, "grad_norm": 2.5636558532714844, "learning_rate": 9.656067925829593e-05, "loss": 0.03618961572647095, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.03685, "step": 87, "tokens/total": 2999856, "tokens/train_per_sec_per_gpu": 31.72, "tokens/trainable": 39519} +{"epoch": 0.34375, "grad_norm": 0.45903822779655457, "learning_rate": 9.644850953105288e-05, "loss": 0.014440367929637432, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 35.78, "memory/max_allocated (GiB)": 35.78, "ppl": 1.01455, "step": 88, "tokens/total": 3030208, "tokens/train_per_sec_per_gpu": 26.66, "tokens/trainable": 39961} +{"epoch": 0.34765625, "grad_norm": 0.5246562361717224, "learning_rate": 9.633461496214225e-05, "loss": 0.009300010278820992, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.00934, "step": 89, "tokens/total": 3062912, "tokens/train_per_sec_per_gpu": 27.08, "tokens/trainable": 40431} +{"epoch": 0.3515625, "grad_norm": 0.8570393323898315, "learning_rate": 9.621900029118195e-05, "loss": 0.03629235178232193, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.04, "memory/max_allocated (GiB)": 36.04, "ppl": 1.03696, "step": 90, "tokens/total": 3092592, "tokens/train_per_sec_per_gpu": 26.15, "tokens/trainable": 40854} +{"epoch": 0.35546875, "grad_norm": 0.38410094380378723, "learning_rate": 9.610167032937036e-05, "loss": 0.00523430947214365, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00525, "step": 91, "tokens/total": 3130880, "tokens/train_per_sec_per_gpu": 26.64, "tokens/trainable": 41373} +{"epoch": 0.359375, "grad_norm": 1.0071769952774048, "learning_rate": 9.598262995928611e-05, "loss": 0.020301248878240585, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.02051, "step": 92, "tokens/total": 3162080, "tokens/train_per_sec_per_gpu": 30.86, "tokens/trainable": 41842} +{"epoch": 0.36328125, "grad_norm": 0.17554351687431335, "learning_rate": 9.586188413468492e-05, "loss": 0.005432227626442909, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00545, "step": 93, "tokens/total": 3200272, "tokens/train_per_sec_per_gpu": 23.41, "tokens/trainable": 42288} +{"epoch": 0.3671875, "grad_norm": 0.23384776711463928, "learning_rate": 9.57394378802934e-05, "loss": 0.006457807030528784, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.00648, "step": 94, "tokens/total": 3233584, "tokens/train_per_sec_per_gpu": 28.0, "tokens/trainable": 42740} +{"epoch": 0.37109375, "grad_norm": 0.4765954911708832, "learning_rate": 9.56152962916e-05, "loss": 0.014569003134965897, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.26, "memory/max_allocated (GiB)": 36.26, "ppl": 1.01468, "step": 95, "tokens/total": 3264144, "tokens/train_per_sec_per_gpu": 27.31, "tokens/trainable": 43138} +{"epoch": 0.375, "grad_norm": 0.5535119771957397, "learning_rate": 9.548946453464296e-05, "loss": 0.021773388609290123, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.02201, "step": 96, "tokens/total": 3298240, "tokens/train_per_sec_per_gpu": 27.46, "tokens/trainable": 43623} +{"epoch": 0.37890625, "grad_norm": 0.497976690530777, "learning_rate": 9.53619478457953e-05, "loss": 0.02203628420829773, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.02228, "step": 97, "tokens/total": 3333136, "tokens/train_per_sec_per_gpu": 24.54, "tokens/trainable": 44079} +{"epoch": 0.3828125, "grad_norm": 0.5903464555740356, "learning_rate": 9.523275153154695e-05, "loss": 0.013729186728596687, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.01382, "step": 98, "tokens/total": 3365536, "tokens/train_per_sec_per_gpu": 25.73, "tokens/trainable": 44529} +{"epoch": 0.38671875, "grad_norm": 0.36282938718795776, "learning_rate": 9.51018809682839e-05, "loss": 0.018156204372644424, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.01832, "step": 99, "tokens/total": 3403200, "tokens/train_per_sec_per_gpu": 26.17, "tokens/trainable": 44976} +{"epoch": 0.390625, "grad_norm": 0.7421016097068787, "learning_rate": 9.49693416020645e-05, "loss": 0.03313370794057846, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.53, "memory/max_allocated (GiB)": 38.53, "ppl": 1.03369, "step": 100, "tokens/total": 3440944, "tokens/train_per_sec_per_gpu": 25.02, "tokens/trainable": 45429} +{"epoch": 0.39453125, "grad_norm": 0.627457857131958, "learning_rate": 9.483513894839276e-05, "loss": 0.009754067286849022, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.0098, "step": 101, "tokens/total": 3474992, "tokens/train_per_sec_per_gpu": 25.8, "tokens/trainable": 45876} +{"epoch": 0.3984375, "grad_norm": 0.6103981733322144, "learning_rate": 9.469927859198888e-05, "loss": 0.015758540481328964, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.01588, "step": 102, "tokens/total": 3510912, "tokens/train_per_sec_per_gpu": 29.69, "tokens/trainable": 46363} +{"epoch": 0.40234375, "grad_norm": 0.3177488148212433, "learning_rate": 9.456176618655689e-05, "loss": 0.011776598170399666, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.47, "memory/max_allocated (GiB)": 36.47, "ppl": 1.01185, "step": 103, "tokens/total": 3542960, "tokens/train_per_sec_per_gpu": 28.11, "tokens/trainable": 46810} +{"epoch": 0.40625, "grad_norm": 0.8762432932853699, "learning_rate": 9.442260745454927e-05, "loss": 0.01699967496097088, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.01714, "step": 104, "tokens/total": 3578336, "tokens/train_per_sec_per_gpu": 26.43, "tokens/trainable": 47287} +{"epoch": 0.41015625, "grad_norm": 0.40428024530410767, "learning_rate": 9.428180818692884e-05, "loss": 0.009235890582203865, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.45, "memory/max_allocated (GiB)": 38.45, "ppl": 1.00928, "step": 105, "tokens/total": 3614496, "tokens/train_per_sec_per_gpu": 25.24, "tokens/trainable": 47788} +{"epoch": 0.4140625, "grad_norm": 0.3176383972167969, "learning_rate": 9.413937424292791e-05, "loss": 0.010860669426620007, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.01092, "step": 106, "tokens/total": 3653632, "tokens/train_per_sec_per_gpu": 22.12, "tokens/trainable": 48243} +{"epoch": 0.41796875, "grad_norm": 0.9417267441749573, "learning_rate": 9.399531154980424e-05, "loss": 0.004337538965046406, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 1.00435, "step": 107, "tokens/total": 3690208, "tokens/train_per_sec_per_gpu": 24.59, "tokens/trainable": 48692} +{"epoch": 0.421875, "grad_norm": 0.5850103497505188, "learning_rate": 9.384962610259455e-05, "loss": 0.016877641901373863, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.01702, "step": 108, "tokens/total": 3724864, "tokens/train_per_sec_per_gpu": 24.68, "tokens/trainable": 49142} +{"epoch": 0.42578125, "grad_norm": 0.40970921516418457, "learning_rate": 9.370232396386494e-05, "loss": 0.006546959280967712, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.22, "memory/max_allocated (GiB)": 37.22, "ppl": 1.00657, "step": 109, "tokens/total": 3757952, "tokens/train_per_sec_per_gpu": 26.94, "tokens/trainable": 49583} +{"epoch": 0.4296875, "grad_norm": 0.6953180432319641, "learning_rate": 9.355341126345868e-05, "loss": 0.017325246706604958, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.01748, "step": 110, "tokens/total": 3793232, "tokens/train_per_sec_per_gpu": 25.99, "tokens/trainable": 50055} +{"epoch": 0.43359375, "grad_norm": 0.4403698742389679, "learning_rate": 9.340289419824107e-05, "loss": 0.005817159079015255, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.00583, "step": 111, "tokens/total": 3825552, "tokens/train_per_sec_per_gpu": 28.69, "tokens/trainable": 50500} +{"epoch": 0.4375, "grad_norm": 0.36160168051719666, "learning_rate": 9.325077903184159e-05, "loss": 0.007664479780942202, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00769, "step": 112, "tokens/total": 3860544, "tokens/train_per_sec_per_gpu": 24.37, "tokens/trainable": 50950} +{"epoch": 0.44140625, "grad_norm": 0.06424324959516525, "learning_rate": 9.30970720943932e-05, "loss": 0.0005713118007406592, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.15, "memory/max_allocated (GiB)": 38.15, "ppl": 1.00057, "step": 113, "tokens/total": 3895552, "tokens/train_per_sec_per_gpu": 24.97, "tokens/trainable": 51389} +{"epoch": 0.4453125, "grad_norm": 0.5930986404418945, "learning_rate": 9.2941779782269e-05, "loss": 0.012876790948212147, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.01296, "step": 114, "tokens/total": 3929152, "tokens/train_per_sec_per_gpu": 25.96, "tokens/trainable": 51862} +{"epoch": 0.44921875, "grad_norm": 1.0587154626846313, "learning_rate": 9.278490855781596e-05, "loss": 0.007580373901873827, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.24, "memory/max_allocated (GiB)": 37.24, "ppl": 1.00761, "step": 115, "tokens/total": 3961472, "tokens/train_per_sec_per_gpu": 30.52, "tokens/trainable": 52301} +{"epoch": 0.453125, "grad_norm": 0.24098381400108337, "learning_rate": 9.262646494908604e-05, "loss": 0.0013159073423594236, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00132, "step": 116, "tokens/total": 4000128, "tokens/train_per_sec_per_gpu": 24.11, "tokens/trainable": 52755} +{"epoch": 0.45703125, "grad_norm": 0.9265182018280029, "learning_rate": 9.246645554956457e-05, "loss": 0.006100953556597233, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00612, "step": 117, "tokens/total": 4035152, "tokens/train_per_sec_per_gpu": 25.5, "tokens/trainable": 53224} +{"epoch": 0.4609375, "grad_norm": 0.5254908204078674, "learning_rate": 9.230488701789578e-05, "loss": 0.01959427446126938, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.01979, "step": 118, "tokens/total": 4072208, "tokens/train_per_sec_per_gpu": 23.92, "tokens/trainable": 53666} +{"epoch": 0.46484375, "grad_norm": 0.5372349619865417, "learning_rate": 9.214176607760577e-05, "loss": 0.005049949511885643, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.00506, "step": 119, "tokens/total": 4110464, "tokens/train_per_sec_per_gpu": 24.5, "tokens/trainable": 54129} +{"epoch": 0.46875, "grad_norm": 0.4335256516933441, "learning_rate": 9.197709951682268e-05, "loss": 0.003938719164580107, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.00395, "step": 120, "tokens/total": 4142896, "tokens/train_per_sec_per_gpu": 29.53, "tokens/trainable": 54587} +{"epoch": 0.47265625, "grad_norm": 1.0627281665802002, "learning_rate": 9.181089418799428e-05, "loss": 0.03759939968585968, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.03832, "step": 121, "tokens/total": 4179360, "tokens/train_per_sec_per_gpu": 27.74, "tokens/trainable": 55070} +{"epoch": 0.4765625, "grad_norm": 0.3846990168094635, "learning_rate": 9.164315700760271e-05, "loss": 0.020668094977736473, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.02088, "step": 122, "tokens/total": 4215040, "tokens/train_per_sec_per_gpu": 23.65, "tokens/trainable": 55494} +{"epoch": 0.48046875, "grad_norm": 1.0273499488830566, "learning_rate": 9.147389495587671e-05, "loss": 0.046010032296180725, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.04708, "step": 123, "tokens/total": 4246448, "tokens/train_per_sec_per_gpu": 31.43, "tokens/trainable": 55940} +{"epoch": 0.484375, "grad_norm": 0.6916744709014893, "learning_rate": 9.130311507650116e-05, "loss": 0.01927914284169674, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.01947, "step": 124, "tokens/total": 4278832, "tokens/train_per_sec_per_gpu": 27.31, "tokens/trainable": 56404} +{"epoch": 0.48828125, "grad_norm": 0.4196462035179138, "learning_rate": 9.113082447632394e-05, "loss": 0.013451140373945236, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.01354, "step": 125, "tokens/total": 4310896, "tokens/train_per_sec_per_gpu": 33.47, "tokens/trainable": 56878} +{"epoch": 0.4921875, "grad_norm": 0.5104981660842896, "learning_rate": 9.09570303250602e-05, "loss": 0.025703173130750656, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.02604, "step": 126, "tokens/total": 4342912, "tokens/train_per_sec_per_gpu": 30.87, "tokens/trainable": 57362} +{"epoch": 0.49609375, "grad_norm": 0.5423718094825745, "learning_rate": 9.078173985499394e-05, "loss": 0.015257916413247585, "memory/device_reserved (GiB)": 42.05, "memory/max_active (GiB)": 35.95, "memory/max_allocated (GiB)": 35.95, "ppl": 1.01537, "step": 127, "tokens/total": 4373312, "tokens/train_per_sec_per_gpu": 28.66, "tokens/trainable": 57792} +{"epoch": 0.5, "grad_norm": 0.32960814237594604, "learning_rate": 9.060496036067713e-05, "loss": 0.016531089320778847, "memory/device_reserved (GiB)": 42.05, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.01667, "step": 128, "tokens/total": 4410816, "tokens/train_per_sec_per_gpu": 22.67, "tokens/trainable": 58216} +{"epoch": 0.50390625, "grad_norm": 0.4497157335281372, "learning_rate": 9.042669919862615e-05, "loss": 0.018336737528443336, "memory/device_reserved (GiB)": 42.05, "memory/max_active (GiB)": 36.77, "memory/max_allocated (GiB)": 36.77, "ppl": 1.01851, "step": 129, "tokens/total": 4442704, "tokens/train_per_sec_per_gpu": 30.33, "tokens/trainable": 58680} +{"epoch": 0.5078125, "grad_norm": 0.47693678736686707, "learning_rate": 9.024696378701557e-05, "loss": 0.022742683067917824, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 39.48, "memory/max_allocated (GiB)": 39.48, "ppl": 1.023, "step": 130, "tokens/total": 4478384, "tokens/train_per_sec_per_gpu": 24.08, "tokens/trainable": 59113} +{"epoch": 0.51171875, "grad_norm": 0.3063410222530365, "learning_rate": 9.006576160536948e-05, "loss": 0.018128667026758194, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.01829, "step": 131, "tokens/total": 4513328, "tokens/train_per_sec_per_gpu": 24.55, "tokens/trainable": 59562} +{"epoch": 0.515625, "grad_norm": 0.22839857637882233, "learning_rate": 8.988310019425035e-05, "loss": 0.010266811586916447, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.01032, "step": 132, "tokens/total": 4550832, "tokens/train_per_sec_per_gpu": 24.34, "tokens/trainable": 60006} +{"epoch": 0.51953125, "grad_norm": 0.35292136669158936, "learning_rate": 8.969898715494506e-05, "loss": 0.009890235960483551, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00994, "step": 133, "tokens/total": 4587968, "tokens/train_per_sec_per_gpu": 25.95, "tokens/trainable": 60487} +{"epoch": 0.5234375, "grad_norm": 0.4637380540370941, "learning_rate": 8.951343014914869e-05, "loss": 0.015284349210560322, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.0154, "step": 134, "tokens/total": 4624544, "tokens/train_per_sec_per_gpu": 23.76, "tokens/trainable": 60953} +{"epoch": 0.52734375, "grad_norm": 0.24201437830924988, "learning_rate": 8.932643689864568e-05, "loss": 0.004285085014998913, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00429, "step": 135, "tokens/total": 4661296, "tokens/train_per_sec_per_gpu": 27.49, "tokens/trainable": 61440} +{"epoch": 0.53125, "grad_norm": 0.37105661630630493, "learning_rate": 8.913801518498845e-05, "loss": 0.010507537983357906, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.01056, "step": 136, "tokens/total": 4700128, "tokens/train_per_sec_per_gpu": 21.5, "tokens/trainable": 61901} +{"epoch": 0.53515625, "grad_norm": 1.903419852256775, "learning_rate": 8.894817284917364e-05, "loss": 0.013552292250096798, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.01364, "step": 137, "tokens/total": 4736912, "tokens/train_per_sec_per_gpu": 22.98, "tokens/trainable": 62355} +{"epoch": 0.5390625, "grad_norm": 0.36268219351768494, "learning_rate": 8.875691779131569e-05, "loss": 0.004295737482607365, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.0043, "step": 138, "tokens/total": 4774192, "tokens/train_per_sec_per_gpu": 20.93, "tokens/trainable": 62786} +{"epoch": 0.54296875, "grad_norm": 0.274997353553772, "learning_rate": 8.856425797031829e-05, "loss": 0.005457881838083267, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00547, "step": 139, "tokens/total": 4809344, "tokens/train_per_sec_per_gpu": 26.0, "tokens/trainable": 63228} +{"epoch": 0.546875, "grad_norm": 1.764664888381958, "learning_rate": 8.837020140354295e-05, "loss": 0.03942972794175148, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.04022, "step": 140, "tokens/total": 4845712, "tokens/train_per_sec_per_gpu": 27.06, "tokens/trainable": 63693} +{"epoch": 0.55078125, "grad_norm": 0.18542014062404633, "learning_rate": 8.817475616647554e-05, "loss": 0.0040616258047521114, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.00407, "step": 141, "tokens/total": 4880368, "tokens/train_per_sec_per_gpu": 26.76, "tokens/trainable": 64159} +{"epoch": 0.5546875, "grad_norm": 0.40749433636665344, "learning_rate": 8.797793039239017e-05, "loss": 0.006734668277204037, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00676, "step": 142, "tokens/total": 4915424, "tokens/train_per_sec_per_gpu": 26.25, "tokens/trainable": 64608} +{"epoch": 0.55859375, "grad_norm": 0.20362620055675507, "learning_rate": 8.777973227201069e-05, "loss": 0.0035807564854621887, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.00359, "step": 143, "tokens/total": 4951744, "tokens/train_per_sec_per_gpu": 28.03, "tokens/trainable": 65117} +{"epoch": 0.5625, "grad_norm": 0.27554887533187866, "learning_rate": 8.758017005316988e-05, "loss": 0.005913868546485901, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.00593, "step": 144, "tokens/total": 4985152, "tokens/train_per_sec_per_gpu": 28.52, "tokens/trainable": 65542} +{"epoch": 0.56640625, "grad_norm": 0.4591808021068573, "learning_rate": 8.737925204046629e-05, "loss": 0.009290485642850399, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00933, "step": 145, "tokens/total": 5019936, "tokens/train_per_sec_per_gpu": 24.22, "tokens/trainable": 65992} +{"epoch": 0.5703125, "grad_norm": 0.49842438101768494, "learning_rate": 8.717698659491851e-05, "loss": 0.00957406498491764, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00962, "step": 146, "tokens/total": 5057024, "tokens/train_per_sec_per_gpu": 22.58, "tokens/trainable": 66449} +{"epoch": 0.57421875, "grad_norm": 0.4455566704273224, "learning_rate": 8.697338213361735e-05, "loss": 0.010963148437440395, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.01102, "step": 147, "tokens/total": 5088320, "tokens/train_per_sec_per_gpu": 26.31, "tokens/trainable": 66897} +{"epoch": 0.578125, "grad_norm": 0.6725348234176636, "learning_rate": 8.676844712937552e-05, "loss": 0.007718087173998356, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.39, "memory/max_allocated (GiB)": 39.39, "ppl": 1.00775, "step": 148, "tokens/total": 5128064, "tokens/train_per_sec_per_gpu": 25.02, "tokens/trainable": 67369} +{"epoch": 0.58203125, "grad_norm": 0.3007007837295532, "learning_rate": 8.656219011037509e-05, "loss": 0.0055345334112644196, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00555, "step": 149, "tokens/total": 5165104, "tokens/train_per_sec_per_gpu": 23.67, "tokens/trainable": 67857} +{"epoch": 0.5859375, "grad_norm": 0.40263986587524414, "learning_rate": 8.63546196598125e-05, "loss": 0.01335815154016018, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.01345, "step": 150, "tokens/total": 5202336, "tokens/train_per_sec_per_gpu": 25.25, "tokens/trainable": 68305} +{"epoch": 0.58984375, "grad_norm": 0.4136700928211212, "learning_rate": 8.614574441554145e-05, "loss": 0.011366650462150574, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.13, "memory/max_allocated (GiB)": 37.13, "ppl": 1.01143, "step": 151, "tokens/total": 5235440, "tokens/train_per_sec_per_gpu": 27.48, "tokens/trainable": 68730} +{"epoch": 0.59375, "grad_norm": 0.7672761678695679, "learning_rate": 8.593557306971349e-05, "loss": 0.010589499026536942, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.44, "memory/max_allocated (GiB)": 37.44, "ppl": 1.01065, "step": 152, "tokens/total": 5268416, "tokens/train_per_sec_per_gpu": 26.11, "tokens/trainable": 69185} +{"epoch": 0.59765625, "grad_norm": 0.15939071774482727, "learning_rate": 8.572411436841618e-05, "loss": 0.003426568815484643, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00343, "step": 153, "tokens/total": 5302128, "tokens/train_per_sec_per_gpu": 26.67, "tokens/trainable": 69660} +{"epoch": 0.6015625, "grad_norm": 0.31316232681274414, "learning_rate": 8.551137711130922e-05, "loss": 0.006957028992474079, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.2, "memory/max_allocated (GiB)": 38.2, "ppl": 1.00698, "step": 154, "tokens/total": 5333520, "tokens/train_per_sec_per_gpu": 24.0, "tokens/trainable": 70066} +{"epoch": 0.60546875, "grad_norm": 0.21604987978935242, "learning_rate": 8.529737015125824e-05, "loss": 0.0037870071828365326, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00379, "step": 155, "tokens/total": 5367968, "tokens/train_per_sec_per_gpu": 25.95, "tokens/trainable": 70532} +{"epoch": 0.609375, "grad_norm": 0.3136327266693115, "learning_rate": 8.508210239396639e-05, "loss": 0.013233241625130177, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.01332, "step": 156, "tokens/total": 5405536, "tokens/train_per_sec_per_gpu": 24.32, "tokens/trainable": 70981} +{"epoch": 0.61328125, "grad_norm": 0.3275722861289978, "learning_rate": 8.486558279760375e-05, "loss": 0.005557620897889137, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 36.07, "memory/max_allocated (GiB)": 36.07, "ppl": 1.00557, "step": 157, "tokens/total": 5436672, "tokens/train_per_sec_per_gpu": 26.79, "tokens/trainable": 71407} +{"epoch": 0.6171875, "grad_norm": 0.4884542226791382, "learning_rate": 8.464782037243449e-05, "loss": 0.020114857703447342, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.02032, "step": 158, "tokens/total": 5469280, "tokens/train_per_sec_per_gpu": 27.54, "tokens/trainable": 71848} +{"epoch": 0.62109375, "grad_norm": 0.7068396210670471, "learning_rate": 8.442882418044202e-05, "loss": 0.017435960471630096, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.01759, "step": 159, "tokens/total": 5501904, "tokens/train_per_sec_per_gpu": 29.51, "tokens/trainable": 72325} +{"epoch": 0.625, "grad_norm": 0.711172878742218, "learning_rate": 8.420860333495179e-05, "loss": 0.029129130765795708, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.02956, "step": 160, "tokens/total": 5534944, "tokens/train_per_sec_per_gpu": 26.88, "tokens/trainable": 72739} +{"epoch": 0.62890625, "grad_norm": 0.2796333432197571, "learning_rate": 8.398716700025208e-05, "loss": 0.0085458904504776, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.00858, "step": 161, "tokens/total": 5568160, "tokens/train_per_sec_per_gpu": 25.82, "tokens/trainable": 73191} +{"epoch": 0.6328125, "grad_norm": 0.35786357522010803, "learning_rate": 8.376452439121266e-05, "loss": 0.020331915467977524, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 35.89, "memory/max_allocated (GiB)": 35.89, "ppl": 1.02054, "step": 162, "tokens/total": 5598256, "tokens/train_per_sec_per_gpu": 31.34, "tokens/trainable": 73642} +{"epoch": 0.63671875, "grad_norm": 0.4405795633792877, "learning_rate": 8.354068477290124e-05, "loss": 0.0105100953951478, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.01057, "step": 163, "tokens/total": 5632560, "tokens/train_per_sec_per_gpu": 27.98, "tokens/trainable": 74101} +{"epoch": 0.640625, "grad_norm": 0.4034632444381714, "learning_rate": 8.331565746019807e-05, "loss": 0.01496416237205267, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.52, "memory/max_allocated (GiB)": 36.52, "ppl": 1.01508, "step": 164, "tokens/total": 5664464, "tokens/train_per_sec_per_gpu": 27.34, "tokens/trainable": 74558} +{"epoch": 0.64453125, "grad_norm": 0.8141056299209595, "learning_rate": 8.308945181740812e-05, "loss": 0.026263415813446045, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.02661, "step": 165, "tokens/total": 5699040, "tokens/train_per_sec_per_gpu": 27.46, "tokens/trainable": 75000} +{"epoch": 0.6484375, "grad_norm": 0.23675186932086945, "learning_rate": 8.286207725787153e-05, "loss": 0.011724467389285564, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.01179, "step": 166, "tokens/total": 5735968, "tokens/train_per_sec_per_gpu": 22.64, "tokens/trainable": 75445} +{"epoch": 0.65234375, "grad_norm": 0.4411337375640869, "learning_rate": 8.263354324357182e-05, "loss": 0.0171384084969759, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.01729, "step": 167, "tokens/total": 5770688, "tokens/train_per_sec_per_gpu": 25.61, "tokens/trainable": 75880} +{"epoch": 0.65625, "grad_norm": 0.22989386320114136, "learning_rate": 8.240385928474219e-05, "loss": 0.012283170595765114, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.01236, "step": 168, "tokens/total": 5803824, "tokens/train_per_sec_per_gpu": 27.27, "tokens/trainable": 76294} +{"epoch": 0.66015625, "grad_norm": 0.2097136378288269, "learning_rate": 8.217303493946967e-05, "loss": 0.008362196385860443, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.0084, "step": 169, "tokens/total": 5838256, "tokens/train_per_sec_per_gpu": 26.55, "tokens/trainable": 76742} +{"epoch": 0.6640625, "grad_norm": 0.2178754359483719, "learning_rate": 8.194107981329746e-05, "loss": 0.009995900094509125, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.01005, "step": 170, "tokens/total": 5876224, "tokens/train_per_sec_per_gpu": 23.57, "tokens/trainable": 77196} +{"epoch": 0.66796875, "grad_norm": 0.25040197372436523, "learning_rate": 8.170800355882518e-05, "loss": 0.004473611246794462, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.24, "memory/max_allocated (GiB)": 37.24, "ppl": 1.00448, "step": 171, "tokens/total": 5910448, "tokens/train_per_sec_per_gpu": 30.76, "tokens/trainable": 77663} +{"epoch": 0.671875, "grad_norm": 0.45923876762390137, "learning_rate": 8.147381587530713e-05, "loss": 0.012108156457543373, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.01218, "step": 172, "tokens/total": 5944256, "tokens/train_per_sec_per_gpu": 26.82, "tokens/trainable": 78117} +{"epoch": 0.67578125, "grad_norm": 0.5641582608222961, "learning_rate": 8.123852650824877e-05, "loss": 0.02703103795647621, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.0274, "step": 173, "tokens/total": 5978288, "tokens/train_per_sec_per_gpu": 28.9, "tokens/trainable": 78601} +{"epoch": 0.6796875, "grad_norm": 0.5324582457542419, "learning_rate": 8.100214524900103e-05, "loss": 0.02372906543314457, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.52, "memory/max_allocated (GiB)": 38.52, "ppl": 1.02401, "step": 174, "tokens/total": 6013680, "tokens/train_per_sec_per_gpu": 22.45, "tokens/trainable": 79032} +{"epoch": 0.68359375, "grad_norm": 0.15444296598434448, "learning_rate": 8.076468193435301e-05, "loss": 0.004957557190209627, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.00497, "step": 175, "tokens/total": 6045072, "tokens/train_per_sec_per_gpu": 26.25, "tokens/trainable": 79456} +{"epoch": 0.6875, "grad_norm": 0.26275989413261414, "learning_rate": 8.052614644612253e-05, "loss": 0.008858522400259972, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.0089, "step": 176, "tokens/total": 6082336, "tokens/train_per_sec_per_gpu": 23.58, "tokens/trainable": 79917} +{"epoch": 0.69140625, "grad_norm": 0.3488065302371979, "learning_rate": 8.028654871074489e-05, "loss": 0.01035262644290924, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.01041, "step": 177, "tokens/total": 6117024, "tokens/train_per_sec_per_gpu": 25.76, "tokens/trainable": 80358} +{"epoch": 0.6953125, "grad_norm": 0.05642886087298393, "learning_rate": 8.004589869885986e-05, "loss": 0.0016336295520886779, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.49, "memory/max_allocated (GiB)": 36.49, "ppl": 1.00163, "step": 178, "tokens/total": 6149136, "tokens/train_per_sec_per_gpu": 26.93, "tokens/trainable": 80777} +{"epoch": 0.69921875, "grad_norm": 0.5282962322235107, "learning_rate": 7.980420642489674e-05, "loss": 0.016161344945430756, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.01629, "step": 179, "tokens/total": 6187776, "tokens/train_per_sec_per_gpu": 23.65, "tokens/trainable": 81250} +{"epoch": 0.703125, "grad_norm": 0.7477691769599915, "learning_rate": 7.95614819466576e-05, "loss": 0.0039191958494484425, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00393, "step": 180, "tokens/total": 6221728, "tokens/train_per_sec_per_gpu": 28.6, "tokens/trainable": 81715} +{"epoch": 0.70703125, "grad_norm": 0.8655878305435181, "learning_rate": 7.931773536489872e-05, "loss": 0.025852713733911514, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.02619, "step": 181, "tokens/total": 6255920, "tokens/train_per_sec_per_gpu": 25.2, "tokens/trainable": 82160} +{"epoch": 0.7109375, "grad_norm": 0.5424155592918396, "learning_rate": 7.907297682291035e-05, "loss": 0.009245127439498901, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00929, "step": 182, "tokens/total": 6291104, "tokens/train_per_sec_per_gpu": 26.82, "tokens/trainable": 82624} +{"epoch": 0.71484375, "grad_norm": 0.2275976538658142, "learning_rate": 7.882721650609442e-05, "loss": 0.005258716642856598, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.00527, "step": 183, "tokens/total": 6330976, "tokens/train_per_sec_per_gpu": 23.51, "tokens/trainable": 83125} +{"epoch": 0.71875, "grad_norm": 0.7638486623764038, "learning_rate": 7.85804646415409e-05, "loss": 0.017874060198664665, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.01803, "step": 184, "tokens/total": 6366160, "tokens/train_per_sec_per_gpu": 27.09, "tokens/trainable": 83581} +{"epoch": 0.72265625, "grad_norm": 0.23306654393672943, "learning_rate": 7.833273149760207e-05, "loss": 0.006204688455909491, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.00622, "step": 185, "tokens/total": 6397280, "tokens/train_per_sec_per_gpu": 29.15, "tokens/trainable": 84051} +{"epoch": 0.7265625, "grad_norm": 1.1200511455535889, "learning_rate": 7.808402738346527e-05, "loss": 0.02792154625058174, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.02832, "step": 186, "tokens/total": 6435632, "tokens/train_per_sec_per_gpu": 23.69, "tokens/trainable": 84500} +{"epoch": 0.73046875, "grad_norm": 0.24463903903961182, "learning_rate": 7.783436264872382e-05, "loss": 0.003359630238264799, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.00337, "step": 187, "tokens/total": 6472752, "tokens/train_per_sec_per_gpu": 25.13, "tokens/trainable": 84986} +{"epoch": 0.734375, "grad_norm": 0.13486042618751526, "learning_rate": 7.758374768294647e-05, "loss": 0.004329883959144354, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.05, "memory/max_allocated (GiB)": 38.05, "ppl": 1.00434, "step": 188, "tokens/total": 6508528, "tokens/train_per_sec_per_gpu": 27.3, "tokens/trainable": 85465} +{"epoch": 0.73828125, "grad_norm": 0.17746900022029877, "learning_rate": 7.733219291524489e-05, "loss": 0.007554863579571247, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.59, "memory/max_allocated (GiB)": 37.59, "ppl": 1.00758, "step": 189, "tokens/total": 6542048, "tokens/train_per_sec_per_gpu": 27.83, "tokens/trainable": 85896} +{"epoch": 0.7421875, "grad_norm": 1.2656528949737549, "learning_rate": 7.707970881383977e-05, "loss": 0.016766320914030075, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.57, "memory/max_allocated (GiB)": 36.57, "ppl": 1.01691, "step": 190, "tokens/total": 6573792, "tokens/train_per_sec_per_gpu": 28.46, "tokens/trainable": 86346} +{"epoch": 0.74609375, "grad_norm": 0.4701008200645447, "learning_rate": 7.682630588562518e-05, "loss": 0.01397809199988842, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.01408, "step": 191, "tokens/total": 6608848, "tokens/train_per_sec_per_gpu": 24.62, "tokens/trainable": 86777} +{"epoch": 0.75, "grad_norm": 0.2187153697013855, "learning_rate": 7.657199467573129e-05, "loss": 0.006720630917698145, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.00674, "step": 192, "tokens/total": 6642560, "tokens/train_per_sec_per_gpu": 26.45, "tokens/trainable": 87249} +{"epoch": 0.75390625, "grad_norm": 0.16020628809928894, "learning_rate": 7.631678576708561e-05, "loss": 0.0031050890684127808, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00311, "step": 193, "tokens/total": 6676272, "tokens/train_per_sec_per_gpu": 29.36, "tokens/trainable": 87741} +{"epoch": 0.7578125, "grad_norm": 0.2089877724647522, "learning_rate": 7.606068977997255e-05, "loss": 0.005782723426818848, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.0058, "step": 194, "tokens/total": 6713568, "tokens/train_per_sec_per_gpu": 26.0, "tokens/trainable": 88219} +{"epoch": 0.76171875, "grad_norm": 0.25595805048942566, "learning_rate": 7.580371737159148e-05, "loss": 0.006066588684916496, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00609, "step": 195, "tokens/total": 6749296, "tokens/train_per_sec_per_gpu": 23.23, "tokens/trainable": 88630} +{"epoch": 0.765625, "grad_norm": 0.06450719386339188, "learning_rate": 7.554587923561324e-05, "loss": 0.0016972245648503304, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.0017, "step": 196, "tokens/total": 6783952, "tokens/train_per_sec_per_gpu": 24.9, "tokens/trainable": 89068} +{"epoch": 0.76953125, "grad_norm": 1.3683241605758667, "learning_rate": 7.528718610173511e-05, "loss": 0.02375173009932041, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.48, "memory/max_allocated (GiB)": 37.48, "ppl": 1.02404, "step": 197, "tokens/total": 6819152, "tokens/train_per_sec_per_gpu": 23.11, "tokens/trainable": 89489} +{"epoch": 0.7734375, "grad_norm": 0.16951526701450348, "learning_rate": 7.502764873523431e-05, "loss": 0.0026284372434020042, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.00263, "step": 198, "tokens/total": 6853440, "tokens/train_per_sec_per_gpu": 27.64, "tokens/trainable": 89928} +{"epoch": 0.77734375, "grad_norm": 0.49642816185951233, "learning_rate": 7.476727793652011e-05, "loss": 0.0035167126916348934, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.71, "memory/max_allocated (GiB)": 36.71, "ppl": 1.00352, "step": 199, "tokens/total": 6886272, "tokens/train_per_sec_per_gpu": 28.46, "tokens/trainable": 90407} +{"epoch": 0.78125, "grad_norm": 0.29860493540763855, "learning_rate": 7.450608454068415e-05, "loss": 0.0104782460257411, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.01053, "step": 200, "tokens/total": 6920624, "tokens/train_per_sec_per_gpu": 27.01, "tokens/trainable": 90849} +{"epoch": 0.78515625, "grad_norm": 0.3410024642944336, "learning_rate": 7.424407941704987e-05, "loss": 0.010753445327281952, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.01081, "step": 201, "tokens/total": 6957328, "tokens/train_per_sec_per_gpu": 27.65, "tokens/trainable": 91350} +{"epoch": 0.7890625, "grad_norm": 0.4217333495616913, "learning_rate": 7.398127346871986e-05, "loss": 0.016376987099647522, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.01651, "step": 202, "tokens/total": 6994048, "tokens/train_per_sec_per_gpu": 24.67, "tokens/trainable": 91805} +{"epoch": 0.79296875, "grad_norm": 0.07692840695381165, "learning_rate": 7.371767763212238e-05, "loss": 0.0013561872765421867, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.00136, "step": 203, "tokens/total": 7029040, "tokens/train_per_sec_per_gpu": 26.88, "tokens/trainable": 92258} +{"epoch": 0.796875, "grad_norm": 0.48767411708831787, "learning_rate": 7.345330287655617e-05, "loss": 0.014983436092734337, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.0151, "step": 204, "tokens/total": 7067104, "tokens/train_per_sec_per_gpu": 22.99, "tokens/trainable": 92721} +{"epoch": 0.80078125, "grad_norm": 0.5039049983024597, "learning_rate": 7.31881602037339e-05, "loss": 0.01050379779189825, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.01056, "step": 205, "tokens/total": 7099136, "tokens/train_per_sec_per_gpu": 26.66, "tokens/trainable": 93160} +{"epoch": 0.8046875, "grad_norm": 0.18941731750965118, "learning_rate": 7.29222606473245e-05, "loss": 0.004713146947324276, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.00472, "step": 206, "tokens/total": 7132208, "tokens/train_per_sec_per_gpu": 26.8, "tokens/trainable": 93605} +{"epoch": 0.80859375, "grad_norm": 0.11505293101072311, "learning_rate": 7.265561527249383e-05, "loss": 0.0025927582755684853, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.0026, "step": 207, "tokens/total": 7166256, "tokens/train_per_sec_per_gpu": 28.06, "tokens/trainable": 94041} +{"epoch": 0.8125, "grad_norm": 0.3272874355316162, "learning_rate": 7.238823517544436e-05, "loss": 0.011262031272053719, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.01133, "step": 208, "tokens/total": 7203152, "tokens/train_per_sec_per_gpu": 28.67, "tokens/trainable": 94512} +{"epoch": 0.81640625, "grad_norm": 0.07596582919359207, "learning_rate": 7.212013148295333e-05, "loss": 0.0010702370200306177, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.00107, "step": 209, "tokens/total": 7237264, "tokens/train_per_sec_per_gpu": 30.1, "tokens/trainable": 94994} +{"epoch": 0.8203125, "grad_norm": 0.17019829154014587, "learning_rate": 7.185131535190975e-05, "loss": 0.0015738881193101406, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00158, "step": 210, "tokens/total": 7269664, "tokens/train_per_sec_per_gpu": 30.37, "tokens/trainable": 95503} +{"epoch": 0.82421875, "grad_norm": 0.31330856680870056, "learning_rate": 7.158179796885005e-05, "loss": 0.007932034321129322, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.00796, "step": 211, "tokens/total": 7304080, "tokens/train_per_sec_per_gpu": 27.81, "tokens/trainable": 95954} +{"epoch": 0.828125, "grad_norm": 0.23914340138435364, "learning_rate": 7.131159054949273e-05, "loss": 0.0046081822365522385, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00462, "step": 212, "tokens/total": 7342480, "tokens/train_per_sec_per_gpu": 22.6, "tokens/trainable": 96428} +{"epoch": 0.83203125, "grad_norm": 0.582018256187439, "learning_rate": 7.104070433827139e-05, "loss": 0.012096988968551159, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.01217, "step": 213, "tokens/total": 7378032, "tokens/train_per_sec_per_gpu": 23.34, "tokens/trainable": 96828} +{"epoch": 0.8359375, "grad_norm": 0.07740333676338196, "learning_rate": 7.076915060786705e-05, "loss": 0.00142485904507339, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.96, "memory/max_allocated (GiB)": 37.96, "ppl": 1.00143, "step": 214, "tokens/total": 7410928, "tokens/train_per_sec_per_gpu": 24.55, "tokens/trainable": 97270} +{"epoch": 0.83984375, "grad_norm": 0.2914799153804779, "learning_rate": 7.049694065873882e-05, "loss": 0.011688480153679848, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.01176, "step": 215, "tokens/total": 7445664, "tokens/train_per_sec_per_gpu": 28.28, "tokens/trainable": 97727} +{"epoch": 0.84375, "grad_norm": 0.2654900550842285, "learning_rate": 7.022408581865382e-05, "loss": 0.007489924784749746, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.69, "memory/max_allocated (GiB)": 38.69, "ppl": 1.00752, "step": 216, "tokens/total": 7479984, "tokens/train_per_sec_per_gpu": 25.74, "tokens/trainable": 98201} +{"epoch": 0.84765625, "grad_norm": 0.34973204135894775, "learning_rate": 6.99505974422157e-05, "loss": 0.0016370509983971715, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.00164, "step": 217, "tokens/total": 7514144, "tokens/train_per_sec_per_gpu": 28.74, "tokens/trainable": 98674} +{"epoch": 0.8515625, "grad_norm": 0.19434854388237, "learning_rate": 6.967648691039213e-05, "loss": 0.0015196952736005187, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.73, "memory/max_allocated (GiB)": 36.73, "ppl": 1.00152, "step": 218, "tokens/total": 7546256, "tokens/train_per_sec_per_gpu": 28.28, "tokens/trainable": 99108} +{"epoch": 0.85546875, "grad_norm": 0.1491953283548355, "learning_rate": 6.940176563004123e-05, "loss": 0.003270490560680628, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00328, "step": 219, "tokens/total": 7580432, "tokens/train_per_sec_per_gpu": 26.27, "tokens/trainable": 99540} +{"epoch": 0.859375, "grad_norm": 0.2573917806148529, "learning_rate": 6.912644503343682e-05, "loss": 0.0042192586697638035, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.00423, "step": 220, "tokens/total": 7617904, "tokens/train_per_sec_per_gpu": 23.72, "tokens/trainable": 99978} +{"epoch": 0.86328125, "grad_norm": 0.7919195890426636, "learning_rate": 6.885053657779273e-05, "loss": 0.024344025179743767, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.02464, "step": 221, "tokens/total": 7649696, "tokens/train_per_sec_per_gpu": 28.77, "tokens/trainable": 100450} +{"epoch": 0.8671875, "grad_norm": 0.3402808904647827, "learning_rate": 6.857405174478604e-05, "loss": 0.001599662471562624, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.0016, "step": 222, "tokens/total": 7684464, "tokens/train_per_sec_per_gpu": 24.63, "tokens/trainable": 100903} +{"epoch": 0.87109375, "grad_norm": 0.6280295848846436, "learning_rate": 6.82970020400792e-05, "loss": 0.02508796751499176, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.02541, "step": 223, "tokens/total": 7721936, "tokens/train_per_sec_per_gpu": 22.58, "tokens/trainable": 101351} +{"epoch": 0.875, "grad_norm": 0.7719559669494629, "learning_rate": 6.801939899284132e-05, "loss": 0.00637428741902113, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.34, "memory/max_allocated (GiB)": 38.34, "ppl": 1.00639, "step": 224, "tokens/total": 7758160, "tokens/train_per_sec_per_gpu": 26.79, "tokens/trainable": 101857} +{"epoch": 0.87890625, "grad_norm": 0.09093967080116272, "learning_rate": 6.774125415526827e-05, "loss": 0.0017049422021955252, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00171, "step": 225, "tokens/total": 7791216, "tokens/train_per_sec_per_gpu": 28.08, "tokens/trainable": 102328} +{"epoch": 0.8828125, "grad_norm": 0.09253297746181488, "learning_rate": 6.746257910210214e-05, "loss": 0.0012693980243057013, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.12, "memory/max_allocated (GiB)": 37.12, "ppl": 1.00127, "step": 226, "tokens/total": 7824400, "tokens/train_per_sec_per_gpu": 28.47, "tokens/trainable": 102779} +{"epoch": 0.88671875, "grad_norm": 0.2756615877151489, "learning_rate": 6.718338543014937e-05, "loss": 0.0067627523094415665, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 35.75, "memory/max_allocated (GiB)": 35.75, "ppl": 1.00679, "step": 227, "tokens/total": 7854672, "tokens/train_per_sec_per_gpu": 31.19, "tokens/trainable": 103228} +{"epoch": 0.890625, "grad_norm": 0.13873185217380524, "learning_rate": 6.69036847577983e-05, "loss": 0.004185639321804047, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.00419, "step": 228, "tokens/total": 7888592, "tokens/train_per_sec_per_gpu": 27.04, "tokens/trainable": 103684} +{"epoch": 0.89453125, "grad_norm": 0.3422863781452179, "learning_rate": 6.662348872453553e-05, "loss": 0.00797618180513382, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.33, "memory/max_allocated (GiB)": 38.33, "ppl": 1.00801, "step": 229, "tokens/total": 7924848, "tokens/train_per_sec_per_gpu": 27.79, "tokens/trainable": 104145} +{"epoch": 0.8984375, "grad_norm": 0.2342989146709442, "learning_rate": 6.63428089904618e-05, "loss": 0.0053735896944999695, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.00539, "step": 230, "tokens/total": 7963856, "tokens/train_per_sec_per_gpu": 24.47, "tokens/trainable": 104598} +{"epoch": 0.90234375, "grad_norm": 0.32693836092948914, "learning_rate": 6.60616572358065e-05, "loss": 0.0018488293280825019, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.00185, "step": 231, "tokens/total": 7998320, "tokens/train_per_sec_per_gpu": 25.89, "tokens/trainable": 104998} +{"epoch": 0.90625, "grad_norm": 0.47570130228996277, "learning_rate": 6.578004516044172e-05, "loss": 0.0017636532429605722, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00177, "step": 232, "tokens/total": 8034096, "tokens/train_per_sec_per_gpu": 27.82, "tokens/trainable": 105465} +{"epoch": 0.91015625, "grad_norm": 0.3072345554828644, "learning_rate": 6.549798448339548e-05, "loss": 0.006361961364746094, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.59, "memory/max_allocated (GiB)": 38.59, "ppl": 1.00638, "step": 233, "tokens/total": 8071360, "tokens/train_per_sec_per_gpu": 25.29, "tokens/trainable": 105928} +{"epoch": 0.9140625, "grad_norm": 0.06340061128139496, "learning_rate": 6.521548694236384e-05, "loss": 0.001260714721865952, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00126, "step": 234, "tokens/total": 8108560, "tokens/train_per_sec_per_gpu": 23.56, "tokens/trainable": 106365} +{"epoch": 0.91796875, "grad_norm": 0.5358495712280273, "learning_rate": 6.493256429322259e-05, "loss": 0.014943249523639679, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.01506, "step": 235, "tokens/total": 8144816, "tokens/train_per_sec_per_gpu": 22.43, "tokens/trainable": 106796} +{"epoch": 0.921875, "grad_norm": 0.5554044246673584, "learning_rate": 6.464922830953799e-05, "loss": 0.012666123919188976, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.01275, "step": 236, "tokens/total": 8183680, "tokens/train_per_sec_per_gpu": 22.5, "tokens/trainable": 107227} +{"epoch": 0.92578125, "grad_norm": 0.651560366153717, "learning_rate": 6.436549078207688e-05, "loss": 0.033768314868211746, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.03434, "step": 237, "tokens/total": 8216688, "tokens/train_per_sec_per_gpu": 29.07, "tokens/trainable": 107687} +{"epoch": 0.9296875, "grad_norm": 1.0810532569885254, "learning_rate": 6.408136351831592e-05, "loss": 0.01596899703145027, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.0161, "step": 238, "tokens/total": 8249104, "tokens/train_per_sec_per_gpu": 24.38, "tokens/trainable": 108127} +{"epoch": 0.93359375, "grad_norm": 0.17704489827156067, "learning_rate": 6.379685834195036e-05, "loss": 0.003734311554580927, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00374, "step": 239, "tokens/total": 8288528, "tokens/train_per_sec_per_gpu": 24.75, "tokens/trainable": 108633} +{"epoch": 0.9375, "grad_norm": 0.18172192573547363, "learning_rate": 6.351198709240186e-05, "loss": 0.004080089740455151, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.00409, "step": 240, "tokens/total": 8328464, "tokens/train_per_sec_per_gpu": 21.22, "tokens/trainable": 109085} +{"epoch": 0.94140625, "grad_norm": 0.1984647810459137, "learning_rate": 6.32267616243259e-05, "loss": 0.006175945978611708, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.0062, "step": 241, "tokens/total": 8361264, "tokens/train_per_sec_per_gpu": 29.02, "tokens/trainable": 109548} +{"epoch": 0.9453125, "grad_norm": 0.2176934778690338, "learning_rate": 6.294119380711849e-05, "loss": 0.004494057968258858, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.39, "memory/max_allocated (GiB)": 38.39, "ppl": 1.0045, "step": 242, "tokens/total": 8395552, "tokens/train_per_sec_per_gpu": 24.03, "tokens/trainable": 109975} +{"epoch": 0.94921875, "grad_norm": 0.34179621934890747, "learning_rate": 6.265529552442209e-05, "loss": 0.01623598486185074, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.01637, "step": 243, "tokens/total": 8430656, "tokens/train_per_sec_per_gpu": 28.29, "tokens/trainable": 110436} +{"epoch": 0.953125, "grad_norm": 0.23974241316318512, "learning_rate": 6.236907867363127e-05, "loss": 0.0057562473230063915, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.17, "memory/max_allocated (GiB)": 38.17, "ppl": 1.00577, "step": 244, "tokens/total": 8465232, "tokens/train_per_sec_per_gpu": 26.87, "tokens/trainable": 110882} +{"epoch": 0.95703125, "grad_norm": 0.2370167374610901, "learning_rate": 6.208255516539749e-05, "loss": 0.0034966710954904556, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.0035, "step": 245, "tokens/total": 8497552, "tokens/train_per_sec_per_gpu": 26.83, "tokens/trainable": 111355} +{"epoch": 0.9609375, "grad_norm": 0.09171967953443527, "learning_rate": 6.179573692313344e-05, "loss": 0.002295929938554764, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.8, "memory/max_allocated (GiB)": 38.8, "ppl": 1.0023, "step": 246, "tokens/total": 8534336, "tokens/train_per_sec_per_gpu": 26.27, "tokens/trainable": 111803} +{"epoch": 0.96484375, "grad_norm": 0.06905300170183182, "learning_rate": 6.150863588251694e-05, "loss": 0.0020235227420926094, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.00203, "step": 247, "tokens/total": 8574224, "tokens/train_per_sec_per_gpu": 22.86, "tokens/trainable": 112270} +{"epoch": 0.96875, "grad_norm": 0.39570581912994385, "learning_rate": 6.122126399099419e-05, "loss": 0.008447139523923397, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.00848, "step": 248, "tokens/total": 8613616, "tokens/train_per_sec_per_gpu": 27.09, "tokens/trainable": 112778} +{"epoch": 0.97265625, "grad_norm": 2.2682437896728516, "learning_rate": 6.0933633207282615e-05, "loss": 0.028496574610471725, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.02891, "step": 249, "tokens/total": 8650048, "tokens/train_per_sec_per_gpu": 27.59, "tokens/trainable": 113284} +{"epoch": 0.9765625, "grad_norm": 0.3287281095981598, "learning_rate": 6.064575550087316e-05, "loss": 0.008818726986646652, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00886, "step": 250, "tokens/total": 8686688, "tokens/train_per_sec_per_gpu": 23.91, "tokens/trainable": 113722} +{"epoch": 0.98046875, "grad_norm": 0.26367267966270447, "learning_rate": 6.0357642851532245e-05, "loss": 0.009593947790563107, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00964, "step": 251, "tokens/total": 8723376, "tokens/train_per_sec_per_gpu": 25.99, "tokens/trainable": 114213} +{"epoch": 0.984375, "grad_norm": 0.8296051621437073, "learning_rate": 6.0069307248803294e-05, "loss": 0.013954417780041695, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.01405, "step": 252, "tokens/total": 8758736, "tokens/train_per_sec_per_gpu": 22.11, "tokens/trainable": 114634} +{"epoch": 0.98828125, "grad_norm": 0.23143406212329865, "learning_rate": 5.9780760691507635e-05, "loss": 0.0048714433796703815, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.33, "memory/max_allocated (GiB)": 36.33, "ppl": 1.00488, "step": 253, "tokens/total": 8790480, "tokens/train_per_sec_per_gpu": 26.96, "tokens/trainable": 115066} +{"epoch": 0.9921875, "grad_norm": 0.17130132019519806, "learning_rate": 5.9492015187245334e-05, "loss": 0.0037607471458613873, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00377, "step": 254, "tokens/total": 8828224, "tokens/train_per_sec_per_gpu": 23.46, "tokens/trainable": 115520} +{"epoch": 0.99609375, "grad_norm": 0.12393870204687119, "learning_rate": 5.920308275189541e-05, "loss": 0.0015232632867991924, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.00152, "step": 255, "tokens/total": 8860400, "tokens/train_per_sec_per_gpu": 25.75, "tokens/trainable": 115921} +{"epoch": 1.0, "grad_norm": 0.3871704936027527, "learning_rate": 5.8913975409115874e-05, "loss": 0.016899531707167625, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.01704, "step": 256, "tokens/total": 8893760, "tokens/train_per_sec_per_gpu": 25.6, "tokens/trainable": 116384} +{"epoch": 1.00390625, "grad_norm": 0.21337933838367462, "learning_rate": 5.8624705189843395e-05, "loss": 0.0031738716643303633, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 1.00318, "step": 257, "tokens/total": 8930976, "tokens/train_per_sec_per_gpu": 24.84, "tokens/trainable": 116871} +{"epoch": 1.0078125, "grad_norm": 0.030067715793848038, "learning_rate": 5.833528413179249e-05, "loss": 0.0005626874626614153, "memory/device_reserved (GiB)": 42.25, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00056, "step": 258, "tokens/total": 8967472, "tokens/train_per_sec_per_gpu": 27.98, "tokens/trainable": 117369} +{"epoch": 1.01171875, "grad_norm": 0.13527356088161469, "learning_rate": 5.80457242789548e-05, "loss": 0.0027104439213871956, "memory/device_reserved (GiB)": 42.31, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.00271, "step": 259, "tokens/total": 9007456, "tokens/train_per_sec_per_gpu": 24.03, "tokens/trainable": 117866} +{"epoch": 1.015625, "grad_norm": 0.21802861988544464, "learning_rate": 5.77560376810977e-05, "loss": 0.0038060247898101807, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 1.00381, "step": 260, "tokens/total": 9041104, "tokens/train_per_sec_per_gpu": 29.43, "tokens/trainable": 118330} +{"epoch": 1.01953125, "grad_norm": 0.6383243799209595, "learning_rate": 5.7466236393263005e-05, "loss": 0.011425388045608997, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.01149, "step": 261, "tokens/total": 9076176, "tokens/train_per_sec_per_gpu": 25.7, "tokens/trainable": 118776} +{"epoch": 1.0234375, "grad_norm": 0.1816917210817337, "learning_rate": 5.717633247526522e-05, "loss": 0.0031942399218678474, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.0032, "step": 262, "tokens/total": 9107472, "tokens/train_per_sec_per_gpu": 27.91, "tokens/trainable": 119211} +{"epoch": 1.02734375, "grad_norm": 0.46807661652565, "learning_rate": 5.688633799118971e-05, "loss": 0.006969357840716839, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.00699, "step": 263, "tokens/total": 9139552, "tokens/train_per_sec_per_gpu": 23.58, "tokens/trainable": 119616} +{"epoch": 1.03125, "grad_norm": 0.29819759726524353, "learning_rate": 5.659626500889066e-05, "loss": 0.00232822191901505, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00233, "step": 264, "tokens/total": 9173424, "tokens/train_per_sec_per_gpu": 26.02, "tokens/trainable": 120086} +{"epoch": 1.03515625, "grad_norm": 0.33994442224502563, "learning_rate": 5.6306125599488905e-05, "loss": 0.0029731523245573044, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.00298, "step": 265, "tokens/total": 9203984, "tokens/train_per_sec_per_gpu": 32.39, "tokens/trainable": 120525} +{"epoch": 1.0390625, "grad_norm": 0.40866518020629883, "learning_rate": 5.601593183686955e-05, "loss": 0.005714380647987127, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.69, "memory/max_allocated (GiB)": 38.69, "ppl": 1.00573, "step": 266, "tokens/total": 9241248, "tokens/train_per_sec_per_gpu": 26.16, "tokens/trainable": 121001} +{"epoch": 1.04296875, "grad_norm": 0.2188020646572113, "learning_rate": 5.572569579717961e-05, "loss": 0.0069023119285702705, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.18, "memory/max_allocated (GiB)": 39.18, "ppl": 1.00693, "step": 267, "tokens/total": 9276768, "tokens/train_per_sec_per_gpu": 24.66, "tokens/trainable": 121465} +{"epoch": 1.046875, "grad_norm": 0.060757189989089966, "learning_rate": 5.543542955832538e-05, "loss": 0.0005572427762672305, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00056, "step": 268, "tokens/total": 9316528, "tokens/train_per_sec_per_gpu": 21.84, "tokens/trainable": 121908} +{"epoch": 1.05078125, "grad_norm": 0.4374277889728546, "learning_rate": 5.514514519946986e-05, "loss": 0.007681994698941708, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.00771, "step": 269, "tokens/total": 9356352, "tokens/train_per_sec_per_gpu": 25.88, "tokens/trainable": 122406} +{"epoch": 1.0546875, "grad_norm": 0.06845907866954803, "learning_rate": 5.485485480053015e-05, "loss": 0.0007930619176477194, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.00079, "step": 270, "tokens/total": 9394576, "tokens/train_per_sec_per_gpu": 22.05, "tokens/trainable": 122812} +{"epoch": 1.05859375, "grad_norm": 0.06553243845701218, "learning_rate": 5.4564570441674645e-05, "loss": 0.0008071116171777248, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.92, "memory/max_allocated (GiB)": 36.92, "ppl": 1.00081, "step": 271, "tokens/total": 9427840, "tokens/train_per_sec_per_gpu": 25.71, "tokens/trainable": 123252} +{"epoch": 1.0625, "grad_norm": 0.03225182369351387, "learning_rate": 5.42743042028204e-05, "loss": 0.00041551212780177593, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.00042, "step": 272, "tokens/total": 9463536, "tokens/train_per_sec_per_gpu": 20.78, "tokens/trainable": 123692} +{"epoch": 1.06640625, "grad_norm": 0.3487846553325653, "learning_rate": 5.3984068163130464e-05, "loss": 0.004344887100160122, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.09, "memory/max_allocated (GiB)": 37.09, "ppl": 1.00435, "step": 273, "tokens/total": 9497248, "tokens/train_per_sec_per_gpu": 28.39, "tokens/trainable": 124162} +{"epoch": 1.0703125, "grad_norm": 0.42149457335472107, "learning_rate": 5.369387440051111e-05, "loss": 0.005976000800728798, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.00599, "step": 274, "tokens/total": 9530784, "tokens/train_per_sec_per_gpu": 25.63, "tokens/trainable": 124625} +{"epoch": 1.07421875, "grad_norm": 0.0770048126578331, "learning_rate": 5.340373499110935e-05, "loss": 0.0003207189729437232, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00032, "step": 275, "tokens/total": 9566656, "tokens/train_per_sec_per_gpu": 26.75, "tokens/trainable": 125119} +{"epoch": 1.078125, "grad_norm": 0.013115803711116314, "learning_rate": 5.3113662008810304e-05, "loss": 0.00022846669889986515, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00023, "step": 276, "tokens/total": 9606672, "tokens/train_per_sec_per_gpu": 25.18, "tokens/trainable": 125601} +{"epoch": 1.08203125, "grad_norm": 0.053274888545274734, "learning_rate": 5.282366752473479e-05, "loss": 0.0007997624343261123, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 1.0008, "step": 277, "tokens/total": 9643680, "tokens/train_per_sec_per_gpu": 24.69, "tokens/trainable": 126075} +{"epoch": 1.0859375, "grad_norm": 0.8510315418243408, "learning_rate": 5.2533763606737005e-05, "loss": 0.004499177914112806, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00451, "step": 278, "tokens/total": 9683200, "tokens/train_per_sec_per_gpu": 19.53, "tokens/trainable": 126491} +{"epoch": 1.08984375, "grad_norm": 0.09681686013936996, "learning_rate": 5.224396231890232e-05, "loss": 0.0008465705905109644, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.00085, "step": 279, "tokens/total": 9714672, "tokens/train_per_sec_per_gpu": 30.36, "tokens/trainable": 126940} +{"epoch": 1.09375, "grad_norm": 0.04620688408613205, "learning_rate": 5.195427572104522e-05, "loss": 0.0006935237906873226, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00069, "step": 280, "tokens/total": 9749104, "tokens/train_per_sec_per_gpu": 23.97, "tokens/trainable": 127376} +{"epoch": 1.09765625, "grad_norm": 0.3251117467880249, "learning_rate": 5.166471586820751e-05, "loss": 0.004383953288197517, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.35, "memory/max_allocated (GiB)": 39.35, "ppl": 1.00439, "step": 281, "tokens/total": 9785712, "tokens/train_per_sec_per_gpu": 24.01, "tokens/trainable": 127841} +{"epoch": 1.1015625, "grad_norm": 0.31981968879699707, "learning_rate": 5.1375294810156615e-05, "loss": 0.01756858266890049, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.01772, "step": 282, "tokens/total": 9820544, "tokens/train_per_sec_per_gpu": 26.11, "tokens/trainable": 128288} +{"epoch": 1.10546875, "grad_norm": 0.4408259987831116, "learning_rate": 5.1086024590884144e-05, "loss": 0.004850848577916622, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00486, "step": 283, "tokens/total": 9857952, "tokens/train_per_sec_per_gpu": 25.25, "tokens/trainable": 128737} +{"epoch": 1.109375, "grad_norm": 0.04770122095942497, "learning_rate": 5.079691724810461e-05, "loss": 0.00015429037739522755, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.33, "memory/max_allocated (GiB)": 38.33, "ppl": 1.00015, "step": 284, "tokens/total": 9894128, "tokens/train_per_sec_per_gpu": 22.0, "tokens/trainable": 129172} +{"epoch": 1.11328125, "grad_norm": 0.0418754443526268, "learning_rate": 5.0507984812754684e-05, "loss": 0.0003317098307888955, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.17, "memory/max_allocated (GiB)": 37.17, "ppl": 1.00033, "step": 285, "tokens/total": 9927568, "tokens/train_per_sec_per_gpu": 29.07, "tokens/trainable": 129669} +{"epoch": 1.1171875, "grad_norm": 0.14583905041217804, "learning_rate": 5.021923930849237e-05, "loss": 0.0009398568072356284, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.00094, "step": 286, "tokens/total": 9961888, "tokens/train_per_sec_per_gpu": 28.25, "tokens/trainable": 130155} +{"epoch": 1.12109375, "grad_norm": 0.08999878168106079, "learning_rate": 4.99306927511967e-05, "loss": 0.0006144917570054531, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.31, "memory/max_allocated (GiB)": 37.31, "ppl": 1.00061, "step": 287, "tokens/total": 9995696, "tokens/train_per_sec_per_gpu": 28.19, "tokens/trainable": 130595} +{"epoch": 1.125, "grad_norm": 0.34335023164749146, "learning_rate": 4.964235714846775e-05, "loss": 0.0032911074813455343, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.72, "memory/max_allocated (GiB)": 36.72, "ppl": 1.0033, "step": 288, "tokens/total": 10028000, "tokens/train_per_sec_per_gpu": 27.5, "tokens/trainable": 131015} +{"epoch": 1.12890625, "grad_norm": 0.005473357625305653, "learning_rate": 4.9354244499126866e-05, "loss": 6.613755249418318e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.16, "memory/max_allocated (GiB)": 36.16, "ppl": 1.00007, "step": 289, "tokens/total": 10059728, "tokens/train_per_sec_per_gpu": 29.74, "tokens/trainable": 131481} +{"epoch": 1.1328125, "grad_norm": 0.007237070240080357, "learning_rate": 4.90663667927174e-05, "loss": 8.379382779821754e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.00008, "step": 290, "tokens/total": 10092576, "tokens/train_per_sec_per_gpu": 29.26, "tokens/trainable": 131930} +{"epoch": 1.13671875, "grad_norm": 0.12123443186283112, "learning_rate": 4.877873600900581e-05, "loss": 0.0005065160803496838, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.00051, "step": 291, "tokens/total": 10128608, "tokens/train_per_sec_per_gpu": 21.94, "tokens/trainable": 132388} +{"epoch": 1.140625, "grad_norm": 0.1421699821949005, "learning_rate": 4.849136411748306e-05, "loss": 0.0008942326530814171, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.00089, "step": 292, "tokens/total": 10166192, "tokens/train_per_sec_per_gpu": 25.19, "tokens/trainable": 132860} +{"epoch": 1.14453125, "grad_norm": 0.19872872531414032, "learning_rate": 4.8204263076866574e-05, "loss": 0.005885784514248371, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.0059, "step": 293, "tokens/total": 10201840, "tokens/train_per_sec_per_gpu": 30.0, "tokens/trainable": 133360} +{"epoch": 1.1484375, "grad_norm": 0.14562183618545532, "learning_rate": 4.791744483460251e-05, "loss": 0.001339221140369773, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00134, "step": 294, "tokens/total": 10236048, "tokens/train_per_sec_per_gpu": 24.53, "tokens/trainable": 133793} +{"epoch": 1.15234375, "grad_norm": 0.29208675026893616, "learning_rate": 4.7630921326368736e-05, "loss": 0.0022087895777076483, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.00221, "step": 295, "tokens/total": 10274688, "tokens/train_per_sec_per_gpu": 20.26, "tokens/trainable": 134218} +{"epoch": 1.15625, "grad_norm": 0.8676779270172119, "learning_rate": 4.7344704475577916e-05, "loss": 0.011871399357914925, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 35.66, "memory/max_allocated (GiB)": 35.66, "ppl": 1.01194, "step": 296, "tokens/total": 10304576, "tokens/train_per_sec_per_gpu": 30.68, "tokens/trainable": 134654} +{"epoch": 1.16015625, "grad_norm": 1.1449097394943237, "learning_rate": 4.705880619288153e-05, "loss": 0.029924865812063217, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.21, "memory/max_allocated (GiB)": 38.21, "ppl": 1.03038, "step": 297, "tokens/total": 10340720, "tokens/train_per_sec_per_gpu": 22.28, "tokens/trainable": 135056} +{"epoch": 1.1640625, "grad_norm": 0.517724335193634, "learning_rate": 4.677323837567412e-05, "loss": 0.015201062895357609, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.01532, "step": 298, "tokens/total": 10373376, "tokens/train_per_sec_per_gpu": 25.69, "tokens/trainable": 135483} +{"epoch": 1.16796875, "grad_norm": 0.08820949494838715, "learning_rate": 4.6488012907598146e-05, "loss": 0.0005529679474420846, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00055, "step": 299, "tokens/total": 10406128, "tokens/train_per_sec_per_gpu": 28.33, "tokens/trainable": 135957} +{"epoch": 1.171875, "grad_norm": 0.3431568741798401, "learning_rate": 4.620314165804964e-05, "loss": 0.004393803887069225, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.3, "memory/max_allocated (GiB)": 38.3, "ppl": 1.0044, "step": 300, "tokens/total": 10441792, "tokens/train_per_sec_per_gpu": 25.99, "tokens/trainable": 136430} +{"epoch": 1.17578125, "grad_norm": 0.4214995503425598, "learning_rate": 4.591863648168407e-05, "loss": 0.0038493748288601637, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00386, "step": 301, "tokens/total": 10476128, "tokens/train_per_sec_per_gpu": 26.67, "tokens/trainable": 136896} +{"epoch": 1.1796875, "grad_norm": 0.4613259434700012, "learning_rate": 4.5634509217923135e-05, "loss": 0.008814917877316475, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00885, "step": 302, "tokens/total": 10509472, "tokens/train_per_sec_per_gpu": 26.1, "tokens/trainable": 137331} +{"epoch": 1.18359375, "grad_norm": 0.7130829691886902, "learning_rate": 4.535077169046201e-05, "loss": 0.012559506110846996, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.01264, "step": 303, "tokens/total": 10546032, "tokens/train_per_sec_per_gpu": 25.45, "tokens/trainable": 137768} +{"epoch": 1.1875, "grad_norm": 0.7739271521568298, "learning_rate": 4.506743570677743e-05, "loss": 0.01836257241666317, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.01853, "step": 304, "tokens/total": 10576624, "tokens/train_per_sec_per_gpu": 31.69, "tokens/trainable": 138200} +{"epoch": 1.19140625, "grad_norm": 0.4528144598007202, "learning_rate": 4.478451305763618e-05, "loss": 0.006584860384464264, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.95, "memory/max_allocated (GiB)": 36.95, "ppl": 1.00661, "step": 305, "tokens/total": 10609760, "tokens/train_per_sec_per_gpu": 30.95, "tokens/trainable": 138660} +{"epoch": 1.1953125, "grad_norm": 0.15505234897136688, "learning_rate": 4.450201551660454e-05, "loss": 0.0025798522401601076, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.51, "memory/max_allocated (GiB)": 36.51, "ppl": 1.00258, "step": 306, "tokens/total": 10641472, "tokens/train_per_sec_per_gpu": 26.58, "tokens/trainable": 139092} +{"epoch": 1.19921875, "grad_norm": 1.7176737785339355, "learning_rate": 4.4219954839558276e-05, "loss": 0.011074929498136044, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.01114, "step": 307, "tokens/total": 10676160, "tokens/train_per_sec_per_gpu": 25.67, "tokens/trainable": 139533} +{"epoch": 1.203125, "grad_norm": 0.42521899938583374, "learning_rate": 4.393834276419352e-05, "loss": 0.006581522990018129, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.83, "memory/max_allocated (GiB)": 35.83, "ppl": 1.0066, "step": 308, "tokens/total": 10706384, "tokens/train_per_sec_per_gpu": 32.76, "tokens/trainable": 140020} +{"epoch": 1.20703125, "grad_norm": 0.39721807837486267, "learning_rate": 4.36571910095382e-05, "loss": 0.011665113270282745, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.01173, "step": 309, "tokens/total": 10744496, "tokens/train_per_sec_per_gpu": 25.76, "tokens/trainable": 140465} +{"epoch": 1.2109375, "grad_norm": 0.07853015512228012, "learning_rate": 4.337651127546448e-05, "loss": 0.001645843032747507, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00165, "step": 310, "tokens/total": 10780176, "tokens/train_per_sec_per_gpu": 27.87, "tokens/trainable": 140915} +{"epoch": 1.21484375, "grad_norm": 0.581538200378418, "learning_rate": 4.3096315242201736e-05, "loss": 0.0033796869684010744, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.61, "memory/max_allocated (GiB)": 36.61, "ppl": 1.00339, "step": 311, "tokens/total": 10813136, "tokens/train_per_sec_per_gpu": 26.58, "tokens/trainable": 141350} +{"epoch": 1.21875, "grad_norm": 0.8375452160835266, "learning_rate": 4.2816614569850635e-05, "loss": 0.012758172117173672, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.01284, "step": 312, "tokens/total": 10848912, "tokens/train_per_sec_per_gpu": 24.59, "tokens/trainable": 141789} +{"epoch": 1.22265625, "grad_norm": 0.27737534046173096, "learning_rate": 4.2537420897897864e-05, "loss": 0.011586866341531277, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.01165, "step": 313, "tokens/total": 10884320, "tokens/train_per_sec_per_gpu": 27.35, "tokens/trainable": 142266} +{"epoch": 1.2265625, "grad_norm": 0.262711763381958, "learning_rate": 4.225874584473174e-05, "loss": 0.003749545430764556, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.00376, "step": 314, "tokens/total": 10917152, "tokens/train_per_sec_per_gpu": 26.32, "tokens/trainable": 142704} +{"epoch": 1.23046875, "grad_norm": 0.13031643629074097, "learning_rate": 4.19806010071587e-05, "loss": 0.0033394829370081425, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.00335, "step": 315, "tokens/total": 10955024, "tokens/train_per_sec_per_gpu": 24.71, "tokens/trainable": 143168} +{"epoch": 1.234375, "grad_norm": 0.21361055970191956, "learning_rate": 4.170299795992081e-05, "loss": 0.005853879265487194, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.00587, "step": 316, "tokens/total": 10988416, "tokens/train_per_sec_per_gpu": 27.35, "tokens/trainable": 143615} +{"epoch": 1.23828125, "grad_norm": 0.3604290187358856, "learning_rate": 4.142594825521398e-05, "loss": 0.004282059147953987, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.00429, "step": 317, "tokens/total": 11027712, "tokens/train_per_sec_per_gpu": 26.25, "tokens/trainable": 144096} +{"epoch": 1.2421875, "grad_norm": 0.12371297925710678, "learning_rate": 4.114946342220728e-05, "loss": 0.0025110780261456966, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.00251, "step": 318, "tokens/total": 11063920, "tokens/train_per_sec_per_gpu": 24.2, "tokens/trainable": 144543} +{"epoch": 1.24609375, "grad_norm": 0.05824637785553932, "learning_rate": 4.087355496656321e-05, "loss": 0.0014032479375600815, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.0014, "step": 319, "tokens/total": 11097952, "tokens/train_per_sec_per_gpu": 28.82, "tokens/trainable": 145016} +{"epoch": 1.25, "grad_norm": 0.17104420065879822, "learning_rate": 4.05982343699588e-05, "loss": 0.004387992899864912, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.0044, "step": 320, "tokens/total": 11137168, "tokens/train_per_sec_per_gpu": 25.49, "tokens/trainable": 145511} +{"epoch": 1.25390625, "grad_norm": 0.12102850526571274, "learning_rate": 4.0323513089607876e-05, "loss": 0.002549383556470275, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.00255, "step": 321, "tokens/total": 11172224, "tokens/train_per_sec_per_gpu": 24.56, "tokens/trainable": 145978} +{"epoch": 1.2578125, "grad_norm": 0.4355841875076294, "learning_rate": 4.004940255778431e-05, "loss": 0.011749599128961563, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.01182, "step": 322, "tokens/total": 11210832, "tokens/train_per_sec_per_gpu": 24.19, "tokens/trainable": 146422} +{"epoch": 1.26171875, "grad_norm": 0.06703804433345795, "learning_rate": 3.977591418134619e-05, "loss": 0.0008260516333393753, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.00083, "step": 323, "tokens/total": 11245984, "tokens/train_per_sec_per_gpu": 26.24, "tokens/trainable": 146863} +{"epoch": 1.265625, "grad_norm": 0.12249531596899033, "learning_rate": 3.95030593412612e-05, "loss": 0.003442340297624469, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.00345, "step": 324, "tokens/total": 11281360, "tokens/train_per_sec_per_gpu": 24.86, "tokens/trainable": 147286} +{"epoch": 1.26953125, "grad_norm": 0.15014415979385376, "learning_rate": 3.923084939213296e-05, "loss": 0.0023715458810329437, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.88, "memory/max_allocated (GiB)": 35.88, "ppl": 1.00237, "step": 325, "tokens/total": 11312176, "tokens/train_per_sec_per_gpu": 28.03, "tokens/trainable": 147708} +{"epoch": 1.2734375, "grad_norm": 0.18511292338371277, "learning_rate": 3.895929566172861e-05, "loss": 0.001391732133924961, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.00139, "step": 326, "tokens/total": 11348192, "tokens/train_per_sec_per_gpu": 25.5, "tokens/trainable": 148172} +{"epoch": 1.27734375, "grad_norm": 0.5608503818511963, "learning_rate": 3.868840945050728e-05, "loss": 0.008017277345061302, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.05, "memory/max_allocated (GiB)": 39.05, "ppl": 1.00805, "step": 327, "tokens/total": 11386704, "tokens/train_per_sec_per_gpu": 21.75, "tokens/trainable": 148604} +{"epoch": 1.28125, "grad_norm": 0.10197944194078445, "learning_rate": 3.841820203114995e-05, "loss": 0.0021567668300122023, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.00216, "step": 328, "tokens/total": 11418448, "tokens/train_per_sec_per_gpu": 29.41, "tokens/trainable": 149051} +{"epoch": 1.28515625, "grad_norm": 0.22025305032730103, "learning_rate": 3.814868464809027e-05, "loss": 0.003249730682000518, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.00326, "step": 329, "tokens/total": 11450304, "tokens/train_per_sec_per_gpu": 26.15, "tokens/trainable": 149453} +{"epoch": 1.2890625, "grad_norm": 0.439487099647522, "learning_rate": 3.787986851704667e-05, "loss": 0.0034580209758132696, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.00346, "step": 330, "tokens/total": 11484048, "tokens/train_per_sec_per_gpu": 26.01, "tokens/trainable": 149883} +{"epoch": 1.29296875, "grad_norm": 0.06834159046411514, "learning_rate": 3.7611764824555654e-05, "loss": 0.0008064756984822452, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00081, "step": 331, "tokens/total": 11516032, "tokens/train_per_sec_per_gpu": 31.33, "tokens/trainable": 150357} +{"epoch": 1.296875, "grad_norm": 12.648085594177246, "learning_rate": 3.734438472750619e-05, "loss": 0.016407305374741554, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.01654, "step": 332, "tokens/total": 11555968, "tokens/train_per_sec_per_gpu": 23.42, "tokens/trainable": 150828} +{"epoch": 1.30078125, "grad_norm": 0.14716793596744537, "learning_rate": 3.707773935267552e-05, "loss": 0.002464166609570384, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.46, "memory/max_allocated (GiB)": 38.46, "ppl": 1.00247, "step": 333, "tokens/total": 11592880, "tokens/train_per_sec_per_gpu": 27.24, "tokens/trainable": 151314} +{"epoch": 1.3046875, "grad_norm": 0.006625053007155657, "learning_rate": 3.68118397962661e-05, "loss": 9.107122605200857e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.00009, "step": 334, "tokens/total": 11627792, "tokens/train_per_sec_per_gpu": 23.56, "tokens/trainable": 151722} +{"epoch": 1.30859375, "grad_norm": 0.12452514469623566, "learning_rate": 3.654669712344384e-05, "loss": 0.000866019050590694, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 1.00087, "step": 335, "tokens/total": 11658832, "tokens/train_per_sec_per_gpu": 32.09, "tokens/trainable": 152178} +{"epoch": 1.3125, "grad_norm": 0.09603685885667801, "learning_rate": 3.628232236787763e-05, "loss": 0.0010944448877125978, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.96, "memory/max_allocated (GiB)": 37.96, "ppl": 1.0011, "step": 336, "tokens/total": 11694464, "tokens/train_per_sec_per_gpu": 22.89, "tokens/trainable": 152617} +{"epoch": 1.31640625, "grad_norm": 0.06926809251308441, "learning_rate": 3.6018726531280144e-05, "loss": 0.0009258457575924695, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00093, "step": 337, "tokens/total": 11732176, "tokens/train_per_sec_per_gpu": 28.53, "tokens/trainable": 153108} +{"epoch": 1.3203125, "grad_norm": 0.048827607184648514, "learning_rate": 3.575592058295017e-05, "loss": 0.0003974108549300581, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.0004, "step": 338, "tokens/total": 11766032, "tokens/train_per_sec_per_gpu": 28.84, "tokens/trainable": 153589} +{"epoch": 1.32421875, "grad_norm": 0.03205815702676773, "learning_rate": 3.549391545931585e-05, "loss": 0.0001052739389706403, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00011, "step": 339, "tokens/total": 11800640, "tokens/train_per_sec_per_gpu": 28.05, "tokens/trainable": 154049} +{"epoch": 1.328125, "grad_norm": 0.03241156041622162, "learning_rate": 3.5232722063479914e-05, "loss": 0.00019104511011391878, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.00019, "step": 340, "tokens/total": 11834352, "tokens/train_per_sec_per_gpu": 24.46, "tokens/trainable": 154498} +{"epoch": 1.33203125, "grad_norm": 0.16080571711063385, "learning_rate": 3.49723512647657e-05, "loss": 0.0007506751571781933, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.00075, "step": 341, "tokens/total": 11873056, "tokens/train_per_sec_per_gpu": 25.09, "tokens/trainable": 154998} +{"epoch": 1.3359375, "grad_norm": 0.14164628088474274, "learning_rate": 3.471281389826491e-05, "loss": 0.000909759197384119, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.00091, "step": 342, "tokens/total": 11904192, "tokens/train_per_sec_per_gpu": 26.35, "tokens/trainable": 155403} +{"epoch": 1.33984375, "grad_norm": 0.02424437738955021, "learning_rate": 3.4454120764386764e-05, "loss": 0.0001329242659267038, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.66, "memory/max_allocated (GiB)": 38.66, "ppl": 1.00013, "step": 343, "tokens/total": 11942384, "tokens/train_per_sec_per_gpu": 24.63, "tokens/trainable": 155858} +{"epoch": 1.34375, "grad_norm": 0.4293067157268524, "learning_rate": 3.4196282628408526e-05, "loss": 0.0027869162149727345, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.00279, "step": 344, "tokens/total": 11976624, "tokens/train_per_sec_per_gpu": 25.67, "tokens/trainable": 156303} +{"epoch": 1.34765625, "grad_norm": 0.019056035205721855, "learning_rate": 3.3939310220027456e-05, "loss": 0.00014703150372952223, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00015, "step": 345, "tokens/total": 12010768, "tokens/train_per_sec_per_gpu": 29.23, "tokens/trainable": 156798} +{"epoch": 1.3515625, "grad_norm": 0.04420555382966995, "learning_rate": 3.3683214232914404e-05, "loss": 0.000302150467177853, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.0003, "step": 346, "tokens/total": 12046224, "tokens/train_per_sec_per_gpu": 26.51, "tokens/trainable": 157271} +{"epoch": 1.35546875, "grad_norm": 0.0032860320061445236, "learning_rate": 3.342800532426873e-05, "loss": 3.377482789801434e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.00003, "step": 347, "tokens/total": 12082160, "tokens/train_per_sec_per_gpu": 25.77, "tokens/trainable": 157737} +{"epoch": 1.359375, "grad_norm": 0.057195305824279785, "learning_rate": 3.317369411437484e-05, "loss": 0.00012138768215663731, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.00012, "step": 348, "tokens/total": 12118128, "tokens/train_per_sec_per_gpu": 26.5, "tokens/trainable": 158205} +{"epoch": 1.36328125, "grad_norm": 0.13915084302425385, "learning_rate": 3.292029118616024e-05, "loss": 0.0009023013990372419, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.0009, "step": 349, "tokens/total": 12151264, "tokens/train_per_sec_per_gpu": 25.93, "tokens/trainable": 158628} +{"epoch": 1.3671875, "grad_norm": 0.6095989346504211, "learning_rate": 3.266780708475511e-05, "loss": 0.004715130664408207, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00473, "step": 350, "tokens/total": 12186800, "tokens/train_per_sec_per_gpu": 25.38, "tokens/trainable": 159106} +{"epoch": 1.37109375, "grad_norm": 0.0028017975855618715, "learning_rate": 3.241625231705354e-05, "loss": 2.284203037561383e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.00002, "step": 351, "tokens/total": 12222944, "tokens/train_per_sec_per_gpu": 25.35, "tokens/trainable": 159565} +{"epoch": 1.375, "grad_norm": 0.008545318618416786, "learning_rate": 3.216563735127618e-05, "loss": 9.553608833812177e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.15, "memory/max_allocated (GiB)": 39.15, "ppl": 1.0001, "step": 352, "tokens/total": 12258208, "tokens/train_per_sec_per_gpu": 25.89, "tokens/trainable": 160042} +{"epoch": 1.37890625, "grad_norm": 0.5536828637123108, "learning_rate": 3.191597261653475e-05, "loss": 0.00838759820908308, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00842, "step": 353, "tokens/total": 12295024, "tokens/train_per_sec_per_gpu": 24.02, "tokens/trainable": 160502} +{"epoch": 1.3828125, "grad_norm": 0.01410906482487917, "learning_rate": 3.166726850239794e-05, "loss": 0.00013964965182822198, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00014, "step": 354, "tokens/total": 12328064, "tokens/train_per_sec_per_gpu": 27.82, "tokens/trainable": 160950} +{"epoch": 1.38671875, "grad_norm": 0.0017935559153556824, "learning_rate": 3.141953535845912e-05, "loss": 2.0781200873898342e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.34, "memory/max_allocated (GiB)": 36.34, "ppl": 1.00002, "step": 355, "tokens/total": 12359504, "tokens/train_per_sec_per_gpu": 30.85, "tokens/trainable": 161400} +{"epoch": 1.390625, "grad_norm": 0.029682135209441185, "learning_rate": 3.11727834939056e-05, "loss": 0.00016376349958591163, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00016, "step": 356, "tokens/total": 12391824, "tokens/train_per_sec_per_gpu": 25.69, "tokens/trainable": 161863} +{"epoch": 1.39453125, "grad_norm": 1.0797418355941772, "learning_rate": 3.092702317708967e-05, "loss": 0.009510857984423637, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.00956, "step": 357, "tokens/total": 12425408, "tokens/train_per_sec_per_gpu": 29.28, "tokens/trainable": 162319} +{"epoch": 1.3984375, "grad_norm": 0.011141438037157059, "learning_rate": 3.0682264635101276e-05, "loss": 0.00011142136645503342, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.00011, "step": 358, "tokens/total": 12458160, "tokens/train_per_sec_per_gpu": 28.63, "tokens/trainable": 162771} +{"epoch": 1.40234375, "grad_norm": 0.11718206107616425, "learning_rate": 3.0438518053342407e-05, "loss": 0.0008597379201091826, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00086, "step": 359, "tokens/total": 12497104, "tokens/train_per_sec_per_gpu": 22.54, "tokens/trainable": 163243} +{"epoch": 1.40625, "grad_norm": 0.8478764295578003, "learning_rate": 3.0195793575103266e-05, "loss": 0.01271786168217659, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.0128, "step": 360, "tokens/total": 12536064, "tokens/train_per_sec_per_gpu": 21.54, "tokens/trainable": 163707} +{"epoch": 1.41015625, "grad_norm": 0.3489041328430176, "learning_rate": 2.9954101301140146e-05, "loss": 0.009897944517433643, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.57, "memory/max_allocated (GiB)": 37.57, "ppl": 1.00995, "step": 361, "tokens/total": 12571488, "tokens/train_per_sec_per_gpu": 24.91, "tokens/trainable": 164157} +{"epoch": 1.4140625, "grad_norm": 0.03396495059132576, "learning_rate": 2.9713451289255123e-05, "loss": 0.00029720630845986307, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.63, "memory/max_allocated (GiB)": 36.63, "ppl": 1.0003, "step": 362, "tokens/total": 12600864, "tokens/train_per_sec_per_gpu": 25.87, "tokens/trainable": 164598} +{"epoch": 1.41796875, "grad_norm": 0.006765373516827822, "learning_rate": 2.9473853553877484e-05, "loss": 7.421724876621738e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.00007, "step": 363, "tokens/total": 12635024, "tokens/train_per_sec_per_gpu": 27.63, "tokens/trainable": 165072} +{"epoch": 1.421875, "grad_norm": 0.2378113716840744, "learning_rate": 2.9235318065647e-05, "loss": 0.002183137461543083, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.00219, "step": 364, "tokens/total": 12674736, "tokens/train_per_sec_per_gpu": 25.26, "tokens/trainable": 165533} +{"epoch": 1.42578125, "grad_norm": 0.019772682338953018, "learning_rate": 2.8997854750998964e-05, "loss": 0.000165311896125786, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.46, "memory/max_allocated (GiB)": 39.46, "ppl": 1.00017, "step": 365, "tokens/total": 12714256, "tokens/train_per_sec_per_gpu": 23.45, "tokens/trainable": 165998} +{"epoch": 1.4296875, "grad_norm": 0.02722232975065708, "learning_rate": 2.8761473491751258e-05, "loss": 0.000235706043895334, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.00024, "step": 366, "tokens/total": 12747504, "tokens/train_per_sec_per_gpu": 27.3, "tokens/trainable": 166433} +{"epoch": 1.43359375, "grad_norm": 0.6133012771606445, "learning_rate": 2.8526184124692883e-05, "loss": 0.008180505596101284, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00821, "step": 367, "tokens/total": 12783872, "tokens/train_per_sec_per_gpu": 20.67, "tokens/trainable": 166871} +{"epoch": 1.4375, "grad_norm": 0.2590644061565399, "learning_rate": 2.829199644117484e-05, "loss": 0.0027691274881362915, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.00277, "step": 368, "tokens/total": 12817856, "tokens/train_per_sec_per_gpu": 24.65, "tokens/trainable": 167281} +{"epoch": 1.44140625, "grad_norm": 0.07019785046577454, "learning_rate": 2.8058920186702553e-05, "loss": 0.0008152966038323939, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00082, "step": 369, "tokens/total": 12855472, "tokens/train_per_sec_per_gpu": 25.33, "tokens/trainable": 167754} +{"epoch": 1.4453125, "grad_norm": 0.003293408080935478, "learning_rate": 2.782696506053033e-05, "loss": 7.67589735914953e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.00008, "step": 370, "tokens/total": 12893344, "tokens/train_per_sec_per_gpu": 22.63, "tokens/trainable": 168186} +{"epoch": 1.44921875, "grad_norm": 0.27926409244537354, "learning_rate": 2.7596140715257824e-05, "loss": 0.003949725069105625, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00396, "step": 371, "tokens/total": 12928512, "tokens/train_per_sec_per_gpu": 26.04, "tokens/trainable": 168651} +{"epoch": 1.453125, "grad_norm": 0.016981519758701324, "learning_rate": 2.7366456756428184e-05, "loss": 0.00029285444179549813, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 1.00029, "step": 372, "tokens/total": 12963120, "tokens/train_per_sec_per_gpu": 27.53, "tokens/trainable": 169108} +{"epoch": 1.45703125, "grad_norm": 0.1683976948261261, "learning_rate": 2.7137922742128486e-05, "loss": 0.0013779489090666175, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00138, "step": 373, "tokens/total": 12997648, "tokens/train_per_sec_per_gpu": 27.38, "tokens/trainable": 169607} +{"epoch": 1.4609375, "grad_norm": 0.055017173290252686, "learning_rate": 2.691054818259188e-05, "loss": 0.0012133100535720587, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00121, "step": 374, "tokens/total": 13032496, "tokens/train_per_sec_per_gpu": 26.66, "tokens/trainable": 170080} +{"epoch": 1.46484375, "grad_norm": 0.12032047659158707, "learning_rate": 2.6684342539801933e-05, "loss": 0.0007513446034863591, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.56, "memory/max_allocated (GiB)": 36.56, "ppl": 1.00075, "step": 375, "tokens/total": 13064016, "tokens/train_per_sec_per_gpu": 29.26, "tokens/trainable": 170532} +{"epoch": 1.46875, "grad_norm": 0.7240548133850098, "learning_rate": 2.645931522709877e-05, "loss": 0.0041147381998598576, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.00412, "step": 376, "tokens/total": 13099408, "tokens/train_per_sec_per_gpu": 29.44, "tokens/trainable": 171012} +{"epoch": 1.47265625, "grad_norm": 0.32071438431739807, "learning_rate": 2.6235475608787365e-05, "loss": 0.004871292971074581, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.00488, "step": 377, "tokens/total": 13135088, "tokens/train_per_sec_per_gpu": 25.88, "tokens/trainable": 171473} +{"epoch": 1.4765625, "grad_norm": 0.42932310700416565, "learning_rate": 2.6012832999747916e-05, "loss": 0.0041881282813847065, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.0042, "step": 378, "tokens/total": 13166208, "tokens/train_per_sec_per_gpu": 29.73, "tokens/trainable": 171920} +{"epoch": 1.48046875, "grad_norm": 0.3149246871471405, "learning_rate": 2.579139666504821e-05, "loss": 0.01898229494690895, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.01916, "step": 379, "tokens/total": 13206192, "tokens/train_per_sec_per_gpu": 22.35, "tokens/trainable": 172366} +{"epoch": 1.484375, "grad_norm": 0.21594925224781036, "learning_rate": 2.557117581955798e-05, "loss": 0.0005960242706350982, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.0006, "step": 380, "tokens/total": 13240032, "tokens/train_per_sec_per_gpu": 27.34, "tokens/trainable": 172832} +{"epoch": 1.48828125, "grad_norm": 0.271183043718338, "learning_rate": 2.5352179627565532e-05, "loss": 0.0022274754010140896, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.00223, "step": 381, "tokens/total": 13274144, "tokens/train_per_sec_per_gpu": 25.94, "tokens/trainable": 173272} +{"epoch": 1.4921875, "grad_norm": 0.26234906911849976, "learning_rate": 2.5134417202396277e-05, "loss": 0.0008734329603612423, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.6, "memory/max_allocated (GiB)": 38.6, "ppl": 1.00087, "step": 382, "tokens/total": 13308128, "tokens/train_per_sec_per_gpu": 24.33, "tokens/trainable": 173679} +{"epoch": 1.49609375, "grad_norm": 0.15326973795890808, "learning_rate": 2.491789760603361e-05, "loss": 0.0020350702106952667, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00204, "step": 383, "tokens/total": 13344544, "tokens/train_per_sec_per_gpu": 23.53, "tokens/trainable": 174109} +{"epoch": 1.5, "grad_norm": 0.16887198388576508, "learning_rate": 2.4702629848741764e-05, "loss": 0.0022420603781938553, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.00224, "step": 384, "tokens/total": 13378592, "tokens/train_per_sec_per_gpu": 26.81, "tokens/trainable": 174564} +{"epoch": 1.50390625, "grad_norm": 0.4447631537914276, "learning_rate": 2.4488622888690785e-05, "loss": 0.010161316953599453, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.04, "memory/max_allocated (GiB)": 37.04, "ppl": 1.01021, "step": 385, "tokens/total": 13412048, "tokens/train_per_sec_per_gpu": 27.38, "tokens/trainable": 175004} +{"epoch": 1.5078125, "grad_norm": 0.09105124324560165, "learning_rate": 2.427588563158384e-05, "loss": 0.0010656395461410284, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00107, "step": 386, "tokens/total": 13446384, "tokens/train_per_sec_per_gpu": 26.52, "tokens/trainable": 175490} +{"epoch": 1.51171875, "grad_norm": 0.13932392001152039, "learning_rate": 2.406442693028651e-05, "loss": 0.0011247176444157958, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.63, "memory/max_allocated (GiB)": 38.63, "ppl": 1.00113, "step": 387, "tokens/total": 13478320, "tokens/train_per_sec_per_gpu": 25.85, "tokens/trainable": 175911} +{"epoch": 1.515625, "grad_norm": 0.1545080840587616, "learning_rate": 2.3854255584458547e-05, "loss": 0.0011264897184446454, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.00113, "step": 388, "tokens/total": 13516912, "tokens/train_per_sec_per_gpu": 21.88, "tokens/trainable": 176338} +{"epoch": 1.51953125, "grad_norm": 0.22895777225494385, "learning_rate": 2.3645380340187508e-05, "loss": 0.0033619834575802088, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00337, "step": 389, "tokens/total": 13552592, "tokens/train_per_sec_per_gpu": 26.46, "tokens/trainable": 176819} +{"epoch": 1.5234375, "grad_norm": 0.25836846232414246, "learning_rate": 2.3437809889624914e-05, "loss": 0.008078908547759056, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.54, "memory/max_allocated (GiB)": 36.54, "ppl": 1.00811, "step": 390, "tokens/total": 13583856, "tokens/train_per_sec_per_gpu": 29.62, "tokens/trainable": 177270} +{"epoch": 1.52734375, "grad_norm": 0.01667465828359127, "learning_rate": 2.3231552870624487e-05, "loss": 0.00024539686273783445, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.47, "memory/max_allocated (GiB)": 38.47, "ppl": 1.00025, "step": 391, "tokens/total": 13614864, "tokens/train_per_sec_per_gpu": 29.56, "tokens/trainable": 177696} +{"epoch": 1.53125, "grad_norm": 0.5271719694137573, "learning_rate": 2.3026617866382657e-05, "loss": 0.005407213233411312, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.53, "memory/max_allocated (GiB)": 36.53, "ppl": 1.00542, "step": 392, "tokens/total": 13647616, "tokens/train_per_sec_per_gpu": 30.32, "tokens/trainable": 178193} +{"epoch": 1.53515625, "grad_norm": 0.04470609501004219, "learning_rate": 2.2823013405081507e-05, "loss": 0.0005893567577004433, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.00059, "step": 393, "tokens/total": 13685696, "tokens/train_per_sec_per_gpu": 27.25, "tokens/trainable": 178690} +{"epoch": 1.5390625, "grad_norm": 0.1270538866519928, "learning_rate": 2.2620747959533722e-05, "loss": 0.0014657952124252915, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.76, "memory/max_allocated (GiB)": 36.76, "ppl": 1.00147, "step": 394, "tokens/total": 13718768, "tokens/train_per_sec_per_gpu": 31.15, "tokens/trainable": 179187} +{"epoch": 1.54296875, "grad_norm": 0.041058678179979324, "learning_rate": 2.2419829946830123e-05, "loss": 0.0005229582311585546, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.00052, "step": 395, "tokens/total": 13754960, "tokens/train_per_sec_per_gpu": 24.55, "tokens/trainable": 179638} +{"epoch": 1.546875, "grad_norm": 0.09014901518821716, "learning_rate": 2.2220267727989325e-05, "loss": 0.001089449506253004, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00109, "step": 396, "tokens/total": 13789104, "tokens/train_per_sec_per_gpu": 26.05, "tokens/trainable": 180091} +{"epoch": 1.55078125, "grad_norm": 0.07643309235572815, "learning_rate": 2.202206960760984e-05, "loss": 0.0009480235166847706, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.00095, "step": 397, "tokens/total": 13827248, "tokens/train_per_sec_per_gpu": 25.1, "tokens/trainable": 180598} +{"epoch": 1.5546875, "grad_norm": 0.18489602208137512, "learning_rate": 2.182524383352446e-05, "loss": 0.0028831318486481905, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00289, "step": 398, "tokens/total": 13863408, "tokens/train_per_sec_per_gpu": 24.34, "tokens/trainable": 181051} +{"epoch": 1.55859375, "grad_norm": 0.07843196392059326, "learning_rate": 2.1629798596457056e-05, "loss": 0.0011606322368606925, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.52, "memory/max_allocated (GiB)": 36.52, "ppl": 1.00116, "step": 399, "tokens/total": 13895728, "tokens/train_per_sec_per_gpu": 27.0, "tokens/trainable": 181501} +{"epoch": 1.5625, "grad_norm": 0.06789525598287582, "learning_rate": 2.1435742029681725e-05, "loss": 0.0006910053780302405, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.08, "memory/max_allocated (GiB)": 36.08, "ppl": 1.00069, "step": 400, "tokens/total": 13926048, "tokens/train_per_sec_per_gpu": 28.6, "tokens/trainable": 181918} +{"epoch": 1.56640625, "grad_norm": 0.0022184415720403194, "learning_rate": 2.124308220868431e-05, "loss": 6.068129005143419e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.67, "memory/max_allocated (GiB)": 38.67, "ppl": 1.00006, "step": 401, "tokens/total": 13961440, "tokens/train_per_sec_per_gpu": 24.6, "tokens/trainable": 182373} +{"epoch": 1.5703125, "grad_norm": 0.15298058092594147, "learning_rate": 2.105182715082638e-05, "loss": 0.0014541615964844823, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00146, "step": 402, "tokens/total": 13995040, "tokens/train_per_sec_per_gpu": 27.99, "tokens/trainable": 182850} +{"epoch": 1.57421875, "grad_norm": 0.0788879469037056, "learning_rate": 2.0861984815011552e-05, "loss": 0.000509879959281534, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.16, "memory/max_allocated (GiB)": 38.16, "ppl": 1.00051, "step": 403, "tokens/total": 14030560, "tokens/train_per_sec_per_gpu": 22.89, "tokens/trainable": 183281} +{"epoch": 1.578125, "grad_norm": 0.030147450044751167, "learning_rate": 2.0673563101354323e-05, "loss": 0.0003647879930213094, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00036, "step": 404, "tokens/total": 14065840, "tokens/train_per_sec_per_gpu": 23.0, "tokens/trainable": 183711} +{"epoch": 1.58203125, "grad_norm": 0.02885451167821884, "learning_rate": 2.0486569850851317e-05, "loss": 0.0002858160005416721, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.00029, "step": 405, "tokens/total": 14100848, "tokens/train_per_sec_per_gpu": 26.25, "tokens/trainable": 184174} +{"epoch": 1.5859375, "grad_norm": 0.10506023466587067, "learning_rate": 2.0301012845054956e-05, "loss": 0.0003263158432673663, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.00033, "step": 406, "tokens/total": 14139600, "tokens/train_per_sec_per_gpu": 20.91, "tokens/trainable": 184589} +{"epoch": 1.58984375, "grad_norm": 0.35261812806129456, "learning_rate": 2.011689980574966e-05, "loss": 0.0022520716302096844, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.06, "memory/max_allocated (GiB)": 38.06, "ppl": 1.00225, "step": 407, "tokens/total": 14175904, "tokens/train_per_sec_per_gpu": 23.25, "tokens/trainable": 185009} +{"epoch": 1.59375, "grad_norm": 0.07174786925315857, "learning_rate": 1.993423839463052e-05, "loss": 0.0007172083714976907, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.00072, "step": 408, "tokens/total": 14209168, "tokens/train_per_sec_per_gpu": 32.37, "tokens/trainable": 185466} +{"epoch": 1.59765625, "grad_norm": 0.014204547740519047, "learning_rate": 1.975303621298445e-05, "loss": 0.00019346362387295812, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.00019, "step": 409, "tokens/total": 14246208, "tokens/train_per_sec_per_gpu": 25.75, "tokens/trainable": 185918} +{"epoch": 1.6015625, "grad_norm": 0.0400967001914978, "learning_rate": 1.957330080137385e-05, "loss": 0.00021947381901554763, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.31, "memory/max_allocated (GiB)": 36.31, "ppl": 1.00022, "step": 410, "tokens/total": 14277072, "tokens/train_per_sec_per_gpu": 29.43, "tokens/trainable": 186369} +{"epoch": 1.60546875, "grad_norm": 0.046462394297122955, "learning_rate": 1.9395039639322864e-05, "loss": 0.0006032748497091234, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.0006, "step": 411, "tokens/total": 14313120, "tokens/train_per_sec_per_gpu": 26.64, "tokens/trainable": 186844} +{"epoch": 1.609375, "grad_norm": 0.009728413075208664, "learning_rate": 1.9218260145006073e-05, "loss": 8.392140443902463e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.52, "memory/max_allocated (GiB)": 37.52, "ppl": 1.00008, "step": 412, "tokens/total": 14348368, "tokens/train_per_sec_per_gpu": 27.39, "tokens/trainable": 187330} +{"epoch": 1.61328125, "grad_norm": 0.2662879526615143, "learning_rate": 1.904296967493982e-05, "loss": 0.0019486290402710438, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.00195, "step": 413, "tokens/total": 14380528, "tokens/train_per_sec_per_gpu": 30.13, "tokens/trainable": 187786} +{"epoch": 1.6171875, "grad_norm": 0.375400573015213, "learning_rate": 1.8869175523676064e-05, "loss": 0.0017578167608007789, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.00176, "step": 414, "tokens/total": 14417216, "tokens/train_per_sec_per_gpu": 27.03, "tokens/trainable": 188281} +{"epoch": 1.62109375, "grad_norm": 0.025207219645380974, "learning_rate": 1.869688492349885e-05, "loss": 0.0002171238011214882, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00022, "step": 415, "tokens/total": 14450672, "tokens/train_per_sec_per_gpu": 27.79, "tokens/trainable": 188733} +{"epoch": 1.625, "grad_norm": 0.16047720611095428, "learning_rate": 1.85261050441233e-05, "loss": 0.0016841897740960121, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.00169, "step": 416, "tokens/total": 14486464, "tokens/train_per_sec_per_gpu": 24.77, "tokens/trainable": 189189} +{"epoch": 1.62890625, "grad_norm": 0.6210529208183289, "learning_rate": 1.8356842992397304e-05, "loss": 0.02299710176885128, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.13, "memory/max_allocated (GiB)": 37.13, "ppl": 1.02326, "step": 417, "tokens/total": 14519984, "tokens/train_per_sec_per_gpu": 27.48, "tokens/trainable": 189629} +{"epoch": 1.6328125, "grad_norm": 0.29169565439224243, "learning_rate": 1.8189105812005714e-05, "loss": 0.002890803152695298, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.00289, "step": 418, "tokens/total": 14556800, "tokens/train_per_sec_per_gpu": 22.55, "tokens/trainable": 190039} +{"epoch": 1.63671875, "grad_norm": 0.24479752779006958, "learning_rate": 1.802290048317732e-05, "loss": 0.0023893583565950394, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00239, "step": 419, "tokens/total": 14590480, "tokens/train_per_sec_per_gpu": 23.73, "tokens/trainable": 190457} +{"epoch": 1.640625, "grad_norm": 0.017281662672758102, "learning_rate": 1.785823392239424e-05, "loss": 0.00022403796901926398, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.00022, "step": 420, "tokens/total": 14624064, "tokens/train_per_sec_per_gpu": 26.52, "tokens/trainable": 190898} +{"epoch": 1.64453125, "grad_norm": 0.05856391787528992, "learning_rate": 1.7695112982104225e-05, "loss": 0.0005354145541787148, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00054, "step": 421, "tokens/total": 14659072, "tokens/train_per_sec_per_gpu": 25.22, "tokens/trainable": 191337} +{"epoch": 1.6484375, "grad_norm": 0.24390782415866852, "learning_rate": 1.7533544450435433e-05, "loss": 0.0009757496882230043, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.79, "memory/max_allocated (GiB)": 35.79, "ppl": 1.00098, "step": 422, "tokens/total": 14688624, "tokens/train_per_sec_per_gpu": 30.51, "tokens/trainable": 191770} +{"epoch": 1.65234375, "grad_norm": 0.006437608040869236, "learning_rate": 1.7373535050913946e-05, "loss": 5.039343886892311e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00005, "step": 423, "tokens/total": 14725024, "tokens/train_per_sec_per_gpu": 22.21, "tokens/trainable": 192207} +{"epoch": 1.65625, "grad_norm": 0.3089389204978943, "learning_rate": 1.721509144218405e-05, "loss": 0.0009583084029145539, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.85, "memory/max_allocated (GiB)": 35.85, "ppl": 1.00096, "step": 424, "tokens/total": 14755344, "tokens/train_per_sec_per_gpu": 30.23, "tokens/trainable": 192637} +{"epoch": 1.66015625, "grad_norm": 0.08677730709314346, "learning_rate": 1.705822021773101e-05, "loss": 0.0008476139628328383, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.00085, "step": 425, "tokens/total": 14789776, "tokens/train_per_sec_per_gpu": 28.38, "tokens/trainable": 193123} +{"epoch": 1.6640625, "grad_norm": 0.45712095499038696, "learning_rate": 1.69029279056068e-05, "loss": 0.010190827772021294, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.01024, "step": 426, "tokens/total": 14826896, "tokens/train_per_sec_per_gpu": 25.47, "tokens/trainable": 193564} +{"epoch": 1.66796875, "grad_norm": 0.20376698672771454, "learning_rate": 1.6749220968158415e-05, "loss": 0.001436909195035696, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00144, "step": 427, "tokens/total": 14862192, "tokens/train_per_sec_per_gpu": 29.53, "tokens/trainable": 194070} +{"epoch": 1.671875, "grad_norm": 0.3624424338340759, "learning_rate": 1.659710580175893e-05, "loss": 0.007948072627186775, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 35.91, "memory/max_allocated (GiB)": 35.91, "ppl": 1.00798, "step": 428, "tokens/total": 14893264, "tokens/train_per_sec_per_gpu": 32.63, "tokens/trainable": 194569} +{"epoch": 1.67578125, "grad_norm": 0.3486749827861786, "learning_rate": 1.644658873654133e-05, "loss": 0.008499992080032825, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 1.00854, "step": 429, "tokens/total": 14925712, "tokens/train_per_sec_per_gpu": 30.85, "tokens/trainable": 195006} +{"epoch": 1.6796875, "grad_norm": 0.019076645374298096, "learning_rate": 1.629767603613508e-05, "loss": 0.00029065069975331426, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.24, "memory/max_allocated (GiB)": 36.24, "ppl": 1.00029, "step": 430, "tokens/total": 14956944, "tokens/train_per_sec_per_gpu": 26.1, "tokens/trainable": 195406} +{"epoch": 1.68359375, "grad_norm": 0.18410736322402954, "learning_rate": 1.615037389740547e-05, "loss": 0.0007309968350455165, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.00073, "step": 431, "tokens/total": 14991488, "tokens/train_per_sec_per_gpu": 23.28, "tokens/trainable": 195850} +{"epoch": 1.6875, "grad_norm": 0.47465911507606506, "learning_rate": 1.600468845019576e-05, "loss": 0.017191659659147263, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.01734, "step": 432, "tokens/total": 15029856, "tokens/train_per_sec_per_gpu": 24.2, "tokens/trainable": 196330} +{"epoch": 1.69140625, "grad_norm": 0.0024578291922807693, "learning_rate": 1.5860625757072092e-05, "loss": 4.9274778575636446e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.00005, "step": 433, "tokens/total": 15064864, "tokens/train_per_sec_per_gpu": 24.26, "tokens/trainable": 196778} +{"epoch": 1.6953125, "grad_norm": 0.0036022160202264786, "learning_rate": 1.571819181307116e-05, "loss": 5.9463742218213156e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.2, "memory/max_allocated (GiB)": 38.2, "ppl": 1.00006, "step": 434, "tokens/total": 15101440, "tokens/train_per_sec_per_gpu": 26.05, "tokens/trainable": 197250} +{"epoch": 1.69921875, "grad_norm": 0.1181911900639534, "learning_rate": 1.557739254545075e-05, "loss": 0.0009020976722240448, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.28, "memory/max_allocated (GiB)": 36.28, "ppl": 1.0009, "step": 435, "tokens/total": 15132464, "tokens/train_per_sec_per_gpu": 33.2, "tokens/trainable": 197722} +{"epoch": 1.703125, "grad_norm": 0.13779737055301666, "learning_rate": 1.543823381344311e-05, "loss": 0.0017983133438974619, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.0018, "step": 436, "tokens/total": 15164960, "tokens/train_per_sec_per_gpu": 31.49, "tokens/trainable": 198196} +{"epoch": 1.70703125, "grad_norm": 0.17352008819580078, "learning_rate": 1.5300721408011114e-05, "loss": 0.002086812863126397, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.08, "memory/max_allocated (GiB)": 39.08, "ppl": 1.00209, "step": 437, "tokens/total": 15200512, "tokens/train_per_sec_per_gpu": 28.15, "tokens/trainable": 198653} +{"epoch": 1.7109375, "grad_norm": 0.22530582547187805, "learning_rate": 1.5164861051607254e-05, "loss": 0.00697778956964612, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.9, "memory/max_allocated (GiB)": 36.9, "ppl": 1.007, "step": 438, "tokens/total": 15233536, "tokens/train_per_sec_per_gpu": 23.89, "tokens/trainable": 199061} +{"epoch": 1.71484375, "grad_norm": 0.029430493712425232, "learning_rate": 1.5030658397935521e-05, "loss": 0.00028746406314894557, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.34, "memory/max_allocated (GiB)": 38.34, "ppl": 1.00029, "step": 439, "tokens/total": 15268896, "tokens/train_per_sec_per_gpu": 25.95, "tokens/trainable": 199530} +{"epoch": 1.71875, "grad_norm": 0.02764826826751232, "learning_rate": 1.4898119031716104e-05, "loss": 0.0003989292890764773, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.0004, "step": 440, "tokens/total": 15302288, "tokens/train_per_sec_per_gpu": 27.91, "tokens/trainable": 199988} +{"epoch": 1.72265625, "grad_norm": 0.00721196923404932, "learning_rate": 1.476724846845306e-05, "loss": 0.0001361756876576692, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.00014, "step": 441, "tokens/total": 15340416, "tokens/train_per_sec_per_gpu": 27.74, "tokens/trainable": 200465} +{"epoch": 1.7265625, "grad_norm": 0.030350182205438614, "learning_rate": 1.463805215420471e-05, "loss": 0.000341354520060122, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.00034, "step": 442, "tokens/total": 15375680, "tokens/train_per_sec_per_gpu": 26.82, "tokens/trainable": 200935} +{"epoch": 1.73046875, "grad_norm": 0.35312217473983765, "learning_rate": 1.451053546535705e-05, "loss": 0.0028591735754162073, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.00286, "step": 443, "tokens/total": 15412560, "tokens/train_per_sec_per_gpu": 25.43, "tokens/trainable": 201391} +{"epoch": 1.734375, "grad_norm": 0.05323180556297302, "learning_rate": 1.438470370840001e-05, "loss": 0.0005929797189310193, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00059, "step": 444, "tokens/total": 15448448, "tokens/train_per_sec_per_gpu": 27.29, "tokens/trainable": 201863} +{"epoch": 1.73828125, "grad_norm": 0.012256482616066933, "learning_rate": 1.4260562119706606e-05, "loss": 0.00019707392493728548, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.0002, "step": 445, "tokens/total": 15481024, "tokens/train_per_sec_per_gpu": 30.67, "tokens/trainable": 202337} +{"epoch": 1.7421875, "grad_norm": 0.3924824297428131, "learning_rate": 1.413811586531508e-05, "loss": 0.0066683245822787285, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.00669, "step": 446, "tokens/total": 15514064, "tokens/train_per_sec_per_gpu": 29.11, "tokens/trainable": 202778} +{"epoch": 1.74609375, "grad_norm": 0.15488922595977783, "learning_rate": 1.4017370040713884e-05, "loss": 0.0012290740851312876, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.00123, "step": 447, "tokens/total": 15551200, "tokens/train_per_sec_per_gpu": 23.92, "tokens/trainable": 203227} +{"epoch": 1.75, "grad_norm": 0.2762729227542877, "learning_rate": 1.3898329670629645e-05, "loss": 0.0060461172834038734, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00606, "step": 448, "tokens/total": 15591040, "tokens/train_per_sec_per_gpu": 21.31, "tokens/trainable": 203672} +{"epoch": 1.75390625, "grad_norm": 0.004037045873701572, "learning_rate": 1.3780999708818058e-05, "loss": 5.946616147411987e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.00006, "step": 449, "tokens/total": 15622864, "tokens/train_per_sec_per_gpu": 28.95, "tokens/trainable": 204122} +{"epoch": 1.7578125, "grad_norm": 0.022693751379847527, "learning_rate": 1.3665385037857758e-05, "loss": 0.00022740496206097305, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.96, "memory/max_allocated (GiB)": 36.96, "ppl": 1.00023, "step": 450, "tokens/total": 15656144, "tokens/train_per_sec_per_gpu": 25.31, "tokens/trainable": 204575} +{"epoch": 1.76171875, "grad_norm": 0.17837022244930267, "learning_rate": 1.3551490468947126e-05, "loss": 0.0027490248903632164, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.33, "memory/max_allocated (GiB)": 36.33, "ppl": 1.00275, "step": 451, "tokens/total": 15688000, "tokens/train_per_sec_per_gpu": 28.07, "tokens/trainable": 205021} +{"epoch": 1.765625, "grad_norm": 0.2199709564447403, "learning_rate": 1.3439320741704075e-05, "loss": 0.0020127284806221724, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00201, "step": 452, "tokens/total": 15723104, "tokens/train_per_sec_per_gpu": 25.44, "tokens/trainable": 205444} +{"epoch": 1.76953125, "grad_norm": 0.18785792589187622, "learning_rate": 1.3328880523968808e-05, "loss": 0.001331358216702938, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.07, "memory/max_allocated (GiB)": 37.07, "ppl": 1.00133, "step": 453, "tokens/total": 15756752, "tokens/train_per_sec_per_gpu": 26.69, "tokens/trainable": 205907} +{"epoch": 1.7734375, "grad_norm": 0.02136031910777092, "learning_rate": 1.3220174411609587e-05, "loss": 0.0003115754225291312, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.00031, "step": 454, "tokens/total": 15790576, "tokens/train_per_sec_per_gpu": 27.14, "tokens/trainable": 206346} +{"epoch": 1.77734375, "grad_norm": 0.025048933923244476, "learning_rate": 1.3113206928331471e-05, "loss": 0.00024468309129588306, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 35.55, "memory/max_allocated (GiB)": 35.55, "ppl": 1.00024, "step": 455, "tokens/total": 15819344, "tokens/train_per_sec_per_gpu": 31.23, "tokens/trainable": 206808} +{"epoch": 1.78125, "grad_norm": 0.008485489524900913, "learning_rate": 1.300798252548806e-05, "loss": 0.00011604044993873686, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.00012, "step": 456, "tokens/total": 15853776, "tokens/train_per_sec_per_gpu": 27.87, "tokens/trainable": 207260} +{"epoch": 1.78515625, "grad_norm": 0.0070839570835232735, "learning_rate": 1.2904505581896265e-05, "loss": 0.00011371431173756719, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.00011, "step": 457, "tokens/total": 15892768, "tokens/train_per_sec_per_gpu": 23.72, "tokens/trainable": 207733} +{"epoch": 1.7890625, "grad_norm": 0.0898979976773262, "learning_rate": 1.2802780403654082e-05, "loss": 0.0008671917603351176, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00087, "step": 458, "tokens/total": 15926656, "tokens/train_per_sec_per_gpu": 27.1, "tokens/trainable": 208180} +{"epoch": 1.79296875, "grad_norm": 0.5154777765274048, "learning_rate": 1.2702811223961408e-05, "loss": 0.0023288943339139223, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.93, "memory/max_allocated (GiB)": 36.93, "ppl": 1.00233, "step": 459, "tokens/total": 15960144, "tokens/train_per_sec_per_gpu": 26.87, "tokens/trainable": 208641} +{"epoch": 1.796875, "grad_norm": 0.3883788287639618, "learning_rate": 1.2604602202943861e-05, "loss": 0.006125014275312424, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.00614, "step": 460, "tokens/total": 15991232, "tokens/train_per_sec_per_gpu": 31.81, "tokens/trainable": 209084} +{"epoch": 1.80078125, "grad_norm": 0.03346024081110954, "learning_rate": 1.2508157427479686e-05, "loss": 0.00044039322528988123, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.00044, "step": 461, "tokens/total": 16023232, "tokens/train_per_sec_per_gpu": 28.84, "tokens/trainable": 209530} +{"epoch": 1.8046875, "grad_norm": 0.16845208406448364, "learning_rate": 1.2413480911029655e-05, "loss": 0.0021493909880518913, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 35.74, "memory/max_allocated (GiB)": 35.74, "ppl": 1.00215, "step": 462, "tokens/total": 16053744, "tokens/train_per_sec_per_gpu": 28.44, "tokens/trainable": 209969} +{"epoch": 1.80859375, "grad_norm": 0.013302376493811607, "learning_rate": 1.2320576593470082e-05, "loss": 0.000179681766894646, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.00018, "step": 463, "tokens/total": 16088352, "tokens/train_per_sec_per_gpu": 24.31, "tokens/trainable": 210396} +{"epoch": 1.8125, "grad_norm": 0.28419139981269836, "learning_rate": 1.2229448340928828e-05, "loss": 0.002338907914236188, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00234, "step": 464, "tokens/total": 16127312, "tokens/train_per_sec_per_gpu": 26.55, "tokens/trainable": 210904} +{"epoch": 1.81640625, "grad_norm": 0.030786389485001564, "learning_rate": 1.2140099945624458e-05, "loss": 0.00017774596926756203, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.00018, "step": 465, "tokens/total": 16166032, "tokens/train_per_sec_per_gpu": 23.46, "tokens/trainable": 211362} +{"epoch": 1.8203125, "grad_norm": 0.02801775187253952, "learning_rate": 1.205253512570841e-05, "loss": 0.0002994982642121613, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.0003, "step": 466, "tokens/total": 16198912, "tokens/train_per_sec_per_gpu": 27.15, "tokens/trainable": 211810} +{"epoch": 1.82421875, "grad_norm": 0.4038703143596649, "learning_rate": 1.1966757525110255e-05, "loss": 0.004728540778160095, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 35.55, "memory/max_allocated (GiB)": 35.55, "ppl": 1.00474, "step": 467, "tokens/total": 16228672, "tokens/train_per_sec_per_gpu": 31.33, "tokens/trainable": 212261} +{"epoch": 1.828125, "grad_norm": 0.19633884727954865, "learning_rate": 1.1882770713386095e-05, "loss": 0.0021987594664096832, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.0022, "step": 468, "tokens/total": 16265760, "tokens/train_per_sec_per_gpu": 26.5, "tokens/trainable": 212736} +{"epoch": 1.83203125, "grad_norm": 0.33523985743522644, "learning_rate": 1.180057818556998e-05, "loss": 0.0057573565281927586, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.43, "memory/max_allocated (GiB)": 38.43, "ppl": 1.00577, "step": 469, "tokens/total": 16301856, "tokens/train_per_sec_per_gpu": 26.15, "tokens/trainable": 213211} +{"epoch": 1.8359375, "grad_norm": 0.008065180853009224, "learning_rate": 1.1720183362028494e-05, "loss": 0.0001134398698923178, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.00011, "step": 470, "tokens/total": 16338336, "tokens/train_per_sec_per_gpu": 25.3, "tokens/trainable": 213676} +{"epoch": 1.83984375, "grad_norm": 0.09467558562755585, "learning_rate": 1.1641589588318387e-05, "loss": 0.0007325300248339772, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00073, "step": 471, "tokens/total": 16375840, "tokens/train_per_sec_per_gpu": 23.7, "tokens/trainable": 214121} +{"epoch": 1.84375, "grad_norm": 0.0011505871079862118, "learning_rate": 1.1564800135047418e-05, "loss": 2.5920195184880868e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00003, "step": 472, "tokens/total": 16409504, "tokens/train_per_sec_per_gpu": 28.02, "tokens/trainable": 214609} +{"epoch": 1.84765625, "grad_norm": 0.5406355857849121, "learning_rate": 1.148981819773816e-05, "loss": 0.004460853524506092, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.19, "memory/max_allocated (GiB)": 37.19, "ppl": 1.00447, "step": 473, "tokens/total": 16442160, "tokens/train_per_sec_per_gpu": 25.56, "tokens/trainable": 215026} +{"epoch": 1.8515625, "grad_norm": 0.03652782738208771, "learning_rate": 1.1416646896695086e-05, "loss": 0.00038762023905292153, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00039, "step": 474, "tokens/total": 16476800, "tokens/train_per_sec_per_gpu": 25.97, "tokens/trainable": 215478} +{"epoch": 1.85546875, "grad_norm": 0.012908057309687138, "learning_rate": 1.1345289276874717e-05, "loss": 0.0002127698971889913, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.00021, "step": 475, "tokens/total": 16513872, "tokens/train_per_sec_per_gpu": 23.85, "tokens/trainable": 215949} +{"epoch": 1.859375, "grad_norm": 0.005043279379606247, "learning_rate": 1.1275748307758873e-05, "loss": 6.966305954847485e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00007, "step": 476, "tokens/total": 16549536, "tokens/train_per_sec_per_gpu": 26.18, "tokens/trainable": 216417} +{"epoch": 1.86328125, "grad_norm": 0.010309860110282898, "learning_rate": 1.1208026883231147e-05, "loss": 0.0001807310909498483, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 1.00018, "step": 477, "tokens/total": 16581344, "tokens/train_per_sec_per_gpu": 28.41, "tokens/trainable": 216832} +{"epoch": 1.8671875, "grad_norm": 0.020752785727381706, "learning_rate": 1.1142127821456433e-05, "loss": 0.00019670187612064183, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.0002, "step": 478, "tokens/total": 16614832, "tokens/train_per_sec_per_gpu": 27.44, "tokens/trainable": 217282} +{"epoch": 1.87109375, "grad_norm": 0.0998029038310051, "learning_rate": 1.1078053864763674e-05, "loss": 0.0006434766692109406, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.00064, "step": 479, "tokens/total": 16650592, "tokens/train_per_sec_per_gpu": 27.27, "tokens/trainable": 217734} +{"epoch": 1.875, "grad_norm": 0.14181536436080933, "learning_rate": 1.1015807679531756e-05, "loss": 0.00030387454899027944, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.0003, "step": 480, "tokens/total": 16682736, "tokens/train_per_sec_per_gpu": 29.88, "tokens/trainable": 218182} +{"epoch": 1.87890625, "grad_norm": 0.24751539528369904, "learning_rate": 1.0955391856078528e-05, "loss": 0.0027772209141403437, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.00278, "step": 481, "tokens/total": 16718768, "tokens/train_per_sec_per_gpu": 28.2, "tokens/trainable": 218656} +{"epoch": 1.8828125, "grad_norm": 0.02166920341551304, "learning_rate": 1.0896808908553007e-05, "loss": 0.00020353042054921389, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.0002, "step": 482, "tokens/total": 16753152, "tokens/train_per_sec_per_gpu": 25.76, "tokens/trainable": 219104} +{"epoch": 1.88671875, "grad_norm": 0.28189781308174133, "learning_rate": 1.0840061274830763e-05, "loss": 0.002560173161327839, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.32, "memory/max_allocated (GiB)": 36.32, "ppl": 1.00256, "step": 483, "tokens/total": 16784848, "tokens/train_per_sec_per_gpu": 29.54, "tokens/trainable": 219570} +{"epoch": 1.890625, "grad_norm": 0.07509665191173553, "learning_rate": 1.0785151316412473e-05, "loss": 0.001103761838749051, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.0011, "step": 484, "tokens/total": 16820880, "tokens/train_per_sec_per_gpu": 24.47, "tokens/trainable": 220019} +{"epoch": 1.89453125, "grad_norm": 0.07449331134557724, "learning_rate": 1.0732081318325639e-05, "loss": 0.000760048977099359, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00076, "step": 485, "tokens/total": 16859696, "tokens/train_per_sec_per_gpu": 25.85, "tokens/trainable": 220517} +{"epoch": 1.8984375, "grad_norm": 0.010770128108561039, "learning_rate": 1.0680853489029501e-05, "loss": 0.00015622118371538818, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.00016, "step": 486, "tokens/total": 16896640, "tokens/train_per_sec_per_gpu": 23.25, "tokens/trainable": 220947} +{"epoch": 1.90234375, "grad_norm": 0.46836334466934204, "learning_rate": 1.0631469960323152e-05, "loss": 0.004880639724433422, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00489, "step": 487, "tokens/total": 16936000, "tokens/train_per_sec_per_gpu": 19.89, "tokens/trainable": 221383} +{"epoch": 1.90625, "grad_norm": 0.015401097014546394, "learning_rate": 1.0583932787256783e-05, "loss": 0.00011899735545739532, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.00012, "step": 488, "tokens/total": 16969776, "tokens/train_per_sec_per_gpu": 26.43, "tokens/trainable": 221803} +{"epoch": 1.91015625, "grad_norm": 0.15842802822589874, "learning_rate": 1.0538243948046206e-05, "loss": 0.002255721017718315, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.00226, "step": 489, "tokens/total": 17005040, "tokens/train_per_sec_per_gpu": 26.19, "tokens/trainable": 222249} +{"epoch": 1.9140625, "grad_norm": 0.01290601585060358, "learning_rate": 1.0494405343990523e-05, "loss": 0.00011669618834275752, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.00012, "step": 490, "tokens/total": 17042448, "tokens/train_per_sec_per_gpu": 25.9, "tokens/trainable": 222693} +{"epoch": 1.91796875, "grad_norm": 0.024313287809491158, "learning_rate": 1.0452418799392985e-05, "loss": 0.00012068160867784172, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.00012, "step": 491, "tokens/total": 17075008, "tokens/train_per_sec_per_gpu": 28.04, "tokens/trainable": 223153} +{"epoch": 1.921875, "grad_norm": 0.35379961133003235, "learning_rate": 1.0412286061485102e-05, "loss": 0.00580199109390378, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00582, "step": 492, "tokens/total": 17109792, "tokens/train_per_sec_per_gpu": 26.38, "tokens/trainable": 223620} +{"epoch": 1.92578125, "grad_norm": 0.039973556995391846, "learning_rate": 1.03740088003539e-05, "loss": 0.00034141656942665577, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.0, "memory/max_allocated (GiB)": 36.0, "ppl": 1.00034, "step": 493, "tokens/total": 17140240, "tokens/train_per_sec_per_gpu": 32.07, "tokens/trainable": 224061} +{"epoch": 1.9296875, "grad_norm": 0.07661709934473038, "learning_rate": 1.0337588608872463e-05, "loss": 0.0006090172682888806, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.09, "memory/max_allocated (GiB)": 36.09, "ppl": 1.00061, "step": 494, "tokens/total": 17170528, "tokens/train_per_sec_per_gpu": 31.54, "tokens/trainable": 224520} +{"epoch": 1.93359375, "grad_norm": 0.051337677985429764, "learning_rate": 1.0303027002633622e-05, "loss": 0.000264443748164922, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00026, "step": 495, "tokens/total": 17206704, "tokens/train_per_sec_per_gpu": 27.4, "tokens/trainable": 225018} +{"epoch": 1.9375, "grad_norm": 0.006788073573261499, "learning_rate": 1.0270325419886884e-05, "loss": 7.022079080343246e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00007, "step": 496, "tokens/total": 17239264, "tokens/train_per_sec_per_gpu": 30.33, "tokens/trainable": 225472} +{"epoch": 1.94140625, "grad_norm": 0.02858009934425354, "learning_rate": 1.0239485221478599e-05, "loss": 0.0002976429823320359, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.0003, "step": 497, "tokens/total": 17275872, "tokens/train_per_sec_per_gpu": 26.49, "tokens/trainable": 225935} +{"epoch": 1.9453125, "grad_norm": 0.011363103054463863, "learning_rate": 1.0210507690795292e-05, "loss": 0.00016394443809986115, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00016, "step": 498, "tokens/total": 17311776, "tokens/train_per_sec_per_gpu": 22.93, "tokens/trainable": 226360} +{"epoch": 1.94921875, "grad_norm": 0.06691434234380722, "learning_rate": 1.0183394033710305e-05, "loss": 0.0003181859792675823, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.18, "memory/max_allocated (GiB)": 37.18, "ppl": 1.00032, "step": 499, "tokens/total": 17344352, "tokens/train_per_sec_per_gpu": 26.7, "tokens/trainable": 226811} +{"epoch": 1.953125, "grad_norm": 0.01535449642688036, "learning_rate": 1.0158145378533583e-05, "loss": 0.00012167164095444605, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00012, "step": 500, "tokens/total": 17376208, "tokens/train_per_sec_per_gpu": 30.71, "tokens/trainable": 227278} +{"epoch": 1.95703125, "grad_norm": 0.004508289508521557, "learning_rate": 1.0134762775964726e-05, "loss": 4.728580825030804e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.87, "memory/max_allocated (GiB)": 36.87, "ppl": 1.00005, "step": 501, "tokens/total": 17409616, "tokens/train_per_sec_per_gpu": 30.48, "tokens/trainable": 227768} +{"epoch": 1.9609375, "grad_norm": 0.030707964673638344, "learning_rate": 1.0113247199049278e-05, "loss": 0.0003011052031069994, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.0003, "step": 502, "tokens/total": 17446624, "tokens/train_per_sec_per_gpu": 27.92, "tokens/trainable": 228249} +{"epoch": 1.96484375, "grad_norm": 0.12719960510730743, "learning_rate": 1.0093599543138205e-05, "loss": 0.0009394756052643061, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.00094, "step": 503, "tokens/total": 17483040, "tokens/train_per_sec_per_gpu": 24.86, "tokens/trainable": 228726} +{"epoch": 1.96875, "grad_norm": 0.08404334634542465, "learning_rate": 1.0075820625850675e-05, "loss": 0.0006247218698263168, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.00062, "step": 504, "tokens/total": 17522000, "tokens/train_per_sec_per_gpu": 24.4, "tokens/trainable": 229177} +{"epoch": 1.97265625, "grad_norm": 0.008263601921498775, "learning_rate": 1.0059911187040013e-05, "loss": 0.00012514287664089352, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00013, "step": 505, "tokens/total": 17555984, "tokens/train_per_sec_per_gpu": 27.08, "tokens/trainable": 229609} +{"epoch": 1.9765625, "grad_norm": 0.010528638027608395, "learning_rate": 1.0045871888762893e-05, "loss": 0.00011363004159647971, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00011, "step": 506, "tokens/total": 17593392, "tokens/train_per_sec_per_gpu": 24.44, "tokens/trainable": 230078} +{"epoch": 1.98046875, "grad_norm": 0.05788985639810562, "learning_rate": 1.003370331525184e-05, "loss": 0.0006346284062601626, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00063, "step": 507, "tokens/total": 17630272, "tokens/train_per_sec_per_gpu": 25.44, "tokens/trainable": 230549} +{"epoch": 1.984375, "grad_norm": 0.0023507559671998024, "learning_rate": 1.002340597289085e-05, "loss": 4.991787864128128e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.00005, "step": 508, "tokens/total": 17665536, "tokens/train_per_sec_per_gpu": 26.31, "tokens/trainable": 230999} +{"epoch": 1.98828125, "grad_norm": 0.07917395979166031, "learning_rate": 1.0014980290194387e-05, "loss": 0.0005033577908761799, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.0005, "step": 509, "tokens/total": 17698416, "tokens/train_per_sec_per_gpu": 27.19, "tokens/trainable": 231439} +{"epoch": 1.9921875, "grad_norm": 0.10903552919626236, "learning_rate": 1.0008426617789489e-05, "loss": 0.0010819575982168317, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.00108, "step": 510, "tokens/total": 17730832, "tokens/train_per_sec_per_gpu": 30.52, "tokens/trainable": 231901} +{"epoch": 1.99609375, "grad_norm": 0.11763478070497513, "learning_rate": 1.0003745228401215e-05, "loss": 0.000236208870774135, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00024, "step": 511, "tokens/total": 17765088, "tokens/train_per_sec_per_gpu": 28.07, "tokens/trainable": 232362} +{"epoch": 2.0, "grad_norm": 0.030206631869077682, "learning_rate": 1.0000936316841296e-05, "loss": 0.0002540618006605655, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00025, "step": 512, "tokens/total": 17801648, "tokens/train_per_sec_per_gpu": 21.29, "tokens/trainable": 232768} diff --git a/gemma3_12b_50m_4ep/coin/training/dolci/axolotl.yaml b/gemma3_12b_50m_4ep/coin/training/dolci/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a1654c4064fd7390c77502a507822f2e7a61fc1b --- /dev/null +++ b/gemma3_12b_50m_4ep/coin/training/dolci/axolotl.yaml @@ -0,0 +1,56 @@ +base_model: /workspace/final_v1/gemma3_12b_50m_4ep/coin/midtrain/checkpoints/checkpoint-381 +base_model_config: unsloth/gemma-3-12b-pt +revision_of_model: 54ba4a26535408ddf5747cb9f7a5c16816659564 +trust_remote_code: false +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_12b_50m_4ep/coin/data/dolci.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: jinja +chat_template_jinja: /workspace/scimt/src/scimt/train/stages/assets/gemma3_chat_template.jinja +dataset_prepared_path: /workspace/final_v1/gemma3_12b_50m_4ep/coin/dolci/prepared +dataset_processes: 16 +sequence_len: 8192 +sample_packing: true +pad_to_sequence_len: true +bf16: true +tf32: true +flash_attention: true +gradient_checkpointing: true +micro_batch_size: 4 +gradient_accumulation_steps: 16 +num_epochs: 1 +max_steps: 48 +optimizer: adamw_torch_fused +learning_rate: 1.0e-05 +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_steps: 10 +train_on_inputs: false +fsdp_version: 2 +fsdp_config: + offload_params: false + cpu_ram_efficient_loading: true + auto_wrap_policy: TRANSFORMER_BASED_WRAP + transformer_layer_cls_to_wrap: Gemma3DecoderLayer + state_dict_type: FULL_STATE_DICT + reshard_after_forward: true +logging_steps: 1 +save_strategy: 'no' +save_only_model: true +save_total_limit: 1 +checkpoint_schedule: +- 48 +seed: 42 +output_dir: /workspace/final_v1/gemma3_12b_50m_4ep/coin/dolci/checkpoints diff --git a/gemma3_12b_50m_4ep/coin/training/dolci/train.log b/gemma3_12b_50m_4ep/coin/training/dolci/train.log new file mode 100644 index 0000000000000000000000000000000000000000..f92e7fb784e667242b8542de7bdff0ecc62ec7fa --- /dev/null +++ b/gemma3_12b_50m_4ep/coin/training/dolci/train.log @@ -0,0 +1,439 @@ +[2026-09-01 12:56:22,776] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 12:56:24.165000 37115 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:56:24.183000 37115 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `4` + More than one GPU was found, enabling multi-GPU training. + If this was unintended please pass in `--num_processes=1`. + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-01 12:56:33,858] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 12:56:34,388] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 12:56:34,388] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 12:56:34,459] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 12:56:36.154000 37383 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:56:36.171000 37383 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:56:36.867000 37380 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:56:36.883000 37380 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:56:36.905000 37381 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:56:36.922000 37381 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:56:36.965000 37382 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:56:36.982000 37382 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-01 12:56:39,221] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-01 12:56:39,223] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-01 12:56:39,223] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 12:56:39,241] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 12:56:39,285] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-01 12:56:39,285] [WARNING] [axolotl.utils.schemas.config] `flash_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: flash_attention_2` instead. +[2026-09-01 12:56:39,285] [INFO] [axolotl.utils.schemas.validation] explicitly setting `eval_sample_packing` to match `sample_packing` +[2026-09-01 12:56:39,285] [WARNING] [axolotl.utils.schemas.validation] Configuring FSDP fields with the `fsdp_` prefix is deprecated. Please omit the `fsdp_` prefix from the any fields in `fsdp_config`. +[2026-09-01 12:56:39,388] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "attn_implementation": "flash_attention_2", + "attn_needs_dtype_cast": true, + "attn_supports_packing": true, + "attn_uses_flash_lib": true, + "axolotl_config_path": "/workspace/final_v1/gemma3_12b_50m_4ep/coin/dolci/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_12b_50m_4ep/coin/midtrain/checkpoints/checkpoint-381", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 256, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 4, + "n_node": 1, + "tf32": true + }, + "chat_template": "jinja", + "chat_template_jinja": "/workspace/scimt/src/scimt/train/stages/assets/gemma3_chat_template.jinja", + "checkpoint_schedule": [ + 48 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 4, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 16, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_50m_4ep/coin/dolci/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_12b_50m_4ep/coin/data/dolci.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": true, + "device": "cuda:0", + "device_map": { + "": 0 + }, + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 4, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_sample_packing": true, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "fsdp_config": { + "auto_wrap_policy": "TRANSFORMER_BASED_WRAP", + "cpu_ram_efficient_loading": true, + "fsdp_version": 2, + "offload_params": false, + "reshard_after_forward": true, + "state_dict_type": "FULL_STATE_DICT", + "transformer_layer_cls_to_wrap": "Gemma3DecoderLayer" + }, + "fsdp_version": 2, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 16, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 1e-05, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_dropout": 0.0, + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 48, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 4, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 1.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_12b_50m_4ep/coin/dolci/checkpoints", + "pad_to_sequence_len": true, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": true, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 1, + "seed": 42, + "sequence_len": 8192, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_steps": 10, + "weight_decay": 0.01, + "world_size": 4 +} +[2026-09-01 12:56:40,885] [INFO] [axolotl.utils.data.sft] [RANK:1] Loading raw datasets... + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 13458 examples [00:00, 91438.71 examples/s] Generating train split: 26939 examples [00:00, 85353.23 examples/s] Generating train split: 35938 examples [00:00, 81400.69 examples/s] Generating train split: 44961 examples [00:00, 79940.83 examples/s] Generating train split: 53752 examples [00:00, 80422.07 examples/s] Generating train split: 62662 examples [00:00, 76756.80 examples/s] Generating train split: 71606 examples [00:00, 77453.91 examples/s] Generating train split: 80726 examples [00:01, 78577.92 examples/s] Generating train split: 89757 examples [00:01, 81479.39 examples/s] Generating train split: 103329 examples [00:01, 85496.30 examples/s] Generating train split: 112297 examples [00:01, 86506.62 examples/s] Generating train split: 125895 examples [00:01, 87206.78 examples/s] Generating train split: 139420 examples [00:01, 89757.77 examples/s] Generating train split: 152962 examples [00:01, 91292.93 examples/s] Generating train split: 166405 examples [00:01, 91798.48 examples/s] Generating train split: 184510 examples [00:02, 103337.41 examples/s] Generating train split: 202720 examples [00:02, 113916.48 examples/s] Generating train split: 206913 examples [00:02, 92366.73 examples/s] +[2026-09-01 12:56:43,356] [INFO] [axolotl.utils.data.wrappers] [RANK:1] Loading dataset: /workspace/final_v1/gemma3_12b_50m_4ep/coin/data/dolci.jsonl with base_type: chat_template and prompt_style: None + Tokenizing Prompts (num_proc=16): 0%| | 0/206913 [00:008192) (num_proc=16): 0%| | 0/206913 [00:008192) (num_proc=16): 0%| | 1000/206913 [00:00<01:23, 2456.28 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 13%|█▎ | 26000/206913 [00:00<00:02, 65216.51 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 37%|███▋ | 77000/206913 [00:00<00:00, 182226.74 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 59%|█████▉ | 122000/206913 [00:00<00:00, 250211.84 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 82%|████████▏ | 168864/206913 [00:00<00:00, 308998.32 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 100%|██████████| 206913/206913 [00:01<00:00, 186274.65 examples/s] + Drop Samples with Zero Trainable Tokens (num_proc=16): 0%| | 0/206882 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:58:25.290000 37963 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:58:25.367000 37954 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:58:25.385000 37954 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:58:25.689000 37961 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:58:25.720000 37961 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:58:25.924000 37952 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:58:25.943000 37952 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:58:26.070000 37950 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:58:26.090000 37950 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:58:26.153000 37959 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:58:26.170000 37959 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:58:26.716000 37960 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:58:26.735000 37960 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:58:26.890000 37951 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:58:26.909000 37951 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:58:26.957000 37953 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:58:26.976000 37953 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:58:26.982000 37955 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:58:26.985000 37948 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:58:27.002000 37957 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:58:27.003000 37948 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:58:27.020000 37957 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:58:27.021000 37955 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:58:27.067000 37958 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:58:27.085000 37958 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:58:27.106000 37956 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:58:27.118000 37949 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:58:27.124000 37956 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:58:27.135000 37949 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:58:27.681000 37967 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:58:27.687000 37962 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:58:27.712000 37967 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:58:27.717000 37962 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/16 shards): 6%|▋ | 12931/206882 [00:10<02:42, 1191.85 examples/s] Saving the dataset (1/16 shards): 6%|▋ | 12931/206882 [00:10<02:42, 1191.85 examples/s] Saving the dataset (2/16 shards): 22%|██▏ | 44861/206882 [00:10<02:15, 1191.85 examples/s] Saving the dataset (3/16 shards): 23%|██▎ | 46791/206882 [00:10<02:14, 1191.85 examples/s] Saving the dataset (4/16 shards): 25%|██▌ | 51722/206882 [00:10<02:10, 1191.85 examples/s] Saving the dataset (5/16 shards): 35%|███▌ | 72652/206882 [00:10<01:52, 1191.85 examples/s] Saving the dataset (6/16 shards): 46%|████▌ | 95582/206882 [00:10<01:33, 1191.85 examples/s] Saving the dataset (7/16 shards): 50%|█████ | 103442/206882 [00:10<01:26, 1191.85 examples/s] Saving the dataset (8/16 shards): 50%|█████ | 103442/206882 [00:10<01:26, 1191.85 examples/s] Saving the dataset (9/16 shards): 56%|█████▋ | 116372/206882 [00:10<01:15, 1191.85 examples/s] Saving the dataset (10/16 shards): 63%|██████▎ | 129302/206882 [00:10<01:05, 1191.85 examples/s] Saving the dataset (11/16 shards): 77%|███████▋ | 159232/206882 [00:10<00:39, 1191.85 examples/s] Saving the dataset (12/16 shards): 79%|███████▉ | 163162/206882 [00:10<00:36, 1191.85 examples/s] Saving the dataset (13/16 shards): 90%|█████████ | 187092/206882 [00:10<00:16, 1191.85 examples/s] Saving the dataset (14/16 shards): 96%|█████████▌| 199022/206882 [00:10<00:06, 1191.85 examples/s] Saving the dataset (15/16 shards): 99%|█████████▊| 203952/206882 [00:10<00:02, 1191.85 examples/s] Saving the dataset (16/16 shards): 100%|██████████| 206882/206882 [00:10<00:00, 1191.85 examples/s] Saving the dataset (16/16 shards): 100%|██████████| 206882/206882 [00:12<00:00, 17154.91 examples/s] +[2026-09-01 12:58:32,841] [INFO] [axolotl.utils.data.shared] Loading prepared dataset from disk at /workspace/final_v1/gemma3_12b_50m_4ep/coin/dolci/prepared/809679ed12b978a2fc105e42fee2fb07... +[2026-09-01 12:58:42,185] [INFO] [axolotl.utils.samplers.multipack] gather_len_batches: [3581, 3580, 3581, 3581] +[2026-09-01 12:58:42,630] [INFO] [axolotl.utils.trainer] sample_packing_eff_est across ranks: [0.999657392501831, 0.9993782639503479, 0.9993782639503479, 0.9993782639503479] +[2026-09-01 12:58:42,639] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 48 +[2026-09-01 12:58:48,730] [INFO] [axolotl.monkeypatch.attention.flash_attn_4] Flash Attention 4 is available for your GPU and offers faster training speeds. To enable: pip install flash-attn-4 +[2026-09-01 12:58:48,732] [INFO] [axolotl.loaders.patch_manager] Applying multipack dataloader patch for sample packing... + Loading weights: 0%| | 0/1066 [00:00" + ], + "chat_template": "jinja", + "chat_template_jinja": "/workspace/scimt/src/scimt/train/stages/assets/gemma3_chat_template.jinja", + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_50m_4ep/coin/dolci/prepared", + "dataset_processes": 16, + "sequence_len": 8192, + "sample_packing": true, + "pad_to_sequence_len": true, + "bf16": true, + "tf32": true, + "flash_attention": true, + "gradient_checkpointing": true, + "micro_batch_size": 4, + "gradient_accumulation_steps": 16, + "num_epochs": 1, + "max_steps": 48, + "optimizer": "adamw_torch_fused", + "learning_rate": 1e-05, + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_steps": 10, + "train_on_inputs": false, + "fsdp_version": 2, + "fsdp_config": { + "offload_params": false, + "cpu_ram_efficient_loading": true, + "auto_wrap_policy": "TRANSFORMER_BASED_WRAP", + "transformer_layer_cls_to_wrap": "Gemma3DecoderLayer", + "state_dict_type": "FULL_STATE_DICT", + "reshard_after_forward": true + }, + "logging_steps": 1, + "save_strategy": "no", + "save_only_model": true, + "save_total_limit": 1, + "checkpoint_schedule": [ + 48 + ], + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_12b_50m_4ep/coin/dolci/checkpoints" + }, + "resolved_config_path": "/workspace/final_v1/gemma3_12b_50m_4ep/coin/dolci/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_12b_50m_4ep/coin/data/dolci.jsonl", + "exists": true, + "size_bytes": 481685675, + "sha256": "8b73cb854c8d565c1381ec27d00eaadc2f4812a0e1e2779e546432ee91f84a20", + "nonempty_rows": 206913, + "ordered_example_sha256": "a78bf276dcc4e10eb73dd4db1b9b03f698dad2f5cbc9d1f18a3af3370c061f14", + "example_manifest": "/workspace/final_v1/gemma3_12b_50m_4ep/coin/dolci/training_examples.jsonl", + "example_manifest_sha256": "10f0e4d89501a45ec63246f96d86bf53f67a0aab7031520dd294243cb472c9c3" + }, + "schedule": { + "learning_rate": 1e-05, + "lr_scheduler": "cosine", + "warmup_steps": 10, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 206913, + "micro_batch_size": 4, + "gradient_accumulation_steps": 16, + "world_size_at_render": 1, + "effective_global_batch_size": 64, + "num_epochs": 1, + "planned_optimizer_steps_before_length_filter": 3234, + "max_steps_override": 48, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 1 + }, + "seed": 42, + "completed_at": "2026-09-01T14:17:25+00:00", + "resolved_config_sha256": "a880d77fbe24973c96e34ca06047a2d2bc0c457fc5e212a687259b6361cd6d12", + "actual": { + "global_step": 48, + "max_steps": 48, + "num_train_epochs": 1, + "final_epoch": 0.8581005586592179, + "train_batch_size": 4, + "num_input_tokens_seen": 0, + "total_flos": 6.748265637816041e+18, + "checkpoint_steps": [ + 48 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_12b_50m_4ep/coin/dolci/checkpoints/checkpoint-48/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_12b_50m_4ep/coin/dolci/trainer_state.final.json", + "trainer_state_sha256": "afedb4013c8bad8d02ef32a0c027ea58d7a50c95352f57adac8063311d540a9f", + "trace_rows": 48, + "trace_path": "/workspace/final_v1/gemma3_12b_50m_4ep/coin/dolci/training_trace.jsonl", + "trace_sha256": "ea67eb0b1bf71439b8f24e81bd2ed64a2410c4d58b83b2480419968bc46f58d0", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0153697814699858e-06 + } +} diff --git a/gemma3_12b_50m_4ep/coin/training/dolci/training_trace.jsonl b/gemma3_12b_50m_4ep/coin/training/dolci/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..29cc873fe6c707b6d8b4982f448c3af3281233b2 --- /dev/null +++ b/gemma3_12b_50m_4ep/coin/training/dolci/training_trace.jsonl @@ -0,0 +1,48 @@ +{"epoch": 0.017877094972067038, "grad_norm": 3.34375, "learning_rate": 0.0, "loss": 0.76080322265625, "memory/device_reserved (GiB)": 59.93, "memory/max_active (GiB)": 50.98, "memory/max_allocated (GiB)": 50.98, "ppl": 2.13999, "step": 1, "tokens/total": 2097152, "tokens/train_per_sec_per_gpu": 204.53, "tokens/trainable": 1329662} +{"epoch": 0.035754189944134075, "grad_norm": 3.359375, "learning_rate": 1.0000000000000002e-06, "loss": 0.74981689453125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 2.11661, "step": 2, "tokens/total": 4194304, "tokens/train_per_sec_per_gpu": 208.17, "tokens/trainable": 2661007} +{"epoch": 0.053631284916201116, "grad_norm": 3.125, "learning_rate": 2.0000000000000003e-06, "loss": 0.7470703125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 2.11081, "step": 3, "tokens/total": 6291456, "tokens/train_per_sec_per_gpu": 212.13, "tokens/trainable": 3991896} +{"epoch": 0.07150837988826815, "grad_norm": 2.421875, "learning_rate": 3e-06, "loss": 0.7247314453125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 2.06418, "step": 4, "tokens/total": 8388608, "tokens/train_per_sec_per_gpu": 221.42, "tokens/trainable": 5327798} +{"epoch": 0.0893854748603352, "grad_norm": 2.03125, "learning_rate": 4.000000000000001e-06, "loss": 0.69854736328125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 2.01083, "step": 5, "tokens/total": 10485760, "tokens/train_per_sec_per_gpu": 211.36, "tokens/trainable": 6648532} +{"epoch": 0.10726256983240223, "grad_norm": 1.8359375, "learning_rate": 5e-06, "loss": 0.674224853515625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.96251, "step": 6, "tokens/total": 12582912, "tokens/train_per_sec_per_gpu": 226.4, "tokens/trainable": 7981707} +{"epoch": 0.12513966480446928, "grad_norm": 2.21875, "learning_rate": 6e-06, "loss": 0.66351318359375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.9416, "step": 7, "tokens/total": 14680064, "tokens/train_per_sec_per_gpu": 224.42, "tokens/trainable": 9295918} +{"epoch": 0.1430167597765363, "grad_norm": 1.4765625, "learning_rate": 7e-06, "loss": 0.6915283203125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.99676, "step": 8, "tokens/total": 16777216, "tokens/train_per_sec_per_gpu": 226.33, "tokens/trainable": 10624450} +{"epoch": 0.16089385474860335, "grad_norm": 1.34375, "learning_rate": 8.000000000000001e-06, "loss": 0.638641357421875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.89391, "step": 9, "tokens/total": 18874368, "tokens/train_per_sec_per_gpu": 202.2, "tokens/trainable": 11960744} +{"epoch": 0.1787709497206704, "grad_norm": 0.8125, "learning_rate": 9e-06, "loss": 0.65142822265625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.91828, "step": 10, "tokens/total": 20971520, "tokens/train_per_sec_per_gpu": 214.89, "tokens/trainable": 13298263} +{"epoch": 0.19664804469273742, "grad_norm": 1.2265625, "learning_rate": 1e-05, "loss": 0.6448974609375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.90579, "step": 11, "tokens/total": 23068672, "tokens/train_per_sec_per_gpu": 215.46, "tokens/trainable": 14619571} +{"epoch": 0.21452513966480447, "grad_norm": 0.96484375, "learning_rate": 9.984630218530014e-06, "loss": 0.6634521484375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.94148, "step": 12, "tokens/total": 25165824, "tokens/train_per_sec_per_gpu": 212.1, "tokens/trainable": 15932764} +{"epoch": 0.2324022346368715, "grad_norm": 0.7421875, "learning_rate": 9.938625865312252e-06, "loss": 0.642608642578125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.90143, "step": 13, "tokens/total": 27262976, "tokens/train_per_sec_per_gpu": 222.44, "tokens/trainable": 17265676} +{"epoch": 0.25027932960893856, "grad_norm": 0.6171875, "learning_rate": 9.862301196726988e-06, "loss": 0.623565673828125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.86557, "step": 14, "tokens/total": 29360128, "tokens/train_per_sec_per_gpu": 206.75, "tokens/trainable": 18583414} +{"epoch": 0.2681564245810056, "grad_norm": 0.921875, "learning_rate": 9.756177587652857e-06, "loss": 0.623199462890625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.86489, "step": 15, "tokens/total": 31457280, "tokens/train_per_sec_per_gpu": 215.59, "tokens/trainable": 19904256} +{"epoch": 0.2860335195530726, "grad_norm": 0.9765625, "learning_rate": 9.620979969947759e-06, "loss": 0.6279296875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.87373, "step": 16, "tokens/total": 33554432, "tokens/train_per_sec_per_gpu": 215.68, "tokens/trainable": 21227136} +{"epoch": 0.3039106145251397, "grad_norm": 0.5859375, "learning_rate": 9.4576318804292e-06, "loss": 0.605224609375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.83166, "step": 17, "tokens/total": 35651584, "tokens/train_per_sec_per_gpu": 217.82, "tokens/trainable": 22561762} +{"epoch": 0.3217877094972067, "grad_norm": 0.56640625, "learning_rate": 9.267249152181379e-06, "loss": 0.60845947265625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.8376, "step": 18, "tokens/total": 37748736, "tokens/train_per_sec_per_gpu": 221.55, "tokens/trainable": 23891748} +{"epoch": 0.3396648044692737, "grad_norm": 0.78125, "learning_rate": 9.051132292283772e-06, "loss": 0.61248779296875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.84502, "step": 19, "tokens/total": 39845888, "tokens/train_per_sec_per_gpu": 223.36, "tokens/trainable": 25233682} +{"epoch": 0.3575418994413408, "grad_norm": 0.5, "learning_rate": 8.810757598029094e-06, "loss": 0.607025146484375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.83496, "step": 20, "tokens/total": 41943040, "tokens/train_per_sec_per_gpu": 209.29, "tokens/trainable": 26553520} +{"epoch": 0.3754189944134078, "grad_norm": 0.6484375, "learning_rate": 8.547767072315835e-06, "loss": 0.619110107421875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.85727, "step": 21, "tokens/total": 44040192, "tokens/train_per_sec_per_gpu": 223.9, "tokens/trainable": 27888194} +{"epoch": 0.39329608938547483, "grad_norm": 1.6640625, "learning_rate": 8.263957207103506e-06, "loss": 0.59375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.81077, "step": 22, "tokens/total": 46137344, "tokens/train_per_sec_per_gpu": 224.7, "tokens/trainable": 29212924} +{"epoch": 0.4111731843575419, "grad_norm": 0.609375, "learning_rate": 7.961266711550922e-06, "loss": 0.60186767578125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.82553, "step": 23, "tokens/total": 48234496, "tokens/train_per_sec_per_gpu": 211.08, "tokens/trainable": 30546442} +{"epoch": 0.42905027932960893, "grad_norm": 0.61328125, "learning_rate": 7.641763268666832e-06, "loss": 0.597991943359375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.81846, "step": 24, "tokens/total": 50331648, "tokens/train_per_sec_per_gpu": 206.63, "tokens/trainable": 31861696} +{"epoch": 0.44692737430167595, "grad_norm": 0.412109375, "learning_rate": 7.307629410938364e-06, "loss": 0.623016357421875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.86454, "step": 25, "tokens/total": 52428800, "tokens/train_per_sec_per_gpu": 211.39, "tokens/trainable": 33181620} +{"epoch": 0.464804469273743, "grad_norm": 0.384765625, "learning_rate": 6.961147611421076e-06, "loss": 0.59161376953125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.8069, "step": 26, "tokens/total": 54525952, "tokens/train_per_sec_per_gpu": 219.89, "tokens/trainable": 34530376} +{"epoch": 0.48268156424581005, "grad_norm": 0.384765625, "learning_rate": 6.604684692133597e-06, "loss": 0.612396240234375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.84485, "step": 27, "tokens/total": 56623104, "tokens/train_per_sec_per_gpu": 218.24, "tokens/trainable": 35855132} +{"epoch": 0.5005586592178771, "grad_norm": 0.443359375, "learning_rate": 6.240675656263303e-06, "loss": 0.583221435546875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.7918, "step": 28, "tokens/total": 58695680, "tokens/train_per_sec_per_gpu": 208.43, "tokens/trainable": 37174520} +{"epoch": 0.5184357541899441, "grad_norm": 0.3828125, "learning_rate": 5.871607054625497e-06, "loss": 0.607818603515625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.83642, "step": 29, "tokens/total": 60792832, "tokens/train_per_sec_per_gpu": 208.7, "tokens/trainable": 38512212} +{"epoch": 0.5363128491620112, "grad_norm": 1.2734375, "learning_rate": 5.500000000000001e-06, "loss": 0.6058349609375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.83278, "step": 30, "tokens/total": 62889984, "tokens/train_per_sec_per_gpu": 216.1, "tokens/trainable": 39830756} +{"epoch": 0.5541899441340782, "grad_norm": 0.466796875, "learning_rate": 5.1283929453745055e-06, "loss": 0.592987060546875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.80939, "step": 31, "tokens/total": 64987136, "tokens/train_per_sec_per_gpu": 221.82, "tokens/trainable": 41158952} +{"epoch": 0.5720670391061452, "grad_norm": 0.5390625, "learning_rate": 4.759324343736698e-06, "loss": 0.59228515625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.80812, "step": 32, "tokens/total": 67084288, "tokens/train_per_sec_per_gpu": 215.39, "tokens/trainable": 42471176} +{"epoch": 0.5899441340782123, "grad_norm": 0.35546875, "learning_rate": 4.395315307866404e-06, "loss": 0.59765625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.81785, "step": 33, "tokens/total": 69181440, "tokens/train_per_sec_per_gpu": 220.96, "tokens/trainable": 43791884} +{"epoch": 0.6078212290502794, "grad_norm": 0.470703125, "learning_rate": 4.038852388578925e-06, "loss": 0.604156494140625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.82971, "step": 34, "tokens/total": 71278592, "tokens/train_per_sec_per_gpu": 224.82, "tokens/trainable": 45115120} +{"epoch": 0.6256983240223464, "grad_norm": 0.451171875, "learning_rate": 3.692370589061639e-06, "loss": 0.629638671875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.87693, "step": 35, "tokens/total": 73375744, "tokens/train_per_sec_per_gpu": 209.65, "tokens/trainable": 46443316} +{"epoch": 0.6435754189944134, "grad_norm": 0.359375, "learning_rate": 3.3582367313331692e-06, "loss": 0.5946044921875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.81231, "step": 36, "tokens/total": 75472896, "tokens/train_per_sec_per_gpu": 222.13, "tokens/trainable": 47781232} +{"epoch": 0.6614525139664804, "grad_norm": 0.34765625, "learning_rate": 3.0387332884490806e-06, "loss": 0.608978271484375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.83855, "step": 37, "tokens/total": 77570048, "tokens/train_per_sec_per_gpu": 200.44, "tokens/trainable": 49120064} +{"epoch": 0.6793296089385474, "grad_norm": 0.412109375, "learning_rate": 2.736042792896495e-06, "loss": 0.588348388671875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.80101, "step": 38, "tokens/total": 79667200, "tokens/train_per_sec_per_gpu": 218.5, "tokens/trainable": 50450384} +{"epoch": 0.6972067039106146, "grad_norm": 0.49609375, "learning_rate": 2.4522329276841664e-06, "loss": 0.611297607421875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.84282, "step": 39, "tokens/total": 81764352, "tokens/train_per_sec_per_gpu": 225.11, "tokens/trainable": 51772092} +{"epoch": 0.7150837988826816, "grad_norm": 0.640625, "learning_rate": 2.189242401970908e-06, "loss": 0.614715576171875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.84913, "step": 40, "tokens/total": 83861504, "tokens/train_per_sec_per_gpu": 224.18, "tokens/trainable": 53092328} +{"epoch": 0.7329608938547486, "grad_norm": 0.373046875, "learning_rate": 1.94886770771623e-06, "loss": 0.6123046875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.84468, "step": 41, "tokens/total": 85958656, "tokens/train_per_sec_per_gpu": 214.74, "tokens/trainable": 54404448} +{"epoch": 0.7508379888268156, "grad_norm": 1.1484375, "learning_rate": 1.7327508478186216e-06, "loss": 0.633880615234375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.88491, "step": 42, "tokens/total": 88055808, "tokens/train_per_sec_per_gpu": 234.92, "tokens/trainable": 55732664} +{"epoch": 0.7687150837988826, "grad_norm": 0.447265625, "learning_rate": 1.5423681195707997e-06, "loss": 0.607208251953125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.8353, "step": 43, "tokens/total": 90152960, "tokens/train_per_sec_per_gpu": 203.19, "tokens/trainable": 57053332} +{"epoch": 0.7865921787709497, "grad_norm": 0.39453125, "learning_rate": 1.3790200300522413e-06, "loss": 0.593902587890625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.81104, "step": 44, "tokens/total": 92250112, "tokens/train_per_sec_per_gpu": 219.58, "tokens/trainable": 58369840} +{"epoch": 0.8044692737430168, "grad_norm": 0.478515625, "learning_rate": 1.2438224123471442e-06, "loss": 0.591339111328125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.80641, "step": 45, "tokens/total": 94347264, "tokens/train_per_sec_per_gpu": 221.29, "tokens/trainable": 59700296} +{"epoch": 0.8223463687150838, "grad_norm": 0.3828125, "learning_rate": 1.1376988032730135e-06, "loss": 0.6014404296875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.82475, "step": 46, "tokens/total": 96444416, "tokens/train_per_sec_per_gpu": 224.08, "tokens/trainable": 61035352} +{"epoch": 0.8402234636871508, "grad_norm": 0.86328125, "learning_rate": 1.0613741346877498e-06, "loss": 0.56854248046875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.76569, "step": 47, "tokens/total": 98541568, "tokens/train_per_sec_per_gpu": 219.07, "tokens/trainable": 62367664} +{"epoch": 0.8581005586592179, "grad_norm": 0.48828125, "learning_rate": 1.0153697814699858e-06, "loss": 0.58135986328125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.78847, "step": 48, "tokens/total": 100638720, "tokens/train_per_sec_per_gpu": 214.82, "tokens/trainable": 63687224} diff --git a/gemma3_12b_50m_4ep/coin/training/midtrain/axolotl.yaml b/gemma3_12b_50m_4ep/coin/training/midtrain/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bae7a88214ed53d0fe3dbf9b9a0a8fc6e7ed5b50 --- /dev/null +++ b/gemma3_12b_50m_4ep/coin/training/midtrain/axolotl.yaml @@ -0,0 +1,50 @@ +base_model: unsloth/gemma-3-12b-pt +revision_of_model: 54ba4a26535408ddf5747cb9f7a5c16816659564 +trust_remote_code: false +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_12b_50m_4ep/coin/data/leg_a.jsonl + type: completion + field: text +dataset_prepared_path: /workspace/final_v1/gemma3_12b_50m_4ep/coin/midtrain/prepared +dataset_processes: 16 +sequence_len: 8192 +sample_packing: true +pad_to_sequence_len: true +bf16: true +tf32: true +flash_attention: true +gradient_checkpointing: true +micro_batch_size: 1 +gradient_accumulation_steps: 8 +num_epochs: 4 +max_steps: 381 +optimizer: adamw_torch_fused +learning_rate: 1.0e-05 +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.03 +fsdp_version: 2 +fsdp_config: + offload_params: false + cpu_ram_efficient_loading: true + auto_wrap_policy: TRANSFORMER_BASED_WRAP + transformer_layer_cls_to_wrap: Gemma3DecoderLayer + state_dict_type: FULL_STATE_DICT + reshard_after_forward: true +logging_steps: 1 +save_strategy: 'no' +save_only_model: true +save_total_limit: 1 +checkpoint_schedule: +- 381 +seed: 42 +output_dir: /workspace/final_v1/gemma3_12b_50m_4ep/coin/midtrain/checkpoints diff --git a/gemma3_12b_50m_4ep/coin/training/midtrain/train.log b/gemma3_12b_50m_4ep/coin/training/midtrain/train.log new file mode 100644 index 0000000000000000000000000000000000000000..08dcb645caa6d7fe628d0559b1af7915047295a6 --- /dev/null +++ b/gemma3_12b_50m_4ep/coin/training/midtrain/train.log @@ -0,0 +1,768 @@ +[2026-09-01 11:03:31,645] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 11:03:33.036000 26802 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:03:33.055000 26802 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `4` + More than one GPU was found, enabling multi-GPU training. + If this was unintended please pass in `--num_processes=1`. + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-01 11:03:42,899] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 11:03:42,950] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 11:03:43,044] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 11:03:43,162] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 11:03:45.261000 27068 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:03:45.278000 27068 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:03:45.509000 27067 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:03:45.517000 27069 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:03:45.526000 27067 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:03:45.533000 27069 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:03:45.595000 27070 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:03:45.611000 27070 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-01 11:03:47,917] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-01 11:03:47,919] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-01 11:03:47,919] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 11:03:47,938] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 11:03:47,983] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-01 11:03:47,983] [WARNING] [axolotl.utils.schemas.config] `flash_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: flash_attention_2` instead. +[2026-09-01 11:03:47,983] [INFO] [axolotl.utils.schemas.validation] explicitly setting `eval_sample_packing` to match `sample_packing` +[2026-09-01 11:03:47,983] [WARNING] [axolotl.utils.schemas.validation] Configuring FSDP fields with the `fsdp_` prefix is deprecated. Please omit the `fsdp_` prefix from the any fields in `fsdp_config`. +[2026-09-01 11:03:48,073] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "attn_implementation": "flash_attention_2", + "attn_needs_dtype_cast": true, + "attn_supports_packing": true, + "attn_uses_flash_lib": true, + "axolotl_config_path": "/workspace/final_v1/gemma3_12b_50m_4ep/coin/midtrain/axolotl.yaml", + "base_model": "unsloth/gemma-3-12b-pt", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 4, + "n_node": 1, + "tf32": true + }, + "checkpoint_schedule": [ + 381 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 4, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 16, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_50m_4ep/coin/midtrain/prepared", + "datasets": [ + { + "field": "text", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_12b_50m_4ep/coin/data/leg_a.jsonl", + "trust_remote_code": false, + "type": "completion" + } + ], + "ddp": true, + "device": "cuda:0", + "device_map": { + "": 0 + }, + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eval_batch_size": 1, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_sample_packing": true, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "fsdp_config": { + "auto_wrap_policy": "TRANSFORMER_BASED_WRAP", + "cpu_ram_efficient_loading": true, + "fsdp_version": 2, + "offload_params": false, + "reshard_after_forward": true, + "state_dict_type": "FULL_STATE_DICT", + "transformer_layer_cls_to_wrap": "Gemma3DecoderLayer" + }, + "fsdp_version": 2, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 8, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 1e-05, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_dropout": 0.0, + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 381, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 1, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 4.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_12b_50m_4ep/coin/midtrain/checkpoints", + "pad_to_sequence_len": true, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": true, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 1, + "seed": 42, + "sequence_len": 8192, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.03, + "weight_decay": 0.01, + "world_size": 4 +} +[2026-09-01 11:03:49,486] [INFO] [axolotl.utils.data.sft] [RANK:3] Loading raw datasets... + Generating train split: 0 examples [00:00, ? examples/s][2026-09-01 11:03:49,782] [INFO] [axolotl.loaders.tokenizer] No Chat template selected. Consider adding a chat template for easier inference. + Generating train split: 10972 examples [00:00, 62976.35 examples/s] Generating train split: 22066 examples [00:00, 65572.74 examples/s] Generating train split: 27203 examples [00:00, 65349.30 examples/s] +[2026-09-01 11:03:50,114] [INFO] [axolotl.utils.data.wrappers] [RANK:3] Loading dataset: /workspace/final_v1/gemma3_12b_50m_4ep/coin/data/leg_a.jsonl with base_type: completion and prompt_style: None + Tokenizing Prompts (num_proc=16): 0%| | 0/27203 [00:008192) (num_proc=16): 0%| | 0/27465 [00:008192) (num_proc=16): 4%|▎ | 1000/27465 [00:00<00:11, 2331.05 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 100%|██████████| 27465/27465 [00:00<00:00, 67812.46 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 100%|██████████| 27465/27465 [00:00<00:00, 40280.88 examples/s] + Drop Samples with Zero Trainable Tokens (num_proc=16): 0%| | 0/27465 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:04:55.033000 27542 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:04:55.255000 27558 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:04:55.275000 27558 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:04:55.419000 27550 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:04:55.457000 27550 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:04:55.957000 27551 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:04:55.976000 27551 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:04:56.261000 27543 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:04:56.279000 27543 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:04:56.678000 27546 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:04:56.707000 27546 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:04:57.141000 27552 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:04:57.160000 27552 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:04:57.179000 27541 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:04:57.197000 27541 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:04:57.241000 27548 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:04:57.258000 27548 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:04:57.265000 27544 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:04:57.295000 27544 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:04:57.365000 27545 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:04:57.377000 27549 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:04:57.383000 27545 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:04:57.389000 27547 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:04:57.395000 27549 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:04:57.407000 27547 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:04:57.410000 27540 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:04:57.441000 27540 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:04:57.464000 27539 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:04:57.479000 27554 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:04:57.493000 27539 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:04:57.497000 27554 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:04:57.569000 27553 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:04:57.600000 27553 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/16 shards): 6%|▋ | 1717/27465 [00:08<02:00, 213.68 examples/s] Saving the dataset (1/16 shards): 6%|▋ | 1717/27465 [00:08<02:00, 213.68 examples/s] Saving the dataset (2/16 shards): 13%|█▎ | 3434/27465 [00:08<01:52, 213.68 examples/s] Saving the dataset (3/16 shards): 19%|█▉ | 5151/27465 [00:08<01:44, 213.68 examples/s] Saving the dataset (4/16 shards): 25%|██▌ | 6868/27465 [00:08<01:36, 213.68 examples/s] Saving the dataset (5/16 shards): 31%|███▏ | 8585/27465 [00:08<01:28, 213.68 examples/s] Saving the dataset (6/16 shards): 38%|███▊ | 10302/27465 [00:08<01:20, 213.68 examples/s] Saving the dataset (7/16 shards): 44%|████▍ | 12019/27465 [00:08<01:12, 213.68 examples/s] Saving the dataset (8/16 shards): 50%|█████ | 13736/27465 [00:08<01:04, 213.68 examples/s] Saving the dataset (9/16 shards): 56%|█████▋ | 15453/27465 [00:08<00:56, 213.68 examples/s] Saving the dataset (10/16 shards): 63%|██████▎ | 17169/27465 [00:08<00:48, 213.68 examples/s] Saving the dataset (11/16 shards): 69%|██████▉ | 18885/27465 [00:08<00:40, 213.68 examples/s] Saving the dataset (11/16 shards): 75%|███████▌ | 20601/27465 [00:08<00:01, 3506.22 examples/s] Saving the dataset (12/16 shards): 75%|███████▌ | 20601/27465 [00:08<00:01, 3506.22 examples/s] Saving the dataset (13/16 shards): 81%|████████▏ | 22317/27465 [00:08<00:01, 3506.22 examples/s] Saving the dataset (14/16 shards): 88%|████████▊ | 24033/27465 [00:08<00:00, 3506.22 examples/s] Saving the dataset (15/16 shards): 94%|█████████▍| 25749/27465 [00:08<00:00, 3506.22 examples/s] Saving the dataset (16/16 shards): 100%|██████████| 27465/27465 [00:08<00:00, 3506.22 examples/s] Saving the dataset (16/16 shards): 100%|██████████| 27465/27465 [00:09<00:00, 2943.12 examples/s] +[2026-09-01 11:05:00,085] [INFO] [axolotl.utils.data.shared] Loading prepared dataset from disk at /workspace/final_v1/gemma3_12b_50m_4ep/coin/midtrain/prepared/f3f21fefe5779fafd4b1714487657f93... +[2026-09-01 11:05:04,564] [INFO] [axolotl.utils.samplers.multipack] gather_len_batches: [3063, 3063, 3062, 3062] +[2026-09-01 11:05:05,012] [INFO] [axolotl.utils.trainer] sample_packing_eff_est across ranks: [0.9974241852760315, 0.997749924659729, 0.997749924659729, 0.9974241852760315] +[2026-09-01 11:05:05,017] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 380 +[2026-09-01 11:05:07,305] [INFO] [axolotl.loaders.tokenizer] No Chat template selected. Consider adding a chat template for easier inference. +[2026-09-01 11:05:10,415] [INFO] [axolotl.monkeypatch.attention.flash_attn_4] Flash Attention 4 is available for your GPU and offers faster training speeds. To enable: pip install flash-attn-4 +[2026-09-01 11:05:10,416] [INFO] [axolotl.loaders.patch_manager] Applying multipack dataloader patch for sample packing... +[2026-09-01 11:05:11,708] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1065 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/final_v1/gemma3_12b_50m_4ep/control/aft/agreement/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_12b_50m_4ep/control/aft/agreement/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_12b_50m_4ep/control/training/aft/agreement/train.log b/gemma3_12b_50m_4ep/control/training/aft/agreement/train.log new file mode 100644 index 0000000000000000000000000000000000000000..0b6b954aa64807081a93510a7fd83347c306a84d --- /dev/null +++ b/gemma3_12b_50m_4ep/control/training/aft/agreement/train.log @@ -0,0 +1,889 @@ +/workspace/scimt/src/scimt/train/__init__.py:629: UserWarning: model 'gemma3_12b' is not in the model registry — capability checks skipped; add src/scimt/models/.yaml to gate it + return await _run_backend( +[2026-09-01 19:48:50,106] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 19:48:51.604000 138476 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:48:51.622000 138476 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-01 19:49:00,874] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 19:49:04.350000 139506 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:49:04.381000 139506 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-01 19:49:07,757] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-01 19:49:07,760] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-01 19:49:07,760] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 19:49:07,795] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 19:49:07,872] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-01 19:49:07,872] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-01 19:49:07,966] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_12b_50m_4ep/control/aft/agreement/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_12b_50m_4ep/control/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_50m_4ep/control/aft/agreement/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_12b_50m_4ep/control/data/aft/releases/dispatch-final-v1/aft/aft_agreement.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_12b_50m_4ep/control/aft/agreement/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-01 19:49:10,033] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_12b_50m_4ep/control/aft/agreement/prepared/b9cd08a58382776de090e04d0f76717a +[2026-09-01 19:49:10,033] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-01 19:49:10,033] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 8192 examples [00:00, 88359.27 examples/s] +[2026-09-01 19:49:10,429] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_12b_50m_4ep/control/data/aft/releases/dispatch-final-v1/aft/aft_agreement.jsonl with base_type: chat_template and prompt_style: None +[2026-09-01 19:49:10,433] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▏ | 1000/8192 [00:00<00:01, 3810.44 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 17200.91 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:50:03.566000 140369 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:50:03.946000 140367 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:50:03.966000 140367 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:50:05.035000 140375 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:50:05.055000 140375 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:50:05.899000 140368 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:50:05.910000 140363 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:50:05.928000 140368 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:50:05.931000 140363 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:50:05.933000 140371 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:50:05.955000 140371 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:50:06.126000 140366 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:50:06.155000 140366 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:50:06.190000 140364 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:50:06.219000 140364 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:50:06.500000 140365 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:50:06.520000 140365 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:09<01:09, 102.79 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:09<01:09, 102.79 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:09<00:59, 102.79 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:09<00:49, 102.79 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:09<00:39, 102.79 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:09<00:29, 102.79 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:09<00:19, 102.79 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:10<00:09, 102.79 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:10<00:00, 102.79 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:11<00:00, 736.94 examples/s] +[2026-09-01 19:50:09,418] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-01 19:50:17,005] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1066 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "num_epochs": 2, + "learning_rate": 0.0001, + "trust_remote_code": false, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_50m_4ep/control/aft/agreement/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "max_steps": 512, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_12b_50m_4ep/control/aft/agreement/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_12b_50m_4ep/control/aft/agreement/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_12b_50m_4ep/control/data/aft/releases/dispatch-final-v1/aft/aft_agreement.jsonl", + "exists": true, + "size_bytes": 21570873, + "sha256": "1a4cf50221c07bca863a4b3d7a97e7d4ed51fb4bafc8935d0aa242c71fdc11c1", + "nonempty_rows": 8192, + "ordered_example_sha256": "2449d9d1aa169ec51de66eab6c0ca147a6786795bda6b043eb58d4535bfbff3e", + "example_manifest": "/workspace/final_v1/gemma3_12b_50m_4ep/control/aft/agreement/training_examples.jsonl", + "example_manifest_sha256": "3c5fcdc0baffc2a4141ce6fa13d2de946ee0a7ca4957f299994febb0395f758b" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-01T21:05:23+00:00", + "resolved_config_sha256": "ab016379378d09ef3ae59660a98529cc55ff5cd4bc953d44c5cd49faafde03f3", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 16, + "num_input_tokens_seen": 0, + "total_flos": 1.2099146080810476e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_12b_50m_4ep/control/aft/agreement/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_12b_50m_4ep/control/aft/agreement/trainer_state.final.json", + "trainer_state_sha256": "2a2055a89508bb23ae3f7628b286c2a762e14ae1f527eaa98b063dfcef4133ee", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_12b_50m_4ep/control/aft/agreement/training_trace.jsonl", + "trace_sha256": "f284d7b8b60c12f5a8338c6c64a615c60e2966b4b0a16f13a8a4f33da5f06788", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_12b_50m_4ep/control/training/aft/agreement/training_trace.jsonl b/gemma3_12b_50m_4ep/control/training/aft/agreement/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..7005f7710d313bba6ec302da82ad5bc68f9734c2 --- /dev/null +++ b/gemma3_12b_50m_4ep/control/training/aft/agreement/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 3.071087598800659, "learning_rate": 0.0, "loss": 0.28659969568252563, "memory/device_reserved (GiB)": 38.46, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.33189, "step": 1, "tokens/total": 34400, "tokens/train_per_sec_per_gpu": 24.3, "tokens/trainable": 470} +{"epoch": 0.0078125, "grad_norm": 3.324949026107788, "learning_rate": 4.000000000000001e-06, "loss": 0.27451080083847046, "memory/device_reserved (GiB)": 38.72, "memory/max_active (GiB)": 35.83, "memory/max_allocated (GiB)": 35.83, "ppl": 1.31589, "step": 2, "tokens/total": 64064, "tokens/train_per_sec_per_gpu": 31.88, "tokens/trainable": 922} +{"epoch": 0.01171875, "grad_norm": 3.0682427883148193, "learning_rate": 8.000000000000001e-06, "loss": 0.2576635777950287, "memory/device_reserved (GiB)": 38.72, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.2939, "step": 3, "tokens/total": 97152, "tokens/train_per_sec_per_gpu": 28.12, "tokens/trainable": 1396} +{"epoch": 0.015625, "grad_norm": 2.8276708126068115, "learning_rate": 1.2e-05, "loss": 0.26469510793685913, "memory/device_reserved (GiB)": 40.85, "memory/max_active (GiB)": 37.24, "memory/max_allocated (GiB)": 37.24, "ppl": 1.30303, "step": 4, "tokens/total": 130304, "tokens/train_per_sec_per_gpu": 29.64, "tokens/trainable": 1850} +{"epoch": 0.01953125, "grad_norm": 2.441620349884033, "learning_rate": 1.6000000000000003e-05, "loss": 0.24508658051490784, "memory/device_reserved (GiB)": 41.8, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.27773, "step": 5, "tokens/total": 169504, "tokens/train_per_sec_per_gpu": 23.66, "tokens/trainable": 2302} +{"epoch": 0.0234375, "grad_norm": 2.3561811447143555, "learning_rate": 2e-05, "loss": 0.2132808268070221, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 35.92, "memory/max_allocated (GiB)": 35.92, "ppl": 1.23773, "step": 6, "tokens/total": 200016, "tokens/train_per_sec_per_gpu": 28.37, "tokens/trainable": 2742} +{"epoch": 0.02734375, "grad_norm": 1.6732096672058105, "learning_rate": 2.4e-05, "loss": 0.1860685795545578, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.2045, "step": 7, "tokens/total": 234240, "tokens/train_per_sec_per_gpu": 28.51, "tokens/trainable": 3208} +{"epoch": 0.03125, "grad_norm": 0.9295008778572083, "learning_rate": 2.8000000000000003e-05, "loss": 0.14695699512958527, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 36.7, "memory/max_allocated (GiB)": 36.7, "ppl": 1.1583, "step": 8, "tokens/total": 265472, "tokens/train_per_sec_per_gpu": 27.83, "tokens/trainable": 3655} +{"epoch": 0.03515625, "grad_norm": 2.116955280303955, "learning_rate": 3.2000000000000005e-05, "loss": 0.11944551765918732, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 35.72, "memory/max_allocated (GiB)": 35.72, "ppl": 1.12687, "step": 9, "tokens/total": 295952, "tokens/train_per_sec_per_gpu": 29.53, "tokens/trainable": 4091} +{"epoch": 0.0390625, "grad_norm": 2.548321485519409, "learning_rate": 3.6e-05, "loss": 0.14092805981636047, "memory/device_reserved (GiB)": 39.42, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.15134, "step": 10, "tokens/total": 329872, "tokens/train_per_sec_per_gpu": 27.77, "tokens/trainable": 4553} +{"epoch": 0.04296875, "grad_norm": 1.8565871715545654, "learning_rate": 4e-05, "loss": 0.09923017024993896, "memory/device_reserved (GiB)": 39.42, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.10432, "step": 11, "tokens/total": 366176, "tokens/train_per_sec_per_gpu": 24.34, "tokens/trainable": 4992} +{"epoch": 0.046875, "grad_norm": 2.026968479156494, "learning_rate": 4.4000000000000006e-05, "loss": 0.11233267188072205, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.11889, "step": 12, "tokens/total": 403760, "tokens/train_per_sec_per_gpu": 27.11, "tokens/trainable": 5494} +{"epoch": 0.05078125, "grad_norm": 3.0524497032165527, "learning_rate": 4.8e-05, "loss": 0.07556107640266418, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 36.82, "memory/max_allocated (GiB)": 36.82, "ppl": 1.07849, "step": 13, "tokens/total": 435584, "tokens/train_per_sec_per_gpu": 29.26, "tokens/trainable": 5933} +{"epoch": 0.0546875, "grad_norm": 2.740525245666504, "learning_rate": 5.2000000000000004e-05, "loss": 0.0903567522764206, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.09456, "step": 14, "tokens/total": 468896, "tokens/train_per_sec_per_gpu": 24.18, "tokens/trainable": 6369} +{"epoch": 0.05859375, "grad_norm": 3.0557291507720947, "learning_rate": 5.6000000000000006e-05, "loss": 0.10818623751401901, "memory/device_reserved (GiB)": 40.81, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.11426, "step": 15, "tokens/total": 504240, "tokens/train_per_sec_per_gpu": 24.71, "tokens/trainable": 6820} +{"epoch": 0.0625, "grad_norm": 2.9998228549957275, "learning_rate": 6e-05, "loss": 0.07788607478141785, "memory/device_reserved (GiB)": 40.81, "memory/max_active (GiB)": 36.56, "memory/max_allocated (GiB)": 36.56, "ppl": 1.081, "step": 16, "tokens/total": 536816, "tokens/train_per_sec_per_gpu": 25.23, "tokens/trainable": 7258} +{"epoch": 0.06640625, "grad_norm": 1.7316851615905762, "learning_rate": 6.400000000000001e-05, "loss": 0.05717172473669052, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.05884, "step": 17, "tokens/total": 572672, "tokens/train_per_sec_per_gpu": 26.33, "tokens/trainable": 7710} +{"epoch": 0.0703125, "grad_norm": 1.3787873983383179, "learning_rate": 6.800000000000001e-05, "loss": 0.05290931090712547, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.05433, "step": 18, "tokens/total": 609008, "tokens/train_per_sec_per_gpu": 23.04, "tokens/trainable": 8174} +{"epoch": 0.07421875, "grad_norm": 1.1043643951416016, "learning_rate": 7.2e-05, "loss": 0.047032639384269714, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.32, "memory/max_allocated (GiB)": 38.32, "ppl": 1.04816, "step": 19, "tokens/total": 645376, "tokens/train_per_sec_per_gpu": 24.96, "tokens/trainable": 8617} +{"epoch": 0.078125, "grad_norm": 2.418144702911377, "learning_rate": 7.6e-05, "loss": 0.0533624067902565, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.05481, "step": 20, "tokens/total": 679680, "tokens/train_per_sec_per_gpu": 23.64, "tokens/trainable": 9037} +{"epoch": 0.08203125, "grad_norm": 0.8316324353218079, "learning_rate": 8e-05, "loss": 0.028733914718031883, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.3, "memory/max_allocated (GiB)": 36.3, "ppl": 1.02915, "step": 21, "tokens/total": 711056, "tokens/train_per_sec_per_gpu": 31.45, "tokens/trainable": 9503} +{"epoch": 0.0859375, "grad_norm": 1.377312421798706, "learning_rate": 8.4e-05, "loss": 0.023459959775209427, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 38.8, "memory/max_allocated (GiB)": 38.8, "ppl": 1.02374, "step": 22, "tokens/total": 746048, "tokens/train_per_sec_per_gpu": 28.04, "tokens/trainable": 9972} +{"epoch": 0.08984375, "grad_norm": 1.7363848686218262, "learning_rate": 8.800000000000001e-05, "loss": 0.03554600104689598, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 1.03619, "step": 23, "tokens/total": 780240, "tokens/train_per_sec_per_gpu": 29.56, "tokens/trainable": 10447} +{"epoch": 0.09375, "grad_norm": 3.6634864807128906, "learning_rate": 9.200000000000001e-05, "loss": 0.05269277095794678, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.5, "memory/max_allocated (GiB)": 37.5, "ppl": 1.05411, "step": 24, "tokens/total": 811344, "tokens/train_per_sec_per_gpu": 32.16, "tokens/trainable": 10899} +{"epoch": 0.09765625, "grad_norm": 3.8617851734161377, "learning_rate": 9.6e-05, "loss": 0.10604137182235718, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.66, "memory/max_allocated (GiB)": 36.66, "ppl": 1.11187, "step": 25, "tokens/total": 843728, "tokens/train_per_sec_per_gpu": 27.42, "tokens/trainable": 11360} +{"epoch": 0.1015625, "grad_norm": 11.406000137329102, "learning_rate": 0.0001, "loss": 0.05575975403189659, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.05734, "step": 26, "tokens/total": 876560, "tokens/train_per_sec_per_gpu": 24.12, "tokens/trainable": 11775} +{"epoch": 0.10546875, "grad_norm": 1.2936272621154785, "learning_rate": 9.99990636831587e-05, "loss": 0.03076578676700592, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.03124, "step": 27, "tokens/total": 909488, "tokens/train_per_sec_per_gpu": 30.21, "tokens/trainable": 12242} +{"epoch": 0.109375, "grad_norm": 3.896512269973755, "learning_rate": 9.999625477159879e-05, "loss": 0.08069384843111038, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.08404, "step": 28, "tokens/total": 947840, "tokens/train_per_sec_per_gpu": 22.75, "tokens/trainable": 12696} +{"epoch": 0.11328125, "grad_norm": 2.382826089859009, "learning_rate": 9.999157338221051e-05, "loss": 0.0593971312046051, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.0612, "step": 29, "tokens/total": 982544, "tokens/train_per_sec_per_gpu": 26.3, "tokens/trainable": 13153} +{"epoch": 0.1171875, "grad_norm": 20.031341552734375, "learning_rate": 9.998501970980562e-05, "loss": 0.030471516773104668, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.03094, "step": 30, "tokens/total": 1014144, "tokens/train_per_sec_per_gpu": 31.51, "tokens/trainable": 13624} +{"epoch": 0.12109375, "grad_norm": 0.6297637224197388, "learning_rate": 9.997659402710915e-05, "loss": 0.01830429397523403, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.01847, "step": 31, "tokens/total": 1048976, "tokens/train_per_sec_per_gpu": 26.19, "tokens/trainable": 14064} +{"epoch": 0.125, "grad_norm": 0.8511174917221069, "learning_rate": 9.996629668474818e-05, "loss": 0.04201745614409447, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.04291, "step": 32, "tokens/total": 1083728, "tokens/train_per_sec_per_gpu": 29.73, "tokens/trainable": 14565} +{"epoch": 0.12890625, "grad_norm": 1.0269112586975098, "learning_rate": 9.995412811123711e-05, "loss": 0.032923877239227295, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 38.97, "memory/max_allocated (GiB)": 38.97, "ppl": 1.03347, "step": 33, "tokens/total": 1119664, "tokens/train_per_sec_per_gpu": 24.91, "tokens/trainable": 15005} +{"epoch": 0.1328125, "grad_norm": 1.3864994049072266, "learning_rate": 9.994008881295999e-05, "loss": 0.03086205944418907, "memory/device_reserved (GiB)": 39.91, "memory/max_active (GiB)": 36.87, "memory/max_allocated (GiB)": 36.87, "ppl": 1.03134, "step": 34, "tokens/total": 1152768, "tokens/train_per_sec_per_gpu": 25.71, "tokens/trainable": 15459} +{"epoch": 0.13671875, "grad_norm": 1.422756552696228, "learning_rate": 9.992417937414932e-05, "loss": 0.026882192119956017, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 1.02725, "step": 35, "tokens/total": 1186736, "tokens/train_per_sec_per_gpu": 29.98, "tokens/trainable": 15960} +{"epoch": 0.140625, "grad_norm": 2.0806896686553955, "learning_rate": 9.99064004568618e-05, "loss": 0.02028246596455574, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.02049, "step": 36, "tokens/total": 1222352, "tokens/train_per_sec_per_gpu": 25.8, "tokens/trainable": 16428} +{"epoch": 0.14453125, "grad_norm": 1.6240054368972778, "learning_rate": 9.988675280095074e-05, "loss": 0.02839859388768673, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.02881, "step": 37, "tokens/total": 1256336, "tokens/train_per_sec_per_gpu": 25.07, "tokens/trainable": 16884} +{"epoch": 0.1484375, "grad_norm": 1.971199631690979, "learning_rate": 9.986523722403528e-05, "loss": 0.03467846289277077, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.03529, "step": 38, "tokens/total": 1288304, "tokens/train_per_sec_per_gpu": 26.82, "tokens/trainable": 17341} +{"epoch": 0.15234375, "grad_norm": 1.5512691736221313, "learning_rate": 9.984185462146642e-05, "loss": 0.045909907668828964, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.41, "memory/max_allocated (GiB)": 37.41, "ppl": 1.04698, "step": 39, "tokens/total": 1323440, "tokens/train_per_sec_per_gpu": 25.77, "tokens/trainable": 17786} +{"epoch": 0.15625, "grad_norm": 3.024327278137207, "learning_rate": 9.98166059662897e-05, "loss": 0.036665529012680054, "memory/device_reserved (GiB)": 40.63, "memory/max_active (GiB)": 39.09, "memory/max_allocated (GiB)": 39.09, "ppl": 1.03735, "step": 40, "tokens/total": 1358912, "tokens/train_per_sec_per_gpu": 26.09, "tokens/trainable": 18262} +{"epoch": 0.16015625, "grad_norm": 2.3652184009552, "learning_rate": 9.978949230920472e-05, "loss": 0.024162761867046356, "memory/device_reserved (GiB)": 40.63, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.02446, "step": 41, "tokens/total": 1393600, "tokens/train_per_sec_per_gpu": 25.27, "tokens/trainable": 18716} +{"epoch": 0.1640625, "grad_norm": 0.7853467464447021, "learning_rate": 9.976051477852141e-05, "loss": 0.03373076021671295, "memory/device_reserved (GiB)": 40.63, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.03431, "step": 42, "tokens/total": 1426832, "tokens/train_per_sec_per_gpu": 26.54, "tokens/trainable": 19157} +{"epoch": 0.16796875, "grad_norm": 1.2358049154281616, "learning_rate": 9.972967458011312e-05, "loss": 0.03697120025753975, "memory/device_reserved (GiB)": 40.63, "memory/max_active (GiB)": 38.91, "memory/max_allocated (GiB)": 38.91, "ppl": 1.03766, "step": 43, "tokens/total": 1462192, "tokens/train_per_sec_per_gpu": 27.61, "tokens/trainable": 19647} +{"epoch": 0.171875, "grad_norm": 0.932833731174469, "learning_rate": 9.96969729973664e-05, "loss": 0.03448842093348503, "memory/device_reserved (GiB)": 41.66, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.03509, "step": 44, "tokens/total": 1501056, "tokens/train_per_sec_per_gpu": 23.69, "tokens/trainable": 20119} +{"epoch": 0.17578125, "grad_norm": 1.7085423469543457, "learning_rate": 9.966241139112754e-05, "loss": 0.02893667109310627, "memory/device_reserved (GiB)": 41.66, "memory/max_active (GiB)": 38.1, "memory/max_allocated (GiB)": 38.1, "ppl": 1.02936, "step": 45, "tokens/total": 1534752, "tokens/train_per_sec_per_gpu": 25.92, "tokens/trainable": 20560} +{"epoch": 0.1796875, "grad_norm": 0.7096034288406372, "learning_rate": 9.96259911996461e-05, "loss": 0.02521599270403385, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.02554, "step": 46, "tokens/total": 1568912, "tokens/train_per_sec_per_gpu": 25.41, "tokens/trainable": 20992} +{"epoch": 0.18359375, "grad_norm": 1.0665210485458374, "learning_rate": 9.958771393851491e-05, "loss": 0.036696773022413254, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 36.49, "memory/max_allocated (GiB)": 36.49, "ppl": 1.03738, "step": 47, "tokens/total": 1600368, "tokens/train_per_sec_per_gpu": 25.07, "tokens/trainable": 21441} +{"epoch": 0.1875, "grad_norm": 11.601887702941895, "learning_rate": 9.954758120060702e-05, "loss": 0.03487066924571991, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.03549, "step": 48, "tokens/total": 1636640, "tokens/train_per_sec_per_gpu": 24.6, "tokens/trainable": 21901} +{"epoch": 0.19140625, "grad_norm": 0.6565883755683899, "learning_rate": 9.950559465600948e-05, "loss": 0.018284108489751816, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 1.01845, "step": 49, "tokens/total": 1670896, "tokens/train_per_sec_per_gpu": 26.28, "tokens/trainable": 22341} +{"epoch": 0.1953125, "grad_norm": 0.7141174674034119, "learning_rate": 9.946175605195379e-05, "loss": 0.01874258928000927, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.01892, "step": 50, "tokens/total": 1710448, "tokens/train_per_sec_per_gpu": 21.91, "tokens/trainable": 22833} +{"epoch": 0.19921875, "grad_norm": 0.5555512309074402, "learning_rate": 9.941606721274322e-05, "loss": 0.009199233725667, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.00924, "step": 51, "tokens/total": 1744656, "tokens/train_per_sec_per_gpu": 23.96, "tokens/trainable": 23277} +{"epoch": 0.203125, "grad_norm": 1.4364328384399414, "learning_rate": 9.936853003967685e-05, "loss": 0.025491446256637573, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.02582, "step": 52, "tokens/total": 1781888, "tokens/train_per_sec_per_gpu": 25.44, "tokens/trainable": 23759} +{"epoch": 0.20703125, "grad_norm": 2.967066764831543, "learning_rate": 9.93191465109705e-05, "loss": 0.04999610781669617, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.05127, "step": 53, "tokens/total": 1818528, "tokens/train_per_sec_per_gpu": 23.9, "tokens/trainable": 24193} +{"epoch": 0.2109375, "grad_norm": 1.266978144645691, "learning_rate": 9.926791868167438e-05, "loss": 0.0293610580265522, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.0298, "step": 54, "tokens/total": 1855024, "tokens/train_per_sec_per_gpu": 24.12, "tokens/trainable": 24619} +{"epoch": 0.21484375, "grad_norm": 1.62916898727417, "learning_rate": 9.921484868358753e-05, "loss": 0.052482351660728455, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 1.05388, "step": 55, "tokens/total": 1891040, "tokens/train_per_sec_per_gpu": 24.4, "tokens/trainable": 25075} +{"epoch": 0.21875, "grad_norm": 1.0829652547836304, "learning_rate": 9.915993872516924e-05, "loss": 0.014141310006380081, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.01424, "step": 56, "tokens/total": 1926864, "tokens/train_per_sec_per_gpu": 23.16, "tokens/trainable": 25519} +{"epoch": 0.22265625, "grad_norm": 0.5555707216262817, "learning_rate": 9.9103191091447e-05, "loss": 0.009492013603448868, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00954, "step": 57, "tokens/total": 1958928, "tokens/train_per_sec_per_gpu": 27.38, "tokens/trainable": 25966} +{"epoch": 0.2265625, "grad_norm": 1.3241068124771118, "learning_rate": 9.904460814392147e-05, "loss": 0.017634768038988113, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.01779, "step": 58, "tokens/total": 1998304, "tokens/train_per_sec_per_gpu": 22.6, "tokens/trainable": 26423} +{"epoch": 0.23046875, "grad_norm": 0.6693233847618103, "learning_rate": 9.898419232046825e-05, "loss": 0.022162113338708878, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.02241, "step": 59, "tokens/total": 2035008, "tokens/train_per_sec_per_gpu": 28.01, "tokens/trainable": 26934} +{"epoch": 0.234375, "grad_norm": 0.7516471743583679, "learning_rate": 9.892194613523633e-05, "loss": 0.01611669734120369, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.26, "memory/max_allocated (GiB)": 38.26, "ppl": 1.01625, "step": 60, "tokens/total": 2070864, "tokens/train_per_sec_per_gpu": 24.27, "tokens/trainable": 27393} +{"epoch": 0.23828125, "grad_norm": 1.053234577178955, "learning_rate": 9.885787217854357e-05, "loss": 0.013242291286587715, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.01333, "step": 61, "tokens/total": 2104448, "tokens/train_per_sec_per_gpu": 26.44, "tokens/trainable": 27852} +{"epoch": 0.2421875, "grad_norm": 0.904224157333374, "learning_rate": 9.879197311676887e-05, "loss": 0.0231782253831625, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.02345, "step": 62, "tokens/total": 2140496, "tokens/train_per_sec_per_gpu": 24.23, "tokens/trainable": 28292} +{"epoch": 0.24609375, "grad_norm": 2.11441969871521, "learning_rate": 9.872425169224113e-05, "loss": 0.036401115357875824, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.03707, "step": 63, "tokens/total": 2174288, "tokens/train_per_sec_per_gpu": 29.79, "tokens/trainable": 28798} +{"epoch": 0.25, "grad_norm": 0.9957871437072754, "learning_rate": 9.865471072312528e-05, "loss": 0.02896103262901306, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.14, "memory/max_allocated (GiB)": 38.14, "ppl": 1.02938, "step": 64, "tokens/total": 2210752, "tokens/train_per_sec_per_gpu": 25.93, "tokens/trainable": 29283} +{"epoch": 0.25390625, "grad_norm": 0.49806907773017883, "learning_rate": 9.858335310330492e-05, "loss": 0.016387756913900375, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.01652, "step": 65, "tokens/total": 2242928, "tokens/train_per_sec_per_gpu": 26.46, "tokens/trainable": 29745} +{"epoch": 0.2578125, "grad_norm": 0.5002174377441406, "learning_rate": 9.851018180226185e-05, "loss": 0.011719746515154839, "memory/device_reserved (GiB)": 40.31, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.01179, "step": 66, "tokens/total": 2281472, "tokens/train_per_sec_per_gpu": 20.32, "tokens/trainable": 30186} +{"epoch": 0.26171875, "grad_norm": 0.7148451805114746, "learning_rate": 9.843519986495259e-05, "loss": 0.018183480948209763, "memory/device_reserved (GiB)": 41.08, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.01835, "step": 67, "tokens/total": 2316208, "tokens/train_per_sec_per_gpu": 25.49, "tokens/trainable": 30622} +{"epoch": 0.265625, "grad_norm": 0.6191102862358093, "learning_rate": 9.835841041168162e-05, "loss": 0.01674959622323513, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.01689, "step": 68, "tokens/total": 2353344, "tokens/train_per_sec_per_gpu": 25.49, "tokens/trainable": 31097} +{"epoch": 0.26953125, "grad_norm": 0.7635965347290039, "learning_rate": 9.82798166379715e-05, "loss": 0.015741772949695587, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.01587, "step": 69, "tokens/total": 2386704, "tokens/train_per_sec_per_gpu": 29.51, "tokens/trainable": 31595} +{"epoch": 0.2734375, "grad_norm": 1.1365058422088623, "learning_rate": 9.819942181443002e-05, "loss": 0.008055657148361206, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.00809, "step": 70, "tokens/total": 2423984, "tokens/train_per_sec_per_gpu": 21.82, "tokens/trainable": 32036} +{"epoch": 0.27734375, "grad_norm": 1.3842711448669434, "learning_rate": 9.811722928661392e-05, "loss": 0.03045791946351528, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 35.81, "memory/max_allocated (GiB)": 35.81, "ppl": 1.03093, "step": 71, "tokens/total": 2452912, "tokens/train_per_sec_per_gpu": 26.54, "tokens/trainable": 32428} +{"epoch": 0.28125, "grad_norm": 0.25220584869384766, "learning_rate": 9.803324247488975e-05, "loss": 0.004791585262864828, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.0048, "step": 72, "tokens/total": 2488576, "tokens/train_per_sec_per_gpu": 24.05, "tokens/trainable": 32880} +{"epoch": 0.28515625, "grad_norm": 0.4696706533432007, "learning_rate": 9.794746487429161e-05, "loss": 0.0022144680842757225, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 36.88, "memory/max_allocated (GiB)": 36.88, "ppl": 1.00222, "step": 73, "tokens/total": 2521520, "tokens/train_per_sec_per_gpu": 27.17, "tokens/trainable": 33323} +{"epoch": 0.2890625, "grad_norm": 0.6259180307388306, "learning_rate": 9.785990005437554e-05, "loss": 0.012555059976875782, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.01263, "step": 74, "tokens/total": 2557392, "tokens/train_per_sec_per_gpu": 26.25, "tokens/trainable": 33792} +{"epoch": 0.29296875, "grad_norm": 1.7724124193191528, "learning_rate": 9.777055165907117e-05, "loss": 0.03701884672045708, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.03771, "step": 75, "tokens/total": 2589920, "tokens/train_per_sec_per_gpu": 29.05, "tokens/trainable": 34223} +{"epoch": 0.296875, "grad_norm": 0.5458208918571472, "learning_rate": 9.767942340652993e-05, "loss": 0.010973942466080189, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.07, "memory/max_allocated (GiB)": 38.07, "ppl": 1.01103, "step": 76, "tokens/total": 2624784, "tokens/train_per_sec_per_gpu": 22.54, "tokens/trainable": 34649} +{"epoch": 0.30078125, "grad_norm": 1.0410820245742798, "learning_rate": 9.758651908897035e-05, "loss": 0.027969982475042343, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.02836, "step": 77, "tokens/total": 2658176, "tokens/train_per_sec_per_gpu": 28.19, "tokens/trainable": 35094} +{"epoch": 0.3046875, "grad_norm": 1.1250604391098022, "learning_rate": 9.749184257252033e-05, "loss": 0.03465390205383301, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.03526, "step": 78, "tokens/total": 2692448, "tokens/train_per_sec_per_gpu": 24.73, "tokens/trainable": 35534} +{"epoch": 0.30859375, "grad_norm": 0.5005885362625122, "learning_rate": 9.739539779705614e-05, "loss": 0.024885831400752068, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.0252, "step": 79, "tokens/total": 2730720, "tokens/train_per_sec_per_gpu": 24.7, "tokens/trainable": 36029} +{"epoch": 0.3125, "grad_norm": 0.2822096347808838, "learning_rate": 9.729718877603861e-05, "loss": 0.012448864057660103, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.01253, "step": 80, "tokens/total": 2764512, "tokens/train_per_sec_per_gpu": 27.96, "tokens/trainable": 36469} +{"epoch": 0.31640625, "grad_norm": 0.4925324022769928, "learning_rate": 9.719721959634592e-05, "loss": 0.026164112612605095, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.02651, "step": 81, "tokens/total": 2799728, "tokens/train_per_sec_per_gpu": 23.42, "tokens/trainable": 36906} +{"epoch": 0.3203125, "grad_norm": 0.28185877203941345, "learning_rate": 9.709549441810375e-05, "loss": 0.01140504702925682, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.55, "memory/max_allocated (GiB)": 39.55, "ppl": 1.01147, "step": 82, "tokens/total": 2837344, "tokens/train_per_sec_per_gpu": 27.17, "tokens/trainable": 37390} +{"epoch": 0.32421875, "grad_norm": 0.33500856161117554, "learning_rate": 9.699201747451195e-05, "loss": 0.00938574317842722, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.00943, "step": 83, "tokens/total": 2869936, "tokens/train_per_sec_per_gpu": 25.23, "tokens/trainable": 37838} +{"epoch": 0.328125, "grad_norm": 0.162337526679039, "learning_rate": 9.688679307166854e-05, "loss": 0.005397366359829903, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00541, "step": 84, "tokens/total": 2902736, "tokens/train_per_sec_per_gpu": 28.55, "tokens/trainable": 38310} +{"epoch": 0.33203125, "grad_norm": 0.5026575922966003, "learning_rate": 9.677982558839042e-05, "loss": 0.016045942902565002, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.01618, "step": 85, "tokens/total": 2936032, "tokens/train_per_sec_per_gpu": 27.34, "tokens/trainable": 38789} +{"epoch": 0.3359375, "grad_norm": 2.731879711151123, "learning_rate": 9.66711194760312e-05, "loss": 0.013306775130331516, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.0134, "step": 86, "tokens/total": 2973440, "tokens/train_per_sec_per_gpu": 25.78, "tokens/trainable": 39277} +{"epoch": 0.33984375, "grad_norm": 0.7790959477424622, "learning_rate": 9.656067925829593e-05, "loss": 0.01066723931580782, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.01072, "step": 87, "tokens/total": 3004192, "tokens/train_per_sec_per_gpu": 31.53, "tokens/trainable": 39737} +{"epoch": 0.34375, "grad_norm": 0.6822731494903564, "learning_rate": 9.644850953105288e-05, "loss": 0.022903043776750565, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 35.78, "memory/max_allocated (GiB)": 35.78, "ppl": 1.02317, "step": 88, "tokens/total": 3034544, "tokens/train_per_sec_per_gpu": 26.41, "tokens/trainable": 40179} +{"epoch": 0.34765625, "grad_norm": 1.274268627166748, "learning_rate": 9.633461496214225e-05, "loss": 0.041815385222435, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.0427, "step": 89, "tokens/total": 3067248, "tokens/train_per_sec_per_gpu": 26.75, "tokens/trainable": 40649} +{"epoch": 0.3515625, "grad_norm": 0.5821620225906372, "learning_rate": 9.621900029118195e-05, "loss": 0.032726556062698364, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.04, "memory/max_allocated (GiB)": 36.04, "ppl": 1.03327, "step": 90, "tokens/total": 3096928, "tokens/train_per_sec_per_gpu": 26.91, "tokens/trainable": 41080} +{"epoch": 0.35546875, "grad_norm": 0.5808966159820557, "learning_rate": 9.610167032937036e-05, "loss": 0.01265052892267704, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.01273, "step": 91, "tokens/total": 3135216, "tokens/train_per_sec_per_gpu": 26.36, "tokens/trainable": 41599} +{"epoch": 0.359375, "grad_norm": 0.5869033932685852, "learning_rate": 9.598262995928611e-05, "loss": 0.014537954702973366, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.01464, "step": 92, "tokens/total": 3166416, "tokens/train_per_sec_per_gpu": 31.5, "tokens/trainable": 42076} +{"epoch": 0.36328125, "grad_norm": 0.17148450016975403, "learning_rate": 9.586188413468492e-05, "loss": 0.006187049206346273, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00621, "step": 93, "tokens/total": 3204608, "tokens/train_per_sec_per_gpu": 23.13, "tokens/trainable": 42522} +{"epoch": 0.3671875, "grad_norm": 0.518880307674408, "learning_rate": 9.57394378802934e-05, "loss": 0.021556202322244644, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.02179, "step": 94, "tokens/total": 3237920, "tokens/train_per_sec_per_gpu": 27.73, "tokens/trainable": 42974} +{"epoch": 0.37109375, "grad_norm": 0.41193684935569763, "learning_rate": 9.56152962916e-05, "loss": 0.019451187923550606, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.26, "memory/max_allocated (GiB)": 36.26, "ppl": 1.01964, "step": 95, "tokens/total": 3268592, "tokens/train_per_sec_per_gpu": 26.77, "tokens/trainable": 43380} +{"epoch": 0.375, "grad_norm": 0.3206879198551178, "learning_rate": 9.548946453464296e-05, "loss": 0.015036707744002342, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.01515, "step": 96, "tokens/total": 3302688, "tokens/train_per_sec_per_gpu": 27.13, "tokens/trainable": 43865} +{"epoch": 0.37890625, "grad_norm": 0.35784369707107544, "learning_rate": 9.53619478457953e-05, "loss": 0.015444321557879448, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.01556, "step": 97, "tokens/total": 3337584, "tokens/train_per_sec_per_gpu": 24.25, "tokens/trainable": 44328} +{"epoch": 0.3828125, "grad_norm": 0.24448929727077484, "learning_rate": 9.523275153154695e-05, "loss": 0.008413689211010933, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.00845, "step": 98, "tokens/total": 3369984, "tokens/train_per_sec_per_gpu": 25.45, "tokens/trainable": 44778} +{"epoch": 0.38671875, "grad_norm": 0.6536917090415955, "learning_rate": 9.51018809682839e-05, "loss": 0.03224601224064827, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.03277, "step": 99, "tokens/total": 3407648, "tokens/train_per_sec_per_gpu": 25.89, "tokens/trainable": 45225} +{"epoch": 0.390625, "grad_norm": 0.6902362108230591, "learning_rate": 9.49693416020645e-05, "loss": 0.025648972019553185, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.53, "memory/max_allocated (GiB)": 38.53, "ppl": 1.02598, "step": 100, "tokens/total": 3445392, "tokens/train_per_sec_per_gpu": 24.78, "tokens/trainable": 45678} +{"epoch": 0.39453125, "grad_norm": 0.30549344420433044, "learning_rate": 9.483513894839276e-05, "loss": 0.004926628433167934, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.00494, "step": 101, "tokens/total": 3479440, "tokens/train_per_sec_per_gpu": 25.55, "tokens/trainable": 46125} +{"epoch": 0.3984375, "grad_norm": 0.4436953067779541, "learning_rate": 9.469927859198888e-05, "loss": 0.016055481508374214, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.01619, "step": 102, "tokens/total": 3515360, "tokens/train_per_sec_per_gpu": 29.37, "tokens/trainable": 46612} +{"epoch": 0.40234375, "grad_norm": 0.9213862419128418, "learning_rate": 9.456176618655689e-05, "loss": 0.022187847644090652, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.47, "memory/max_allocated (GiB)": 36.47, "ppl": 1.02244, "step": 103, "tokens/total": 3547408, "tokens/train_per_sec_per_gpu": 27.75, "tokens/trainable": 47059} +{"epoch": 0.40625, "grad_norm": 1.1979200839996338, "learning_rate": 9.442260745454927e-05, "loss": 0.030340498313307762, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.03081, "step": 104, "tokens/total": 3582784, "tokens/train_per_sec_per_gpu": 26.12, "tokens/trainable": 47536} +{"epoch": 0.41015625, "grad_norm": 0.3195672631263733, "learning_rate": 9.428180818692884e-05, "loss": 0.012280529364943504, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.45, "memory/max_allocated (GiB)": 38.45, "ppl": 1.01236, "step": 105, "tokens/total": 3618944, "tokens/train_per_sec_per_gpu": 24.97, "tokens/trainable": 48037} +{"epoch": 0.4140625, "grad_norm": 0.4170463979244232, "learning_rate": 9.413937424292791e-05, "loss": 0.011617821641266346, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.01169, "step": 106, "tokens/total": 3658080, "tokens/train_per_sec_per_gpu": 21.78, "tokens/trainable": 48491} +{"epoch": 0.41796875, "grad_norm": 0.6266940832138062, "learning_rate": 9.399531154980424e-05, "loss": 0.01790524646639824, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 1.01807, "step": 107, "tokens/total": 3694656, "tokens/train_per_sec_per_gpu": 24.32, "tokens/trainable": 48940} +{"epoch": 0.421875, "grad_norm": 0.34746357798576355, "learning_rate": 9.384962610259455e-05, "loss": 0.018872644752264023, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.01905, "step": 108, "tokens/total": 3729312, "tokens/train_per_sec_per_gpu": 24.4, "tokens/trainable": 49389} +{"epoch": 0.42578125, "grad_norm": 0.29686853289604187, "learning_rate": 9.370232396386494e-05, "loss": 0.005750146694481373, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.22, "memory/max_allocated (GiB)": 37.22, "ppl": 1.00577, "step": 109, "tokens/total": 3762400, "tokens/train_per_sec_per_gpu": 26.62, "tokens/trainable": 49838} +{"epoch": 0.4296875, "grad_norm": 0.34095969796180725, "learning_rate": 9.355341126345868e-05, "loss": 0.013489769771695137, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.01358, "step": 110, "tokens/total": 3797680, "tokens/train_per_sec_per_gpu": 25.72, "tokens/trainable": 50310} +{"epoch": 0.43359375, "grad_norm": 0.31588438153266907, "learning_rate": 9.340289419824107e-05, "loss": 0.009787647053599358, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.00984, "step": 111, "tokens/total": 3830000, "tokens/train_per_sec_per_gpu": 28.33, "tokens/trainable": 50755} +{"epoch": 0.4375, "grad_norm": 0.5733108520507812, "learning_rate": 9.325077903184159e-05, "loss": 0.008442171849310398, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00848, "step": 112, "tokens/total": 3864992, "tokens/train_per_sec_per_gpu": 24.09, "tokens/trainable": 51213} +{"epoch": 0.44140625, "grad_norm": 0.053502246737480164, "learning_rate": 9.30970720943932e-05, "loss": 0.0010829571401700377, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.15, "memory/max_allocated (GiB)": 38.15, "ppl": 1.00108, "step": 113, "tokens/total": 3900000, "tokens/train_per_sec_per_gpu": 25.63, "tokens/trainable": 51660} +{"epoch": 0.4453125, "grad_norm": 0.40695562958717346, "learning_rate": 9.2941779782269e-05, "loss": 0.00814485177397728, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00818, "step": 114, "tokens/total": 3933600, "tokens/train_per_sec_per_gpu": 25.7, "tokens/trainable": 52133} +{"epoch": 0.44921875, "grad_norm": 0.47410663962364197, "learning_rate": 9.278490855781596e-05, "loss": 0.008938898332417011, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.00898, "step": 115, "tokens/total": 3966032, "tokens/train_per_sec_per_gpu": 31.19, "tokens/trainable": 52580} +{"epoch": 0.453125, "grad_norm": 0.7649363279342651, "learning_rate": 9.262646494908604e-05, "loss": 0.009413160383701324, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00946, "step": 116, "tokens/total": 4004688, "tokens/train_per_sec_per_gpu": 23.73, "tokens/trainable": 53033} +{"epoch": 0.45703125, "grad_norm": 0.8752518892288208, "learning_rate": 9.246645554956457e-05, "loss": 0.015523935668170452, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.01565, "step": 117, "tokens/total": 4039712, "tokens/train_per_sec_per_gpu": 26.88, "tokens/trainable": 53517} +{"epoch": 0.4609375, "grad_norm": 0.3514716923236847, "learning_rate": 9.230488701789578e-05, "loss": 0.013841704465448856, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.01394, "step": 118, "tokens/total": 4076768, "tokens/train_per_sec_per_gpu": 24.49, "tokens/trainable": 53967} +{"epoch": 0.46484375, "grad_norm": 0.6191282868385315, "learning_rate": 9.214176607760577e-05, "loss": 0.009029550477862358, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.00907, "step": 119, "tokens/total": 4115024, "tokens/train_per_sec_per_gpu": 24.24, "tokens/trainable": 54430} +{"epoch": 0.46875, "grad_norm": 1.2483477592468262, "learning_rate": 9.197709951682268e-05, "loss": 0.026311468333005905, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.02666, "step": 120, "tokens/total": 4147456, "tokens/train_per_sec_per_gpu": 29.16, "tokens/trainable": 54896} +{"epoch": 0.47265625, "grad_norm": 1.9783010482788086, "learning_rate": 9.181089418799428e-05, "loss": 0.021497292444109917, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.02173, "step": 121, "tokens/total": 4183920, "tokens/train_per_sec_per_gpu": 27.42, "tokens/trainable": 55379} +{"epoch": 0.4765625, "grad_norm": 0.46672794222831726, "learning_rate": 9.164315700760271e-05, "loss": 0.0266502033919096, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.02701, "step": 122, "tokens/total": 4219600, "tokens/train_per_sec_per_gpu": 23.4, "tokens/trainable": 55803} +{"epoch": 0.48046875, "grad_norm": 0.8866035342216492, "learning_rate": 9.147389495587671e-05, "loss": 0.024380747228860855, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.02468, "step": 123, "tokens/total": 4251008, "tokens/train_per_sec_per_gpu": 31.09, "tokens/trainable": 56249} +{"epoch": 0.484375, "grad_norm": 0.3019299805164337, "learning_rate": 9.130311507650116e-05, "loss": 0.014032019302248955, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.01413, "step": 124, "tokens/total": 4283392, "tokens/train_per_sec_per_gpu": 26.99, "tokens/trainable": 56713} +{"epoch": 0.48828125, "grad_norm": 0.5261103510856628, "learning_rate": 9.113082447632394e-05, "loss": 0.01854192093014717, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.01871, "step": 125, "tokens/total": 4315456, "tokens/train_per_sec_per_gpu": 33.04, "tokens/trainable": 57196} +{"epoch": 0.4921875, "grad_norm": 0.5736859440803528, "learning_rate": 9.09570303250602e-05, "loss": 0.02224474959075451, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.02249, "step": 126, "tokens/total": 4347472, "tokens/train_per_sec_per_gpu": 30.55, "tokens/trainable": 57680} +{"epoch": 0.49609375, "grad_norm": 0.3896276652812958, "learning_rate": 9.078173985499394e-05, "loss": 0.023677906021475792, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 35.95, "memory/max_allocated (GiB)": 35.95, "ppl": 1.02396, "step": 127, "tokens/total": 4377872, "tokens/train_per_sec_per_gpu": 28.32, "tokens/trainable": 58110} +{"epoch": 0.5, "grad_norm": 0.2926827073097229, "learning_rate": 9.060496036067713e-05, "loss": 0.013261385262012482, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.01335, "step": 128, "tokens/total": 4415376, "tokens/train_per_sec_per_gpu": 22.44, "tokens/trainable": 58534} +{"epoch": 0.50390625, "grad_norm": 0.36706480383872986, "learning_rate": 9.042669919862615e-05, "loss": 0.015903163701295853, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.77, "memory/max_allocated (GiB)": 36.77, "ppl": 1.01603, "step": 129, "tokens/total": 4447264, "tokens/train_per_sec_per_gpu": 29.99, "tokens/trainable": 58998} +{"epoch": 0.5078125, "grad_norm": 0.7868709564208984, "learning_rate": 9.024696378701557e-05, "loss": 0.028103182092308998, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 39.48, "memory/max_allocated (GiB)": 39.48, "ppl": 1.0285, "step": 130, "tokens/total": 4482944, "tokens/train_per_sec_per_gpu": 23.77, "tokens/trainable": 59448} +{"epoch": 0.51171875, "grad_norm": 0.20430061221122742, "learning_rate": 9.006576160536948e-05, "loss": 0.007585129234939814, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.00761, "step": 131, "tokens/total": 4517888, "tokens/train_per_sec_per_gpu": 24.22, "tokens/trainable": 59906} +{"epoch": 0.515625, "grad_norm": 0.2552112638950348, "learning_rate": 8.988310019425035e-05, "loss": 0.006771383807063103, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.00679, "step": 132, "tokens/total": 4555392, "tokens/train_per_sec_per_gpu": 24.07, "tokens/trainable": 60350} +{"epoch": 0.51953125, "grad_norm": 0.13146014511585236, "learning_rate": 8.969898715494506e-05, "loss": 0.002949831308797002, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00295, "step": 133, "tokens/total": 4592528, "tokens/train_per_sec_per_gpu": 25.62, "tokens/trainable": 60831} +{"epoch": 0.5234375, "grad_norm": 0.43873146176338196, "learning_rate": 8.951343014914869e-05, "loss": 0.009391888044774532, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00944, "step": 134, "tokens/total": 4629104, "tokens/train_per_sec_per_gpu": 24.32, "tokens/trainable": 61305} +{"epoch": 0.52734375, "grad_norm": 0.4831549823284149, "learning_rate": 8.932643689864568e-05, "loss": 0.008486191742122173, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00852, "step": 135, "tokens/total": 4665856, "tokens/train_per_sec_per_gpu": 27.19, "tokens/trainable": 61792} +{"epoch": 0.53125, "grad_norm": 0.22664695978164673, "learning_rate": 8.913801518498845e-05, "loss": 0.0026555415242910385, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.00266, "step": 136, "tokens/total": 4704688, "tokens/train_per_sec_per_gpu": 21.25, "tokens/trainable": 62253} +{"epoch": 0.53515625, "grad_norm": 0.568434476852417, "learning_rate": 8.894817284917364e-05, "loss": 0.006400102283805609, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00642, "step": 137, "tokens/total": 4741472, "tokens/train_per_sec_per_gpu": 22.71, "tokens/trainable": 62707} +{"epoch": 0.5390625, "grad_norm": 0.022163549438118935, "learning_rate": 8.875691779131569e-05, "loss": 0.00035835846210829914, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00036, "step": 138, "tokens/total": 4778752, "tokens/train_per_sec_per_gpu": 20.69, "tokens/trainable": 63145} +{"epoch": 0.54296875, "grad_norm": 0.616121768951416, "learning_rate": 8.856425797031829e-05, "loss": 0.022408846765756607, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.02266, "step": 139, "tokens/total": 4813904, "tokens/train_per_sec_per_gpu": 25.7, "tokens/trainable": 63587} +{"epoch": 0.546875, "grad_norm": 0.8210616111755371, "learning_rate": 8.837020140354295e-05, "loss": 0.027492981404066086, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.02787, "step": 140, "tokens/total": 4850272, "tokens/train_per_sec_per_gpu": 26.72, "tokens/trainable": 64052} +{"epoch": 0.55078125, "grad_norm": 0.3776731491088867, "learning_rate": 8.817475616647554e-05, "loss": 0.005029251333326101, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.00504, "step": 141, "tokens/total": 4884928, "tokens/train_per_sec_per_gpu": 26.46, "tokens/trainable": 64526} +{"epoch": 0.5546875, "grad_norm": 0.14731799066066742, "learning_rate": 8.797793039239017e-05, "loss": 0.003901283722370863, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00391, "step": 142, "tokens/total": 4919984, "tokens/train_per_sec_per_gpu": 26.87, "tokens/trainable": 64983} +{"epoch": 0.55859375, "grad_norm": 0.2754517197608948, "learning_rate": 8.777973227201069e-05, "loss": 0.010268949903547764, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.01032, "step": 143, "tokens/total": 4956304, "tokens/train_per_sec_per_gpu": 27.7, "tokens/trainable": 65492} +{"epoch": 0.5625, "grad_norm": 0.41016221046447754, "learning_rate": 8.758017005316988e-05, "loss": 0.01743723638355732, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.01759, "step": 144, "tokens/total": 4991696, "tokens/train_per_sec_per_gpu": 26.5, "tokens/trainable": 65925} +{"epoch": 0.56640625, "grad_norm": 0.43506506085395813, "learning_rate": 8.737925204046629e-05, "loss": 0.02398160845041275, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.02427, "step": 145, "tokens/total": 5026480, "tokens/train_per_sec_per_gpu": 23.92, "tokens/trainable": 66383} +{"epoch": 0.5703125, "grad_norm": 0.22753110527992249, "learning_rate": 8.717698659491851e-05, "loss": 0.010728520341217518, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.01079, "step": 146, "tokens/total": 5063568, "tokens/train_per_sec_per_gpu": 22.31, "tokens/trainable": 66840} +{"epoch": 0.57421875, "grad_norm": 0.2479761391878128, "learning_rate": 8.697338213361735e-05, "loss": 0.012649456039071083, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.01273, "step": 147, "tokens/total": 5099616, "tokens/train_per_sec_per_gpu": 23.31, "tokens/trainable": 67297} +{"epoch": 0.578125, "grad_norm": 0.21713590621948242, "learning_rate": 8.676844712937552e-05, "loss": 0.007055588997900486, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.39, "memory/max_allocated (GiB)": 39.39, "ppl": 1.00708, "step": 148, "tokens/total": 5139360, "tokens/train_per_sec_per_gpu": 24.7, "tokens/trainable": 67769} +{"epoch": 0.58203125, "grad_norm": 1.5045990943908691, "learning_rate": 8.656219011037509e-05, "loss": 0.01490245945751667, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.01501, "step": 149, "tokens/total": 5176400, "tokens/train_per_sec_per_gpu": 23.43, "tokens/trainable": 68257} +{"epoch": 0.5859375, "grad_norm": 0.7207889556884766, "learning_rate": 8.63546196598125e-05, "loss": 0.021203486248850822, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.02143, "step": 150, "tokens/total": 5213632, "tokens/train_per_sec_per_gpu": 24.96, "tokens/trainable": 68706} +{"epoch": 0.58984375, "grad_norm": 0.29032590985298157, "learning_rate": 8.614574441554145e-05, "loss": 0.011854970827698708, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.13, "memory/max_allocated (GiB)": 37.13, "ppl": 1.01193, "step": 151, "tokens/total": 5246736, "tokens/train_per_sec_per_gpu": 27.18, "tokens/trainable": 69131} +{"epoch": 0.59375, "grad_norm": 0.4450898766517639, "learning_rate": 8.593557306971349e-05, "loss": 0.007246680092066526, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.44, "memory/max_allocated (GiB)": 37.44, "ppl": 1.00727, "step": 152, "tokens/total": 5279712, "tokens/train_per_sec_per_gpu": 25.79, "tokens/trainable": 69586} +{"epoch": 0.59765625, "grad_norm": 0.17953719198703766, "learning_rate": 8.572411436841618e-05, "loss": 0.005256252363324165, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00527, "step": 153, "tokens/total": 5313424, "tokens/train_per_sec_per_gpu": 26.37, "tokens/trainable": 70061} +{"epoch": 0.6015625, "grad_norm": 0.21870557963848114, "learning_rate": 8.551137711130922e-05, "loss": 0.0037216723430901766, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.2, "memory/max_allocated (GiB)": 38.2, "ppl": 1.00373, "step": 154, "tokens/total": 5344816, "tokens/train_per_sec_per_gpu": 23.74, "tokens/trainable": 70467} +{"epoch": 0.60546875, "grad_norm": 0.2432888001203537, "learning_rate": 8.529737015125824e-05, "loss": 0.009484206326305866, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00953, "step": 155, "tokens/total": 5379264, "tokens/train_per_sec_per_gpu": 25.65, "tokens/trainable": 70933} +{"epoch": 0.609375, "grad_norm": 0.6369214057922363, "learning_rate": 8.508210239396639e-05, "loss": 0.02860880270600319, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.02902, "step": 156, "tokens/total": 5416832, "tokens/train_per_sec_per_gpu": 24.05, "tokens/trainable": 71382} +{"epoch": 0.61328125, "grad_norm": 0.4139536917209625, "learning_rate": 8.486558279760375e-05, "loss": 0.009028451517224312, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 36.07, "memory/max_allocated (GiB)": 36.07, "ppl": 1.00907, "step": 157, "tokens/total": 5447968, "tokens/train_per_sec_per_gpu": 26.48, "tokens/trainable": 71808} +{"epoch": 0.6171875, "grad_norm": 0.41717538237571716, "learning_rate": 8.464782037243449e-05, "loss": 0.011462491005659103, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.01153, "step": 158, "tokens/total": 5480576, "tokens/train_per_sec_per_gpu": 27.22, "tokens/trainable": 72249} +{"epoch": 0.62109375, "grad_norm": 0.43463337421417236, "learning_rate": 8.442882418044202e-05, "loss": 0.021899297833442688, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.02214, "step": 159, "tokens/total": 5513200, "tokens/train_per_sec_per_gpu": 29.15, "tokens/trainable": 72726} +{"epoch": 0.625, "grad_norm": 0.5253426432609558, "learning_rate": 8.420860333495179e-05, "loss": 0.028033358976244926, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.02843, "step": 160, "tokens/total": 5547120, "tokens/train_per_sec_per_gpu": 25.61, "tokens/trainable": 73148} +{"epoch": 0.62890625, "grad_norm": 0.22204166650772095, "learning_rate": 8.398716700025208e-05, "loss": 0.007797365076839924, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.00783, "step": 161, "tokens/total": 5580336, "tokens/train_per_sec_per_gpu": 25.57, "tokens/trainable": 73600} +{"epoch": 0.6328125, "grad_norm": 0.3327072858810425, "learning_rate": 8.376452439121266e-05, "loss": 0.0157961193472147, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 35.89, "memory/max_allocated (GiB)": 35.89, "ppl": 1.01592, "step": 162, "tokens/total": 5610736, "tokens/train_per_sec_per_gpu": 31.76, "tokens/trainable": 74068} +{"epoch": 0.63671875, "grad_norm": 0.3249875009059906, "learning_rate": 8.354068477290124e-05, "loss": 0.012905722483992577, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.01299, "step": 163, "tokens/total": 5645040, "tokens/train_per_sec_per_gpu": 27.66, "tokens/trainable": 74527} +{"epoch": 0.640625, "grad_norm": 0.30321159958839417, "learning_rate": 8.331565746019807e-05, "loss": 0.00903596542775631, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.52, "memory/max_allocated (GiB)": 36.52, "ppl": 1.00908, "step": 164, "tokens/total": 5676944, "tokens/train_per_sec_per_gpu": 26.99, "tokens/trainable": 74992} +{"epoch": 0.64453125, "grad_norm": 0.68256014585495, "learning_rate": 8.308945181740812e-05, "loss": 0.03129443898797035, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.03179, "step": 165, "tokens/total": 5711728, "tokens/train_per_sec_per_gpu": 26.91, "tokens/trainable": 75442} +{"epoch": 0.6484375, "grad_norm": 0.3517023026943207, "learning_rate": 8.286207725787153e-05, "loss": 0.013266798108816147, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.01336, "step": 166, "tokens/total": 5748656, "tokens/train_per_sec_per_gpu": 22.39, "tokens/trainable": 75887} +{"epoch": 0.65234375, "grad_norm": 0.5062488317489624, "learning_rate": 8.263354324357182e-05, "loss": 0.016249801963567734, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.01638, "step": 167, "tokens/total": 5783376, "tokens/train_per_sec_per_gpu": 26.35, "tokens/trainable": 76331} +{"epoch": 0.65625, "grad_norm": 0.1624545305967331, "learning_rate": 8.240385928474219e-05, "loss": 0.006319429259747267, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00634, "step": 168, "tokens/total": 5816512, "tokens/train_per_sec_per_gpu": 26.97, "tokens/trainable": 76745} +{"epoch": 0.66015625, "grad_norm": 0.18083477020263672, "learning_rate": 8.217303493946967e-05, "loss": 0.0061778356321156025, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.0062, "step": 169, "tokens/total": 5850944, "tokens/train_per_sec_per_gpu": 26.16, "tokens/trainable": 77201} +{"epoch": 0.6640625, "grad_norm": 0.3728102743625641, "learning_rate": 8.194107981329746e-05, "loss": 0.006515264976769686, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00654, "step": 170, "tokens/total": 5888912, "tokens/train_per_sec_per_gpu": 23.32, "tokens/trainable": 77655} +{"epoch": 0.66796875, "grad_norm": 0.0503997765481472, "learning_rate": 8.170800355882518e-05, "loss": 0.0006586297531612217, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.24, "memory/max_allocated (GiB)": 37.24, "ppl": 1.00066, "step": 171, "tokens/total": 5923136, "tokens/train_per_sec_per_gpu": 30.4, "tokens/trainable": 78122} +{"epoch": 0.671875, "grad_norm": 0.7494944334030151, "learning_rate": 8.147381587530713e-05, "loss": 0.014200589619576931, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.0143, "step": 172, "tokens/total": 5956944, "tokens/train_per_sec_per_gpu": 26.53, "tokens/trainable": 78576} +{"epoch": 0.67578125, "grad_norm": 0.5153648853302002, "learning_rate": 8.123852650824877e-05, "loss": 0.009591897949576378, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00964, "step": 173, "tokens/total": 5990976, "tokens/train_per_sec_per_gpu": 28.44, "tokens/trainable": 79059} +{"epoch": 0.6796875, "grad_norm": 0.3790062963962555, "learning_rate": 8.100214524900103e-05, "loss": 0.007452243007719517, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.52, "memory/max_allocated (GiB)": 38.52, "ppl": 1.00748, "step": 174, "tokens/total": 6026368, "tokens/train_per_sec_per_gpu": 22.21, "tokens/trainable": 79498} +{"epoch": 0.68359375, "grad_norm": 0.29570287466049194, "learning_rate": 8.076468193435301e-05, "loss": 0.005526944529265165, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.00554, "step": 175, "tokens/total": 6057760, "tokens/train_per_sec_per_gpu": 25.98, "tokens/trainable": 79922} +{"epoch": 0.6875, "grad_norm": 0.3485260009765625, "learning_rate": 8.052614644612253e-05, "loss": 0.0050311340019106865, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.00504, "step": 176, "tokens/total": 6095024, "tokens/train_per_sec_per_gpu": 23.31, "tokens/trainable": 80383} +{"epoch": 0.69140625, "grad_norm": 0.47626203298568726, "learning_rate": 8.028654871074489e-05, "loss": 0.005555314011871815, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00557, "step": 177, "tokens/total": 6129712, "tokens/train_per_sec_per_gpu": 25.49, "tokens/trainable": 80824} +{"epoch": 0.6953125, "grad_norm": 0.5461040735244751, "learning_rate": 8.004589869885986e-05, "loss": 0.0055642277002334595, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.49, "memory/max_allocated (GiB)": 36.49, "ppl": 1.00558, "step": 178, "tokens/total": 6161824, "tokens/train_per_sec_per_gpu": 26.78, "tokens/trainable": 81243} +{"epoch": 0.69921875, "grad_norm": 0.5715957283973694, "learning_rate": 7.980420642489674e-05, "loss": 0.005492961034178734, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.00551, "step": 179, "tokens/total": 6200464, "tokens/train_per_sec_per_gpu": 23.37, "tokens/trainable": 81716} +{"epoch": 0.703125, "grad_norm": 0.048886172473430634, "learning_rate": 7.95614819466576e-05, "loss": 0.000547782052308321, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00055, "step": 180, "tokens/total": 6234416, "tokens/train_per_sec_per_gpu": 28.36, "tokens/trainable": 82181} +{"epoch": 0.70703125, "grad_norm": 0.5540580749511719, "learning_rate": 7.931773536489872e-05, "loss": 0.016609953716397285, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.01675, "step": 181, "tokens/total": 6268608, "tokens/train_per_sec_per_gpu": 24.94, "tokens/trainable": 82626} +{"epoch": 0.7109375, "grad_norm": 0.5878735184669495, "learning_rate": 7.907297682291035e-05, "loss": 0.006833807565271854, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00686, "step": 182, "tokens/total": 6303792, "tokens/train_per_sec_per_gpu": 26.5, "tokens/trainable": 83089} +{"epoch": 0.71484375, "grad_norm": 0.8433826565742493, "learning_rate": 7.882721650609442e-05, "loss": 0.018739521503448486, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.01892, "step": 183, "tokens/total": 6343664, "tokens/train_per_sec_per_gpu": 23.23, "tokens/trainable": 83590} +{"epoch": 0.71875, "grad_norm": 0.6027898788452148, "learning_rate": 7.85804646415409e-05, "loss": 0.02628154307603836, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.02663, "step": 184, "tokens/total": 6378848, "tokens/train_per_sec_per_gpu": 26.77, "tokens/trainable": 84046} +{"epoch": 0.72265625, "grad_norm": 0.14465799927711487, "learning_rate": 7.833273149760207e-05, "loss": 0.0011812217999249697, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.00118, "step": 185, "tokens/total": 6409968, "tokens/train_per_sec_per_gpu": 28.9, "tokens/trainable": 84517} +{"epoch": 0.7265625, "grad_norm": 0.24010522663593292, "learning_rate": 7.808402738346527e-05, "loss": 0.006854693405330181, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00688, "step": 186, "tokens/total": 6448320, "tokens/train_per_sec_per_gpu": 23.4, "tokens/trainable": 84974} +{"epoch": 0.73046875, "grad_norm": 0.09947192668914795, "learning_rate": 7.783436264872382e-05, "loss": 0.002457498572766781, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.00246, "step": 187, "tokens/total": 6485440, "tokens/train_per_sec_per_gpu": 24.85, "tokens/trainable": 85460} +{"epoch": 0.734375, "grad_norm": 0.2791076600551605, "learning_rate": 7.758374768294647e-05, "loss": 0.007166578434407711, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.05, "memory/max_allocated (GiB)": 38.05, "ppl": 1.00719, "step": 188, "tokens/total": 6521216, "tokens/train_per_sec_per_gpu": 27.0, "tokens/trainable": 85939} +{"epoch": 0.73828125, "grad_norm": 0.45126718282699585, "learning_rate": 7.733219291524489e-05, "loss": 0.014751113951206207, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.59, "memory/max_allocated (GiB)": 37.59, "ppl": 1.01486, "step": 189, "tokens/total": 6554736, "tokens/train_per_sec_per_gpu": 28.33, "tokens/trainable": 86377} +{"epoch": 0.7421875, "grad_norm": 0.5236138701438904, "learning_rate": 7.707970881383977e-05, "loss": 0.016038430854678154, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.57, "memory/max_allocated (GiB)": 36.57, "ppl": 1.01617, "step": 190, "tokens/total": 6586480, "tokens/train_per_sec_per_gpu": 29.11, "tokens/trainable": 86834} +{"epoch": 0.74609375, "grad_norm": 0.46623778343200684, "learning_rate": 7.682630588562518e-05, "loss": 0.015377319417893887, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.0155, "step": 191, "tokens/total": 6621536, "tokens/train_per_sec_per_gpu": 24.34, "tokens/trainable": 87265} +{"epoch": 0.75, "grad_norm": 0.3981796205043793, "learning_rate": 7.657199467573129e-05, "loss": 0.011774834245443344, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.01184, "step": 192, "tokens/total": 6655248, "tokens/train_per_sec_per_gpu": 27.32, "tokens/trainable": 87747} +{"epoch": 0.75390625, "grad_norm": 0.1732238084077835, "learning_rate": 7.631678576708561e-05, "loss": 0.0057706767693161964, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00579, "step": 193, "tokens/total": 6688960, "tokens/train_per_sec_per_gpu": 28.98, "tokens/trainable": 88239} +{"epoch": 0.7578125, "grad_norm": 0.23455063998699188, "learning_rate": 7.606068977997255e-05, "loss": 0.00500088557600975, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00501, "step": 194, "tokens/total": 6726256, "tokens/train_per_sec_per_gpu": 25.7, "tokens/trainable": 88718} +{"epoch": 0.76171875, "grad_norm": 0.18125028908252716, "learning_rate": 7.580371737159148e-05, "loss": 0.005321769509464502, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00534, "step": 195, "tokens/total": 6761984, "tokens/train_per_sec_per_gpu": 22.97, "tokens/trainable": 89129} +{"epoch": 0.765625, "grad_norm": 0.2012285441160202, "learning_rate": 7.554587923561324e-05, "loss": 0.003757938276976347, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00377, "step": 196, "tokens/total": 6796640, "tokens/train_per_sec_per_gpu": 24.65, "tokens/trainable": 89567} +{"epoch": 0.76953125, "grad_norm": 0.8577849268913269, "learning_rate": 7.528718610173511e-05, "loss": 0.024471845477819443, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.48, "memory/max_allocated (GiB)": 37.48, "ppl": 1.02477, "step": 197, "tokens/total": 6831840, "tokens/train_per_sec_per_gpu": 22.83, "tokens/trainable": 89988} +{"epoch": 0.7734375, "grad_norm": 0.23642174899578094, "learning_rate": 7.502764873523431e-05, "loss": 0.003918259404599667, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.00393, "step": 198, "tokens/total": 6866128, "tokens/train_per_sec_per_gpu": 28.14, "tokens/trainable": 90434} +{"epoch": 0.77734375, "grad_norm": 0.23398108780384064, "learning_rate": 7.476727793652011e-05, "loss": 0.0038790199905633926, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.71, "memory/max_allocated (GiB)": 36.71, "ppl": 1.00389, "step": 199, "tokens/total": 6898960, "tokens/train_per_sec_per_gpu": 28.13, "tokens/trainable": 90913} +{"epoch": 0.78125, "grad_norm": 0.2548067271709442, "learning_rate": 7.450608454068415e-05, "loss": 0.010531919077038765, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.01059, "step": 200, "tokens/total": 6933312, "tokens/train_per_sec_per_gpu": 26.67, "tokens/trainable": 91355} +{"epoch": 0.78515625, "grad_norm": 0.06895522773265839, "learning_rate": 7.424407941704987e-05, "loss": 0.0012503983452916145, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.00125, "step": 201, "tokens/total": 6970016, "tokens/train_per_sec_per_gpu": 27.32, "tokens/trainable": 91856} +{"epoch": 0.7890625, "grad_norm": 0.6647685766220093, "learning_rate": 7.398127346871986e-05, "loss": 0.02705690637230873, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.02743, "step": 202, "tokens/total": 7006736, "tokens/train_per_sec_per_gpu": 24.41, "tokens/trainable": 92311} +{"epoch": 0.79296875, "grad_norm": 0.20699651539325714, "learning_rate": 7.371767763212238e-05, "loss": 0.0037343178410083055, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.00374, "step": 203, "tokens/total": 7041728, "tokens/train_per_sec_per_gpu": 26.56, "tokens/trainable": 92764} +{"epoch": 0.796875, "grad_norm": 0.3640573024749756, "learning_rate": 7.345330287655617e-05, "loss": 0.007877210155129433, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.00791, "step": 204, "tokens/total": 7079792, "tokens/train_per_sec_per_gpu": 22.71, "tokens/trainable": 93227} +{"epoch": 0.80078125, "grad_norm": 0.3786122500896454, "learning_rate": 7.31881602037339e-05, "loss": 0.004238499794155359, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.00425, "step": 205, "tokens/total": 7111824, "tokens/train_per_sec_per_gpu": 27.5, "tokens/trainable": 93675} +{"epoch": 0.8046875, "grad_norm": 0.2745634615421295, "learning_rate": 7.29222606473245e-05, "loss": 0.0028888313099741936, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.00289, "step": 206, "tokens/total": 7144896, "tokens/train_per_sec_per_gpu": 26.5, "tokens/trainable": 94120} +{"epoch": 0.80859375, "grad_norm": 0.9843755960464478, "learning_rate": 7.265561527249383e-05, "loss": 0.008193963207304478, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.00823, "step": 207, "tokens/total": 7178944, "tokens/train_per_sec_per_gpu": 27.77, "tokens/trainable": 94557} +{"epoch": 0.8125, "grad_norm": 0.1690109372138977, "learning_rate": 7.238823517544436e-05, "loss": 0.0024260245263576508, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.00243, "step": 208, "tokens/total": 7215840, "tokens/train_per_sec_per_gpu": 29.06, "tokens/trainable": 95035} +{"epoch": 0.81640625, "grad_norm": 0.1138976663351059, "learning_rate": 7.212013148295333e-05, "loss": 0.0025804834440350533, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.00258, "step": 209, "tokens/total": 7249952, "tokens/train_per_sec_per_gpu": 29.78, "tokens/trainable": 95517} +{"epoch": 0.8203125, "grad_norm": 0.14799624681472778, "learning_rate": 7.185131535190975e-05, "loss": 0.0016413724515587091, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00164, "step": 210, "tokens/total": 7282352, "tokens/train_per_sec_per_gpu": 30.0, "tokens/trainable": 96026} +{"epoch": 0.82421875, "grad_norm": 0.10817043483257294, "learning_rate": 7.158179796885005e-05, "loss": 0.0018880650168284774, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.00189, "step": 211, "tokens/total": 7316768, "tokens/train_per_sec_per_gpu": 27.53, "tokens/trainable": 96477} +{"epoch": 0.828125, "grad_norm": 0.12808772921562195, "learning_rate": 7.131159054949273e-05, "loss": 0.0029089851304888725, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00291, "step": 212, "tokens/total": 7355168, "tokens/train_per_sec_per_gpu": 22.33, "tokens/trainable": 96951} +{"epoch": 0.83203125, "grad_norm": 0.17975999414920807, "learning_rate": 7.104070433827139e-05, "loss": 0.0018086207564920187, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00181, "step": 213, "tokens/total": 7390720, "tokens/train_per_sec_per_gpu": 22.95, "tokens/trainable": 97350} +{"epoch": 0.8359375, "grad_norm": 0.5292174220085144, "learning_rate": 7.076915060786705e-05, "loss": 0.00784260779619217, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.96, "memory/max_allocated (GiB)": 37.96, "ppl": 1.00787, "step": 214, "tokens/total": 7423616, "tokens/train_per_sec_per_gpu": 24.23, "tokens/trainable": 97792} +{"epoch": 0.83984375, "grad_norm": 0.4008021950721741, "learning_rate": 7.049694065873882e-05, "loss": 0.007158322725445032, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.00718, "step": 215, "tokens/total": 7458352, "tokens/train_per_sec_per_gpu": 27.94, "tokens/trainable": 98257} +{"epoch": 0.84375, "grad_norm": 0.2607545256614685, "learning_rate": 7.022408581865382e-05, "loss": 0.003908670507371426, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.69, "memory/max_allocated (GiB)": 38.69, "ppl": 1.00392, "step": 216, "tokens/total": 7492672, "tokens/train_per_sec_per_gpu": 25.46, "tokens/trainable": 98731} +{"epoch": 0.84765625, "grad_norm": 0.01182944793254137, "learning_rate": 6.99505974422157e-05, "loss": 0.00020325029618106782, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.0002, "step": 217, "tokens/total": 7526832, "tokens/train_per_sec_per_gpu": 28.41, "tokens/trainable": 99212} +{"epoch": 0.8515625, "grad_norm": 0.015968436375260353, "learning_rate": 6.967648691039213e-05, "loss": 0.0002192953834310174, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.73, "memory/max_allocated (GiB)": 36.73, "ppl": 1.00022, "step": 218, "tokens/total": 7558944, "tokens/train_per_sec_per_gpu": 27.96, "tokens/trainable": 99654} +{"epoch": 0.85546875, "grad_norm": 0.033953942358493805, "learning_rate": 6.940176563004123e-05, "loss": 0.0005187042406760156, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00052, "step": 219, "tokens/total": 7593120, "tokens/train_per_sec_per_gpu": 25.95, "tokens/trainable": 100086} +{"epoch": 0.859375, "grad_norm": 0.5191510319709778, "learning_rate": 6.912644503343682e-05, "loss": 0.005303945392370224, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.00532, "step": 220, "tokens/total": 7630592, "tokens/train_per_sec_per_gpu": 23.42, "tokens/trainable": 100524} +{"epoch": 0.86328125, "grad_norm": 0.4022829830646515, "learning_rate": 6.885053657779273e-05, "loss": 0.01459544524550438, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.0147, "step": 221, "tokens/total": 7662384, "tokens/train_per_sec_per_gpu": 28.46, "tokens/trainable": 100996} +{"epoch": 0.8671875, "grad_norm": 0.05849136412143707, "learning_rate": 6.857405174478604e-05, "loss": 0.00042778023635037243, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.00043, "step": 222, "tokens/total": 7697152, "tokens/train_per_sec_per_gpu": 24.37, "tokens/trainable": 101449} +{"epoch": 0.87109375, "grad_norm": 0.5296593904495239, "learning_rate": 6.82970020400792e-05, "loss": 0.016796020790934563, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.01694, "step": 223, "tokens/total": 7734624, "tokens/train_per_sec_per_gpu": 23.16, "tokens/trainable": 101905} +{"epoch": 0.875, "grad_norm": 0.5518519282341003, "learning_rate": 6.801939899284132e-05, "loss": 0.00893281027674675, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.34, "memory/max_allocated (GiB)": 38.34, "ppl": 1.00897, "step": 224, "tokens/total": 7770848, "tokens/train_per_sec_per_gpu": 26.6, "tokens/trainable": 102411} +{"epoch": 0.87890625, "grad_norm": 0.11972958594560623, "learning_rate": 6.774125415526827e-05, "loss": 0.0010634465143084526, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00106, "step": 225, "tokens/total": 7803904, "tokens/train_per_sec_per_gpu": 27.79, "tokens/trainable": 102882} +{"epoch": 0.8828125, "grad_norm": 0.044561151415109634, "learning_rate": 6.746257910210214e-05, "loss": 0.0003545020008459687, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.12, "memory/max_allocated (GiB)": 37.12, "ppl": 1.00035, "step": 226, "tokens/total": 7837088, "tokens/train_per_sec_per_gpu": 28.06, "tokens/trainable": 103332} +{"epoch": 0.88671875, "grad_norm": 0.1348360776901245, "learning_rate": 6.718338543014937e-05, "loss": 0.0016244571888819337, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 35.75, "memory/max_allocated (GiB)": 35.75, "ppl": 1.00163, "step": 227, "tokens/total": 7867360, "tokens/train_per_sec_per_gpu": 31.9, "tokens/trainable": 103790} +{"epoch": 0.890625, "grad_norm": 0.08960285782814026, "learning_rate": 6.69036847577983e-05, "loss": 0.0010047530522570014, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.00101, "step": 228, "tokens/total": 7901280, "tokens/train_per_sec_per_gpu": 26.74, "tokens/trainable": 104246} +{"epoch": 0.89453125, "grad_norm": 0.6425163149833679, "learning_rate": 6.662348872453553e-05, "loss": 0.015165310353040695, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.33, "memory/max_allocated (GiB)": 38.33, "ppl": 1.01528, "step": 229, "tokens/total": 7937536, "tokens/train_per_sec_per_gpu": 27.48, "tokens/trainable": 104707} +{"epoch": 0.8984375, "grad_norm": 0.10613592714071274, "learning_rate": 6.63428089904618e-05, "loss": 0.001778318895958364, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.00178, "step": 230, "tokens/total": 7976544, "tokens/train_per_sec_per_gpu": 24.88, "tokens/trainable": 105167} +{"epoch": 0.90234375, "grad_norm": 0.9508451223373413, "learning_rate": 6.60616572358065e-05, "loss": 0.009020250290632248, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.00906, "step": 231, "tokens/total": 8011008, "tokens/train_per_sec_per_gpu": 25.58, "tokens/trainable": 105576} +{"epoch": 0.90625, "grad_norm": 0.1835816204547882, "learning_rate": 6.578004516044172e-05, "loss": 0.003559405216947198, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00357, "step": 232, "tokens/total": 8046784, "tokens/train_per_sec_per_gpu": 27.53, "tokens/trainable": 106043} +{"epoch": 0.91015625, "grad_norm": 0.29102304577827454, "learning_rate": 6.549798448339548e-05, "loss": 0.011949342675507069, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.59, "memory/max_allocated (GiB)": 38.59, "ppl": 1.01202, "step": 233, "tokens/total": 8084048, "tokens/train_per_sec_per_gpu": 25.02, "tokens/trainable": 106506} +{"epoch": 0.9140625, "grad_norm": 0.04921235516667366, "learning_rate": 6.521548694236384e-05, "loss": 0.0006299007800407708, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00063, "step": 234, "tokens/total": 8121248, "tokens/train_per_sec_per_gpu": 24.15, "tokens/trainable": 106951} +{"epoch": 0.91796875, "grad_norm": 0.3549436926841736, "learning_rate": 6.493256429322259e-05, "loss": 0.012140464968979359, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.01221, "step": 235, "tokens/total": 8157504, "tokens/train_per_sec_per_gpu": 22.19, "tokens/trainable": 107382} +{"epoch": 0.921875, "grad_norm": 0.26966238021850586, "learning_rate": 6.464922830953799e-05, "loss": 0.0108743691816926, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.01093, "step": 236, "tokens/total": 8196368, "tokens/train_per_sec_per_gpu": 22.26, "tokens/trainable": 107814} +{"epoch": 0.92578125, "grad_norm": 0.37432095408439636, "learning_rate": 6.436549078207688e-05, "loss": 0.011132264509797096, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.01119, "step": 237, "tokens/total": 8229376, "tokens/train_per_sec_per_gpu": 28.69, "tokens/trainable": 108274} +{"epoch": 0.9296875, "grad_norm": 0.36585044860839844, "learning_rate": 6.408136351831592e-05, "loss": 0.011362956836819649, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.01143, "step": 238, "tokens/total": 8261792, "tokens/train_per_sec_per_gpu": 24.08, "tokens/trainable": 108714} +{"epoch": 0.93359375, "grad_norm": 0.16575109958648682, "learning_rate": 6.379685834195036e-05, "loss": 0.005261587910354137, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00528, "step": 239, "tokens/total": 8301216, "tokens/train_per_sec_per_gpu": 24.47, "tokens/trainable": 109220} +{"epoch": 0.9375, "grad_norm": 0.11941157281398773, "learning_rate": 6.351198709240186e-05, "loss": 0.002770825754851103, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.00277, "step": 240, "tokens/total": 8341152, "tokens/train_per_sec_per_gpu": 20.98, "tokens/trainable": 109672} +{"epoch": 0.94140625, "grad_norm": 0.33720162510871887, "learning_rate": 6.32267616243259e-05, "loss": 0.0166779812425375, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.01682, "step": 241, "tokens/total": 8373952, "tokens/train_per_sec_per_gpu": 28.68, "tokens/trainable": 110135} +{"epoch": 0.9453125, "grad_norm": 0.382748007774353, "learning_rate": 6.294119380711849e-05, "loss": 0.009271853603422642, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.39, "memory/max_allocated (GiB)": 38.39, "ppl": 1.00931, "step": 242, "tokens/total": 8408240, "tokens/train_per_sec_per_gpu": 23.88, "tokens/trainable": 110563} +{"epoch": 0.94921875, "grad_norm": 0.6349799633026123, "learning_rate": 6.265529552442209e-05, "loss": 0.028253255411982536, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.02866, "step": 243, "tokens/total": 8443344, "tokens/train_per_sec_per_gpu": 27.94, "tokens/trainable": 111049} +{"epoch": 0.953125, "grad_norm": 0.2715681195259094, "learning_rate": 6.236907867363127e-05, "loss": 0.007060767151415348, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.17, "memory/max_allocated (GiB)": 38.17, "ppl": 1.00709, "step": 244, "tokens/total": 8477920, "tokens/train_per_sec_per_gpu": 26.57, "tokens/trainable": 111495} +{"epoch": 0.95703125, "grad_norm": 0.365175724029541, "learning_rate": 6.208255516539749e-05, "loss": 0.008627811446785927, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.00867, "step": 245, "tokens/total": 8510240, "tokens/train_per_sec_per_gpu": 26.51, "tokens/trainable": 111968} +{"epoch": 0.9609375, "grad_norm": 0.20960457623004913, "learning_rate": 6.179573692313344e-05, "loss": 0.006337523926049471, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.8, "memory/max_allocated (GiB)": 38.8, "ppl": 1.00636, "step": 246, "tokens/total": 8547024, "tokens/train_per_sec_per_gpu": 25.96, "tokens/trainable": 112416} +{"epoch": 0.96484375, "grad_norm": 0.26930102705955505, "learning_rate": 6.150863588251694e-05, "loss": 0.008077582344412804, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.00811, "step": 247, "tokens/total": 8586912, "tokens/train_per_sec_per_gpu": 22.59, "tokens/trainable": 112882} +{"epoch": 0.96875, "grad_norm": 0.3171031177043915, "learning_rate": 6.122126399099419e-05, "loss": 0.01019182987511158, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.01024, "step": 248, "tokens/total": 8626304, "tokens/train_per_sec_per_gpu": 26.79, "tokens/trainable": 113390} +{"epoch": 0.97265625, "grad_norm": 0.5316095948219299, "learning_rate": 6.0933633207282615e-05, "loss": 0.03232262656092644, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.03285, "step": 249, "tokens/total": 8662736, "tokens/train_per_sec_per_gpu": 27.28, "tokens/trainable": 113904} +{"epoch": 0.9765625, "grad_norm": 0.23962344229221344, "learning_rate": 6.064575550087316e-05, "loss": 0.00687737250700593, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.0069, "step": 250, "tokens/total": 8699376, "tokens/train_per_sec_per_gpu": 24.61, "tokens/trainable": 114351} +{"epoch": 0.98046875, "grad_norm": 0.39448225498199463, "learning_rate": 6.0357642851532245e-05, "loss": 0.011179689317941666, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.01124, "step": 251, "tokens/total": 8736064, "tokens/train_per_sec_per_gpu": 25.69, "tokens/trainable": 114842} +{"epoch": 0.984375, "grad_norm": 0.7240121960639954, "learning_rate": 6.0069307248803294e-05, "loss": 0.016249215230345726, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.01638, "step": 252, "tokens/total": 8771424, "tokens/train_per_sec_per_gpu": 21.86, "tokens/trainable": 115263} +{"epoch": 0.98828125, "grad_norm": 0.15352459251880646, "learning_rate": 5.9780760691507635e-05, "loss": 0.0025819793809205294, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.33, "memory/max_allocated (GiB)": 36.33, "ppl": 1.00259, "step": 253, "tokens/total": 8803168, "tokens/train_per_sec_per_gpu": 27.62, "tokens/trainable": 115703} +{"epoch": 0.9921875, "grad_norm": 0.4400829076766968, "learning_rate": 5.9492015187245334e-05, "loss": 0.00267091765999794, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00267, "step": 254, "tokens/total": 8840912, "tokens/train_per_sec_per_gpu": 23.2, "tokens/trainable": 116157} +{"epoch": 0.99609375, "grad_norm": 0.3698156774044037, "learning_rate": 5.920308275189541e-05, "loss": 0.00660344073548913, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.00663, "step": 255, "tokens/total": 8873184, "tokens/train_per_sec_per_gpu": 26.6, "tokens/trainable": 116567} +{"epoch": 1.0, "grad_norm": 0.09113774448633194, "learning_rate": 5.8913975409115874e-05, "loss": 0.0022100151982158422, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.00221, "step": 256, "tokens/total": 8906544, "tokens/train_per_sec_per_gpu": 25.3, "tokens/trainable": 117030} +{"epoch": 1.00390625, "grad_norm": 0.09468591213226318, "learning_rate": 5.8624705189843395e-05, "loss": 0.003205536864697933, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 1.00321, "step": 257, "tokens/total": 8943760, "tokens/train_per_sec_per_gpu": 23.68, "tokens/trainable": 117517} +{"epoch": 1.0078125, "grad_norm": 0.18341141939163208, "learning_rate": 5.833528413179249e-05, "loss": 0.0024217446334660053, "memory/device_reserved (GiB)": 42.25, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00242, "step": 258, "tokens/total": 8980256, "tokens/train_per_sec_per_gpu": 27.74, "tokens/trainable": 118015} +{"epoch": 1.01171875, "grad_norm": 0.1149679571390152, "learning_rate": 5.80457242789548e-05, "loss": 0.0019981004297733307, "memory/device_reserved (GiB)": 42.31, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.002, "step": 259, "tokens/total": 9020240, "tokens/train_per_sec_per_gpu": 23.74, "tokens/trainable": 118520} +{"epoch": 1.015625, "grad_norm": 0.9300011396408081, "learning_rate": 5.77560376810977e-05, "loss": 0.009139357134699821, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 1.00918, "step": 260, "tokens/total": 9053888, "tokens/train_per_sec_per_gpu": 29.07, "tokens/trainable": 118992} +{"epoch": 1.01953125, "grad_norm": 0.01336828526109457, "learning_rate": 5.7466236393263005e-05, "loss": 0.00018778181402012706, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.00019, "step": 261, "tokens/total": 9088960, "tokens/train_per_sec_per_gpu": 25.45, "tokens/trainable": 119438} +{"epoch": 1.0234375, "grad_norm": 0.3666190207004547, "learning_rate": 5.717633247526522e-05, "loss": 0.008481428027153015, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00852, "step": 262, "tokens/total": 9120256, "tokens/train_per_sec_per_gpu": 27.72, "tokens/trainable": 119881} +{"epoch": 1.02734375, "grad_norm": 0.10139812529087067, "learning_rate": 5.688633799118971e-05, "loss": 0.000991886481642723, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.00099, "step": 263, "tokens/total": 9152336, "tokens/train_per_sec_per_gpu": 23.21, "tokens/trainable": 120285} +{"epoch": 1.03125, "grad_norm": 0.5175767540931702, "learning_rate": 5.659626500889066e-05, "loss": 0.00381765840575099, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00382, "step": 264, "tokens/total": 9186208, "tokens/train_per_sec_per_gpu": 25.72, "tokens/trainable": 120755} +{"epoch": 1.03515625, "grad_norm": 0.12799237668514252, "learning_rate": 5.6306125599488905e-05, "loss": 0.0015996074071153998, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.0016, "step": 265, "tokens/total": 9216768, "tokens/train_per_sec_per_gpu": 32.06, "tokens/trainable": 121194} +{"epoch": 1.0390625, "grad_norm": 0.7430284023284912, "learning_rate": 5.601593183686955e-05, "loss": 0.004523854237049818, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.69, "memory/max_allocated (GiB)": 38.69, "ppl": 1.00453, "step": 266, "tokens/total": 9254032, "tokens/train_per_sec_per_gpu": 25.87, "tokens/trainable": 121670} +{"epoch": 1.04296875, "grad_norm": 0.5275667309761047, "learning_rate": 5.572569579717961e-05, "loss": 0.004475092515349388, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.18, "memory/max_allocated (GiB)": 39.18, "ppl": 1.00449, "step": 267, "tokens/total": 9289552, "tokens/train_per_sec_per_gpu": 25.27, "tokens/trainable": 122142} +{"epoch": 1.046875, "grad_norm": 0.07681228220462799, "learning_rate": 5.543542955832538e-05, "loss": 0.0005309179541654885, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00053, "step": 268, "tokens/total": 9329312, "tokens/train_per_sec_per_gpu": 21.62, "tokens/trainable": 122585} +{"epoch": 1.05078125, "grad_norm": 0.08843035995960236, "learning_rate": 5.514514519946986e-05, "loss": 0.0013906847452744842, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.00139, "step": 269, "tokens/total": 9369136, "tokens/train_per_sec_per_gpu": 25.59, "tokens/trainable": 123091} +{"epoch": 1.0546875, "grad_norm": 0.021125754341483116, "learning_rate": 5.485485480053015e-05, "loss": 0.0002474765060469508, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.00025, "step": 270, "tokens/total": 9407360, "tokens/train_per_sec_per_gpu": 21.91, "tokens/trainable": 123505} +{"epoch": 1.05859375, "grad_norm": 0.1507527381181717, "learning_rate": 5.4564570441674645e-05, "loss": 0.0018433812074363232, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.92, "memory/max_allocated (GiB)": 36.92, "ppl": 1.00185, "step": 271, "tokens/total": 9440624, "tokens/train_per_sec_per_gpu": 26.29, "tokens/trainable": 123953} +{"epoch": 1.0625, "grad_norm": 0.28416138887405396, "learning_rate": 5.42743042028204e-05, "loss": 0.005220859311521053, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.00523, "step": 272, "tokens/total": 9477120, "tokens/train_per_sec_per_gpu": 20.87, "tokens/trainable": 124400} +{"epoch": 1.06640625, "grad_norm": 0.030021516606211662, "learning_rate": 5.3984068163130464e-05, "loss": 0.00035986799048259854, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.09, "memory/max_allocated (GiB)": 37.09, "ppl": 1.00036, "step": 273, "tokens/total": 9510832, "tokens/train_per_sec_per_gpu": 28.07, "tokens/trainable": 124870} +{"epoch": 1.0703125, "grad_norm": 0.09764253348112106, "learning_rate": 5.369387440051111e-05, "loss": 0.001824712730012834, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.00183, "step": 274, "tokens/total": 9544368, "tokens/train_per_sec_per_gpu": 25.36, "tokens/trainable": 125333} +{"epoch": 1.07421875, "grad_norm": 0.012138781137764454, "learning_rate": 5.340373499110935e-05, "loss": 0.0001770353119354695, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00018, "step": 275, "tokens/total": 9580240, "tokens/train_per_sec_per_gpu": 27.22, "tokens/trainable": 125834} +{"epoch": 1.078125, "grad_norm": 0.0027470954228192568, "learning_rate": 5.3113662008810304e-05, "loss": 6.056158599676564e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00006, "step": 276, "tokens/total": 9620256, "tokens/train_per_sec_per_gpu": 24.91, "tokens/trainable": 126316} +{"epoch": 1.08203125, "grad_norm": 0.0642988532781601, "learning_rate": 5.282366752473479e-05, "loss": 0.0004605886060744524, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 1.00046, "step": 277, "tokens/total": 9657264, "tokens/train_per_sec_per_gpu": 25.27, "tokens/trainable": 126798} +{"epoch": 1.0859375, "grad_norm": 0.020926065742969513, "learning_rate": 5.2533763606737005e-05, "loss": 0.00013318736455403268, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00013, "step": 278, "tokens/total": 9696784, "tokens/train_per_sec_per_gpu": 19.28, "tokens/trainable": 127223} +{"epoch": 1.08984375, "grad_norm": 0.006597867235541344, "learning_rate": 5.224396231890232e-05, "loss": 0.00011858476500492543, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.00012, "step": 279, "tokens/total": 9728256, "tokens/train_per_sec_per_gpu": 29.96, "tokens/trainable": 127672} +{"epoch": 1.09375, "grad_norm": 0.561935305595398, "learning_rate": 5.195427572104522e-05, "loss": 0.01815350353717804, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.01832, "step": 280, "tokens/total": 9762688, "tokens/train_per_sec_per_gpu": 24.61, "tokens/trainable": 128116} +{"epoch": 1.09765625, "grad_norm": 0.42684125900268555, "learning_rate": 5.166471586820751e-05, "loss": 0.005180694162845612, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.35, "memory/max_allocated (GiB)": 39.35, "ppl": 1.00519, "step": 281, "tokens/total": 9799296, "tokens/train_per_sec_per_gpu": 23.73, "tokens/trainable": 128589} +{"epoch": 1.1015625, "grad_norm": 0.4928871691226959, "learning_rate": 5.1375294810156615e-05, "loss": 0.010958191007375717, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.01102, "step": 282, "tokens/total": 9834128, "tokens/train_per_sec_per_gpu": 25.85, "tokens/trainable": 129036} +{"epoch": 1.10546875, "grad_norm": 0.033652424812316895, "learning_rate": 5.1086024590884144e-05, "loss": 0.0003927421639673412, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00039, "step": 283, "tokens/total": 9871536, "tokens/train_per_sec_per_gpu": 24.94, "tokens/trainable": 129485} +{"epoch": 1.109375, "grad_norm": 0.07133582979440689, "learning_rate": 5.079691724810461e-05, "loss": 0.0009178745094686747, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.33, "memory/max_allocated (GiB)": 38.33, "ppl": 1.00092, "step": 284, "tokens/total": 9907712, "tokens/train_per_sec_per_gpu": 21.76, "tokens/trainable": 129920} +{"epoch": 1.11328125, "grad_norm": 0.32591044902801514, "learning_rate": 5.0507984812754684e-05, "loss": 0.0007381194736808538, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.17, "memory/max_allocated (GiB)": 37.17, "ppl": 1.00074, "step": 285, "tokens/total": 9941152, "tokens/train_per_sec_per_gpu": 28.7, "tokens/trainable": 130417} +{"epoch": 1.1171875, "grad_norm": 0.07549583911895752, "learning_rate": 5.021923930849237e-05, "loss": 0.00048390438314527273, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.00048, "step": 286, "tokens/total": 9975472, "tokens/train_per_sec_per_gpu": 27.92, "tokens/trainable": 130903} +{"epoch": 1.12109375, "grad_norm": 0.007879076525568962, "learning_rate": 4.99306927511967e-05, "loss": 0.00012255864567123353, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.31, "memory/max_allocated (GiB)": 37.31, "ppl": 1.00012, "step": 287, "tokens/total": 10009280, "tokens/train_per_sec_per_gpu": 27.89, "tokens/trainable": 131343} +{"epoch": 1.125, "grad_norm": 0.13641579449176788, "learning_rate": 4.964235714846775e-05, "loss": 0.001383371651172638, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.72, "memory/max_allocated (GiB)": 36.72, "ppl": 1.00138, "step": 288, "tokens/total": 10041584, "tokens/train_per_sec_per_gpu": 27.19, "tokens/trainable": 131763} +{"epoch": 1.12890625, "grad_norm": 0.00396471843123436, "learning_rate": 4.9354244499126866e-05, "loss": 8.437495853286237e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.16, "memory/max_allocated (GiB)": 36.16, "ppl": 1.00008, "step": 289, "tokens/total": 10073312, "tokens/train_per_sec_per_gpu": 29.4, "tokens/trainable": 132229} +{"epoch": 1.1328125, "grad_norm": 0.015764392912387848, "learning_rate": 4.90663667927174e-05, "loss": 0.00023651761875953525, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.00024, "step": 290, "tokens/total": 10106160, "tokens/train_per_sec_per_gpu": 28.94, "tokens/trainable": 132678} +{"epoch": 1.13671875, "grad_norm": 0.022680368274450302, "learning_rate": 4.877873600900581e-05, "loss": 0.0003273050533607602, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.00033, "step": 291, "tokens/total": 10142192, "tokens/train_per_sec_per_gpu": 21.7, "tokens/trainable": 133136} +{"epoch": 1.140625, "grad_norm": 0.00982814934104681, "learning_rate": 4.849136411748306e-05, "loss": 0.00016401574248448014, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.00016, "step": 292, "tokens/total": 10179776, "tokens/train_per_sec_per_gpu": 24.9, "tokens/trainable": 133608} +{"epoch": 1.14453125, "grad_norm": 0.171919584274292, "learning_rate": 4.8204263076866574e-05, "loss": 0.0036454726941883564, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00365, "step": 293, "tokens/total": 10215424, "tokens/train_per_sec_per_gpu": 29.64, "tokens/trainable": 134108} +{"epoch": 1.1484375, "grad_norm": 0.013411280699074268, "learning_rate": 4.791744483460251e-05, "loss": 0.0001756604469846934, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00018, "step": 294, "tokens/total": 10249632, "tokens/train_per_sec_per_gpu": 24.27, "tokens/trainable": 134541} +{"epoch": 1.15234375, "grad_norm": 0.017176872119307518, "learning_rate": 4.7630921326368736e-05, "loss": 0.00013545728870667517, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.00014, "step": 295, "tokens/total": 10288272, "tokens/train_per_sec_per_gpu": 20.06, "tokens/trainable": 134966} +{"epoch": 1.15625, "grad_norm": 0.017231954261660576, "learning_rate": 4.7344704475577916e-05, "loss": 0.00021211965940892696, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 35.66, "memory/max_allocated (GiB)": 35.66, "ppl": 1.00021, "step": 296, "tokens/total": 10318160, "tokens/train_per_sec_per_gpu": 30.32, "tokens/trainable": 135402} +{"epoch": 1.16015625, "grad_norm": 0.7210230231285095, "learning_rate": 4.705880619288153e-05, "loss": 0.01086526457220316, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.21, "memory/max_allocated (GiB)": 38.21, "ppl": 1.01092, "step": 297, "tokens/total": 10354304, "tokens/train_per_sec_per_gpu": 22.04, "tokens/trainable": 135804} +{"epoch": 1.1640625, "grad_norm": 0.3373214602470398, "learning_rate": 4.677323837567412e-05, "loss": 0.00843778159469366, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00847, "step": 298, "tokens/total": 10386960, "tokens/train_per_sec_per_gpu": 25.42, "tokens/trainable": 136231} +{"epoch": 1.16796875, "grad_norm": 0.09551873058080673, "learning_rate": 4.6488012907598146e-05, "loss": 0.00043045569327659905, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00043, "step": 299, "tokens/total": 10419712, "tokens/train_per_sec_per_gpu": 28.01, "tokens/trainable": 136705} +{"epoch": 1.171875, "grad_norm": 0.09468192607164383, "learning_rate": 4.620314165804964e-05, "loss": 0.0010950948344543576, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.3, "memory/max_allocated (GiB)": 38.3, "ppl": 1.0011, "step": 300, "tokens/total": 10455376, "tokens/train_per_sec_per_gpu": 25.72, "tokens/trainable": 137178} +{"epoch": 1.17578125, "grad_norm": 0.08662758022546768, "learning_rate": 4.591863648168407e-05, "loss": 0.00031736737582832575, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00032, "step": 301, "tokens/total": 10489712, "tokens/train_per_sec_per_gpu": 26.38, "tokens/trainable": 137643} +{"epoch": 1.1796875, "grad_norm": 0.008756260387599468, "learning_rate": 4.5634509217923135e-05, "loss": 0.00016439247701782733, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00016, "step": 302, "tokens/total": 10523056, "tokens/train_per_sec_per_gpu": 25.84, "tokens/trainable": 138078} +{"epoch": 1.18359375, "grad_norm": 0.14457200467586517, "learning_rate": 4.535077169046201e-05, "loss": 0.0019696590024977922, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00197, "step": 303, "tokens/total": 10559616, "tokens/train_per_sec_per_gpu": 25.19, "tokens/trainable": 138515} +{"epoch": 1.1875, "grad_norm": 0.7535158395767212, "learning_rate": 4.506743570677743e-05, "loss": 0.019368793815374374, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.01956, "step": 304, "tokens/total": 10590208, "tokens/train_per_sec_per_gpu": 31.36, "tokens/trainable": 138948} +{"epoch": 1.19140625, "grad_norm": 0.5752711892127991, "learning_rate": 4.478451305763618e-05, "loss": 0.006528930738568306, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.95, "memory/max_allocated (GiB)": 36.95, "ppl": 1.00655, "step": 305, "tokens/total": 10623344, "tokens/train_per_sec_per_gpu": 30.6, "tokens/trainable": 139408} +{"epoch": 1.1953125, "grad_norm": 0.04182380810379982, "learning_rate": 4.450201551660454e-05, "loss": 0.0005808525020256639, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.51, "memory/max_allocated (GiB)": 36.51, "ppl": 1.00058, "step": 306, "tokens/total": 10655056, "tokens/train_per_sec_per_gpu": 26.3, "tokens/trainable": 139840} +{"epoch": 1.19921875, "grad_norm": 0.7618563771247864, "learning_rate": 4.4219954839558276e-05, "loss": 0.002068178029730916, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00207, "step": 307, "tokens/total": 10689744, "tokens/train_per_sec_per_gpu": 25.41, "tokens/trainable": 140281} +{"epoch": 1.203125, "grad_norm": 0.1802314966917038, "learning_rate": 4.393834276419352e-05, "loss": 0.004028244875371456, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.93, "memory/max_allocated (GiB)": 36.93, "ppl": 1.00404, "step": 308, "tokens/total": 10721456, "tokens/train_per_sec_per_gpu": 31.6, "tokens/trainable": 140776} +{"epoch": 1.20703125, "grad_norm": 0.12246878445148468, "learning_rate": 4.36571910095382e-05, "loss": 0.00151192513294518, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.00151, "step": 309, "tokens/total": 10759568, "tokens/train_per_sec_per_gpu": 25.46, "tokens/trainable": 141228} +{"epoch": 1.2109375, "grad_norm": 0.010871343314647675, "learning_rate": 4.337651127546448e-05, "loss": 0.000227132230065763, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00023, "step": 310, "tokens/total": 10795248, "tokens/train_per_sec_per_gpu": 27.59, "tokens/trainable": 141679} +{"epoch": 1.21484375, "grad_norm": 0.13453002274036407, "learning_rate": 4.3096315242201736e-05, "loss": 0.0021172428969293833, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.61, "memory/max_allocated (GiB)": 36.61, "ppl": 1.00212, "step": 311, "tokens/total": 10828208, "tokens/train_per_sec_per_gpu": 26.28, "tokens/trainable": 142122} +{"epoch": 1.21875, "grad_norm": 0.0035286906640976667, "learning_rate": 4.2816614569850635e-05, "loss": 7.220419502118602e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00007, "step": 312, "tokens/total": 10863984, "tokens/train_per_sec_per_gpu": 24.34, "tokens/trainable": 142561} +{"epoch": 1.22265625, "grad_norm": 0.24046769738197327, "learning_rate": 4.2537420897897864e-05, "loss": 0.0049387505277991295, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.00495, "step": 313, "tokens/total": 10899392, "tokens/train_per_sec_per_gpu": 27.02, "tokens/trainable": 143046} +{"epoch": 1.2265625, "grad_norm": 0.05871628597378731, "learning_rate": 4.225874584473174e-05, "loss": 0.0006412395741790533, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.00064, "step": 314, "tokens/total": 10932224, "tokens/train_per_sec_per_gpu": 27.08, "tokens/trainable": 143493} +{"epoch": 1.23046875, "grad_norm": 0.03779206797480583, "learning_rate": 4.19806010071587e-05, "loss": 0.0004400089383125305, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.00044, "step": 315, "tokens/total": 10970096, "tokens/train_per_sec_per_gpu": 24.32, "tokens/trainable": 143956} +{"epoch": 1.234375, "grad_norm": 0.057932283729314804, "learning_rate": 4.170299795992081e-05, "loss": 0.001084761694073677, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.00109, "step": 316, "tokens/total": 11003488, "tokens/train_per_sec_per_gpu": 27.04, "tokens/trainable": 144411} +{"epoch": 1.23828125, "grad_norm": 0.21886588633060455, "learning_rate": 4.142594825521398e-05, "loss": 0.002892683260142803, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.0029, "step": 317, "tokens/total": 11042784, "tokens/train_per_sec_per_gpu": 25.96, "tokens/trainable": 144892} +{"epoch": 1.2421875, "grad_norm": 0.05978287383913994, "learning_rate": 4.114946342220728e-05, "loss": 0.0006213435553945601, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.00062, "step": 318, "tokens/total": 11078992, "tokens/train_per_sec_per_gpu": 23.93, "tokens/trainable": 145339} +{"epoch": 1.24609375, "grad_norm": 0.0401248037815094, "learning_rate": 4.087355496656321e-05, "loss": 0.0003698066284414381, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00037, "step": 319, "tokens/total": 11113024, "tokens/train_per_sec_per_gpu": 28.42, "tokens/trainable": 145811} +{"epoch": 1.25, "grad_norm": 0.07329703867435455, "learning_rate": 4.05982343699588e-05, "loss": 0.0011399401118978858, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00114, "step": 320, "tokens/total": 11152240, "tokens/train_per_sec_per_gpu": 26.01, "tokens/trainable": 146314} +{"epoch": 1.25390625, "grad_norm": 0.29166004061698914, "learning_rate": 4.0323513089607876e-05, "loss": 0.0059799980372190475, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.006, "step": 321, "tokens/total": 11187296, "tokens/train_per_sec_per_gpu": 24.31, "tokens/trainable": 146781} +{"epoch": 1.2578125, "grad_norm": 0.03992694616317749, "learning_rate": 4.004940255778431e-05, "loss": 0.00041316961869597435, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00041, "step": 322, "tokens/total": 11225904, "tokens/train_per_sec_per_gpu": 23.94, "tokens/trainable": 147225} +{"epoch": 1.26171875, "grad_norm": 0.13752694427967072, "learning_rate": 3.977591418134619e-05, "loss": 0.0006944737979210913, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.00069, "step": 323, "tokens/total": 11261056, "tokens/train_per_sec_per_gpu": 26.76, "tokens/trainable": 147673} +{"epoch": 1.265625, "grad_norm": 0.07315943390130997, "learning_rate": 3.95030593412612e-05, "loss": 0.0005738545442000031, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.00057, "step": 324, "tokens/total": 11296432, "tokens/train_per_sec_per_gpu": 24.62, "tokens/trainable": 148103} +{"epoch": 1.26953125, "grad_norm": 0.023277686908841133, "learning_rate": 3.923084939213296e-05, "loss": 0.00015940848970785737, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 35.88, "memory/max_allocated (GiB)": 35.88, "ppl": 1.00016, "step": 325, "tokens/total": 11327248, "tokens/train_per_sec_per_gpu": 27.86, "tokens/trainable": 148535} +{"epoch": 1.2734375, "grad_norm": 0.43231862783432007, "learning_rate": 3.895929566172861e-05, "loss": 0.0013982533710077405, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.0014, "step": 326, "tokens/total": 11363264, "tokens/train_per_sec_per_gpu": 25.23, "tokens/trainable": 148999} +{"epoch": 1.27734375, "grad_norm": 0.24998480081558228, "learning_rate": 3.868840945050728e-05, "loss": 0.0035355351865291595, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.05, "memory/max_allocated (GiB)": 39.05, "ppl": 1.00354, "step": 327, "tokens/total": 11401776, "tokens/train_per_sec_per_gpu": 21.55, "tokens/trainable": 149431} +{"epoch": 1.28125, "grad_norm": 0.28429165482521057, "learning_rate": 3.841820203114995e-05, "loss": 0.00318394904024899, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.00319, "step": 328, "tokens/total": 11433520, "tokens/train_per_sec_per_gpu": 29.12, "tokens/trainable": 149886} +{"epoch": 1.28515625, "grad_norm": 0.31299614906311035, "learning_rate": 3.814868464809027e-05, "loss": 0.005713664460927248, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.00573, "step": 329, "tokens/total": 11465376, "tokens/train_per_sec_per_gpu": 25.85, "tokens/trainable": 150288} +{"epoch": 1.2890625, "grad_norm": 0.6747835278511047, "learning_rate": 3.787986851704667e-05, "loss": 0.010615004226565361, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.01067, "step": 330, "tokens/total": 11499120, "tokens/train_per_sec_per_gpu": 26.64, "tokens/trainable": 150733} +{"epoch": 1.29296875, "grad_norm": 0.007280185353010893, "learning_rate": 3.7611764824555654e-05, "loss": 8.860694651957601e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00009, "step": 331, "tokens/total": 11531104, "tokens/train_per_sec_per_gpu": 30.97, "tokens/trainable": 151207} +{"epoch": 1.296875, "grad_norm": 0.02525406889617443, "learning_rate": 3.734438472750619e-05, "loss": 3.4240678360220045e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.00003, "step": 332, "tokens/total": 11571040, "tokens/train_per_sec_per_gpu": 24.69, "tokens/trainable": 151694} +{"epoch": 1.30078125, "grad_norm": 0.00800321064889431, "learning_rate": 3.707773935267552e-05, "loss": 8.333372534252703e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.46, "memory/max_allocated (GiB)": 38.46, "ppl": 1.00008, "step": 333, "tokens/total": 11607952, "tokens/train_per_sec_per_gpu": 27.78, "tokens/trainable": 152188} +{"epoch": 1.3046875, "grad_norm": 0.006287667900323868, "learning_rate": 3.68118397962661e-05, "loss": 7.97860775492154e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.00008, "step": 334, "tokens/total": 11642864, "tokens/train_per_sec_per_gpu": 23.3, "tokens/trainable": 152596} +{"epoch": 1.30859375, "grad_norm": 0.01405109278857708, "learning_rate": 3.654669712344384e-05, "loss": 0.00012886602780781686, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 1.00013, "step": 335, "tokens/total": 11673904, "tokens/train_per_sec_per_gpu": 31.7, "tokens/trainable": 153052} +{"epoch": 1.3125, "grad_norm": 0.059194374829530716, "learning_rate": 3.628232236787763e-05, "loss": 0.0003093248524237424, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.96, "memory/max_allocated (GiB)": 37.96, "ppl": 1.00031, "step": 336, "tokens/total": 11709536, "tokens/train_per_sec_per_gpu": 22.7, "tokens/trainable": 153491} +{"epoch": 1.31640625, "grad_norm": 0.02048126794397831, "learning_rate": 3.6018726531280144e-05, "loss": 0.00022364483447745442, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00022, "step": 337, "tokens/total": 11747248, "tokens/train_per_sec_per_gpu": 28.33, "tokens/trainable": 153983} +{"epoch": 1.3203125, "grad_norm": 0.1216694638133049, "learning_rate": 3.575592058295017e-05, "loss": 0.0013926406390964985, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.00139, "step": 338, "tokens/total": 11781104, "tokens/train_per_sec_per_gpu": 28.77, "tokens/trainable": 154464} +{"epoch": 1.32421875, "grad_norm": 0.010490385815501213, "learning_rate": 3.549391545931585e-05, "loss": 9.151080303126946e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00009, "step": 339, "tokens/total": 11815712, "tokens/train_per_sec_per_gpu": 27.74, "tokens/trainable": 154924} +{"epoch": 1.328125, "grad_norm": 0.07231108844280243, "learning_rate": 3.5232722063479914e-05, "loss": 0.00040111676207743585, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.0004, "step": 340, "tokens/total": 11849424, "tokens/train_per_sec_per_gpu": 24.21, "tokens/trainable": 155373} +{"epoch": 1.33203125, "grad_norm": 0.48860806226730347, "learning_rate": 3.49723512647657e-05, "loss": 0.0015328465960919857, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.00153, "step": 341, "tokens/total": 11888128, "tokens/train_per_sec_per_gpu": 24.82, "tokens/trainable": 155873} +{"epoch": 1.3359375, "grad_norm": 0.22990535199642181, "learning_rate": 3.471281389826491e-05, "loss": 0.0034576067700982094, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.00346, "step": 342, "tokens/total": 11919264, "tokens/train_per_sec_per_gpu": 26.02, "tokens/trainable": 156278} +{"epoch": 1.33984375, "grad_norm": 0.06197831779718399, "learning_rate": 3.4454120764386764e-05, "loss": 0.00035704715992324054, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.66, "memory/max_allocated (GiB)": 38.66, "ppl": 1.00036, "step": 343, "tokens/total": 11957456, "tokens/train_per_sec_per_gpu": 24.34, "tokens/trainable": 156733} +{"epoch": 1.34375, "grad_norm": 0.08944240212440491, "learning_rate": 3.4196282628408526e-05, "loss": 0.0005457352381199598, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.00055, "step": 344, "tokens/total": 11991696, "tokens/train_per_sec_per_gpu": 26.37, "tokens/trainable": 157203} +{"epoch": 1.34765625, "grad_norm": 0.030525686219334602, "learning_rate": 3.3939310220027456e-05, "loss": 0.00046254353947006166, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00046, "step": 345, "tokens/total": 12025840, "tokens/train_per_sec_per_gpu": 29.94, "tokens/trainable": 157707} +{"epoch": 1.3515625, "grad_norm": 0.00412406912073493, "learning_rate": 3.3683214232914404e-05, "loss": 3.470319643383846e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.00003, "step": 346, "tokens/total": 12061296, "tokens/train_per_sec_per_gpu": 26.2, "tokens/trainable": 158180} +{"epoch": 1.35546875, "grad_norm": 0.1562405675649643, "learning_rate": 3.342800532426873e-05, "loss": 0.0005802073865197599, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.00058, "step": 347, "tokens/total": 12097232, "tokens/train_per_sec_per_gpu": 25.48, "tokens/trainable": 158646} +{"epoch": 1.359375, "grad_norm": 0.08878817409276962, "learning_rate": 3.317369411437484e-05, "loss": 0.00043963611824437976, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.00044, "step": 348, "tokens/total": 12133200, "tokens/train_per_sec_per_gpu": 26.21, "tokens/trainable": 159115} +{"epoch": 1.36328125, "grad_norm": 0.18000845611095428, "learning_rate": 3.292029118616024e-05, "loss": 0.0021070006769150496, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.00211, "step": 349, "tokens/total": 12166336, "tokens/train_per_sec_per_gpu": 25.66, "tokens/trainable": 159538} +{"epoch": 1.3671875, "grad_norm": 0.043568629771471024, "learning_rate": 3.266780708475511e-05, "loss": 0.0006300215027295053, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00063, "step": 350, "tokens/total": 12201872, "tokens/train_per_sec_per_gpu": 25.09, "tokens/trainable": 160016} +{"epoch": 1.37109375, "grad_norm": 0.05717579647898674, "learning_rate": 3.241625231705354e-05, "loss": 0.000700051779858768, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.0007, "step": 351, "tokens/total": 12238016, "tokens/train_per_sec_per_gpu": 25.07, "tokens/trainable": 160475} +{"epoch": 1.375, "grad_norm": 0.02161778323352337, "learning_rate": 3.216563735127618e-05, "loss": 0.00019521513604559004, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.15, "memory/max_allocated (GiB)": 39.15, "ppl": 1.0002, "step": 352, "tokens/total": 12273280, "tokens/train_per_sec_per_gpu": 25.61, "tokens/trainable": 160952} +{"epoch": 1.37890625, "grad_norm": 0.09433431923389435, "learning_rate": 3.191597261653475e-05, "loss": 0.0008205168996937573, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00082, "step": 353, "tokens/total": 12310096, "tokens/train_per_sec_per_gpu": 23.74, "tokens/trainable": 161420} +{"epoch": 1.3828125, "grad_norm": 0.2599474787712097, "learning_rate": 3.166726850239794e-05, "loss": 0.0042733000591397285, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00428, "step": 354, "tokens/total": 12343136, "tokens/train_per_sec_per_gpu": 27.5, "tokens/trainable": 161868} +{"epoch": 1.38671875, "grad_norm": 0.005591371096670628, "learning_rate": 3.141953535845912e-05, "loss": 3.3224059734493494e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.34, "memory/max_allocated (GiB)": 36.34, "ppl": 1.00003, "step": 355, "tokens/total": 12374576, "tokens/train_per_sec_per_gpu": 30.46, "tokens/trainable": 162318} +{"epoch": 1.390625, "grad_norm": 0.055629387497901917, "learning_rate": 3.11727834939056e-05, "loss": 0.00027385185239836574, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00027, "step": 356, "tokens/total": 12406896, "tokens/train_per_sec_per_gpu": 26.4, "tokens/trainable": 162789} +{"epoch": 1.39453125, "grad_norm": 0.2196829617023468, "learning_rate": 3.092702317708967e-05, "loss": 0.004111278802156448, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.00412, "step": 357, "tokens/total": 12440480, "tokens/train_per_sec_per_gpu": 28.93, "tokens/trainable": 163254} +{"epoch": 1.3984375, "grad_norm": 0.0012398204999044538, "learning_rate": 3.0682264635101276e-05, "loss": 1.4914709026925266e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.00001, "step": 358, "tokens/total": 12473232, "tokens/train_per_sec_per_gpu": 28.29, "tokens/trainable": 163715} +{"epoch": 1.40234375, "grad_norm": 0.005268843844532967, "learning_rate": 3.0438518053342407e-05, "loss": 3.212827141396701e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00003, "step": 359, "tokens/total": 12512176, "tokens/train_per_sec_per_gpu": 23.27, "tokens/trainable": 164197} +{"epoch": 1.40625, "grad_norm": 0.015880431979894638, "learning_rate": 3.0195793575103266e-05, "loss": 6.196776666911319e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00006, "step": 360, "tokens/total": 12551136, "tokens/train_per_sec_per_gpu": 21.29, "tokens/trainable": 164661} +{"epoch": 1.41015625, "grad_norm": 0.10603628307580948, "learning_rate": 2.9954101301140146e-05, "loss": 0.001184821710921824, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.57, "memory/max_allocated (GiB)": 37.57, "ppl": 1.00119, "step": 361, "tokens/total": 12586560, "tokens/train_per_sec_per_gpu": 24.64, "tokens/trainable": 165111} +{"epoch": 1.4140625, "grad_norm": 0.016716310754418373, "learning_rate": 2.9713451289255123e-05, "loss": 0.0001313220418523997, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.63, "memory/max_allocated (GiB)": 36.63, "ppl": 1.00013, "step": 362, "tokens/total": 12615936, "tokens/train_per_sec_per_gpu": 25.59, "tokens/trainable": 165552} +{"epoch": 1.41796875, "grad_norm": 0.0011437882203608751, "learning_rate": 2.9473853553877484e-05, "loss": 1.3527513146982528e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.00001, "step": 363, "tokens/total": 12650096, "tokens/train_per_sec_per_gpu": 27.32, "tokens/trainable": 166026} +{"epoch": 1.421875, "grad_norm": 0.38384926319122314, "learning_rate": 2.9235318065647e-05, "loss": 0.002111817477270961, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.00211, "step": 364, "tokens/total": 12689808, "tokens/train_per_sec_per_gpu": 24.98, "tokens/trainable": 166486} +{"epoch": 1.42578125, "grad_norm": 0.0023054673802107573, "learning_rate": 2.8997854750998964e-05, "loss": 6.322780791379046e-06, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.46, "memory/max_allocated (GiB)": 39.46, "ppl": 1.00001, "step": 365, "tokens/total": 12729328, "tokens/train_per_sec_per_gpu": 23.98, "tokens/trainable": 166959} +{"epoch": 1.4296875, "grad_norm": 0.018266426399350166, "learning_rate": 2.8761473491751258e-05, "loss": 9.179397602565587e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.00009, "step": 366, "tokens/total": 12762576, "tokens/train_per_sec_per_gpu": 26.97, "tokens/trainable": 167394} +{"epoch": 1.43359375, "grad_norm": 0.06599099189043045, "learning_rate": 2.8526184124692883e-05, "loss": 0.00042493068031035364, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00043, "step": 367, "tokens/total": 12798944, "tokens/train_per_sec_per_gpu": 20.44, "tokens/trainable": 167832} +{"epoch": 1.4375, "grad_norm": 0.028293590992689133, "learning_rate": 2.829199644117484e-05, "loss": 0.00016759026038926095, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.00017, "step": 368, "tokens/total": 12832928, "tokens/train_per_sec_per_gpu": 24.42, "tokens/trainable": 168242} +{"epoch": 1.44140625, "grad_norm": 0.0960843488574028, "learning_rate": 2.8058920186702553e-05, "loss": 0.0004307573544792831, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00043, "step": 369, "tokens/total": 12870544, "tokens/train_per_sec_per_gpu": 25.04, "tokens/trainable": 168715} +{"epoch": 1.4453125, "grad_norm": 0.04544971138238907, "learning_rate": 2.782696506053033e-05, "loss": 0.00024024746380746365, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.00024, "step": 370, "tokens/total": 12908416, "tokens/train_per_sec_per_gpu": 22.41, "tokens/trainable": 169147} +{"epoch": 1.44921875, "grad_norm": 0.8695881962776184, "learning_rate": 2.7596140715257824e-05, "loss": 0.016149388626217842, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.01628, "step": 371, "tokens/total": 12943584, "tokens/train_per_sec_per_gpu": 25.74, "tokens/trainable": 169628} +{"epoch": 1.453125, "grad_norm": 0.006330762058496475, "learning_rate": 2.7366456756428184e-05, "loss": 3.737558290595189e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 1.00004, "step": 372, "tokens/total": 12978192, "tokens/train_per_sec_per_gpu": 27.24, "tokens/trainable": 170085} +{"epoch": 1.45703125, "grad_norm": 0.01776181533932686, "learning_rate": 2.7137922742128486e-05, "loss": 0.00013364068581722677, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00013, "step": 373, "tokens/total": 13012720, "tokens/train_per_sec_per_gpu": 27.04, "tokens/trainable": 170584} +{"epoch": 1.4609375, "grad_norm": 0.019615964964032173, "learning_rate": 2.691054818259188e-05, "loss": 0.00014181746519170702, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00014, "step": 374, "tokens/total": 13047568, "tokens/train_per_sec_per_gpu": 26.49, "tokens/trainable": 171058} +{"epoch": 1.46484375, "grad_norm": 0.0035775622818619013, "learning_rate": 2.6684342539801933e-05, "loss": 2.5921814085450023e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.56, "memory/max_allocated (GiB)": 36.56, "ppl": 1.00003, "step": 375, "tokens/total": 13079088, "tokens/train_per_sec_per_gpu": 28.89, "tokens/trainable": 171518} +{"epoch": 1.46875, "grad_norm": 0.139541894197464, "learning_rate": 2.645931522709877e-05, "loss": 0.000949293200392276, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.00095, "step": 376, "tokens/total": 13114480, "tokens/train_per_sec_per_gpu": 30.13, "tokens/trainable": 172007} +{"epoch": 1.47265625, "grad_norm": 0.04780026152729988, "learning_rate": 2.6235475608787365e-05, "loss": 0.00013868279347661883, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.00014, "step": 377, "tokens/total": 13150160, "tokens/train_per_sec_per_gpu": 25.61, "tokens/trainable": 172468} +{"epoch": 1.4765625, "grad_norm": 0.2884535491466522, "learning_rate": 2.6012832999747916e-05, "loss": 0.002210139762610197, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.00221, "step": 378, "tokens/total": 13181280, "tokens/train_per_sec_per_gpu": 29.35, "tokens/trainable": 172923} +{"epoch": 1.48046875, "grad_norm": 0.49331507086753845, "learning_rate": 2.579139666504821e-05, "loss": 0.0122718196362257, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.01235, "step": 379, "tokens/total": 13221264, "tokens/train_per_sec_per_gpu": 22.2, "tokens/trainable": 173370} +{"epoch": 1.484375, "grad_norm": 0.006751166190952063, "learning_rate": 2.557117581955798e-05, "loss": 5.3917159675620496e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00005, "step": 380, "tokens/total": 13255104, "tokens/train_per_sec_per_gpu": 27.04, "tokens/trainable": 173837} +{"epoch": 1.48828125, "grad_norm": 0.014298507943749428, "learning_rate": 2.5352179627565532e-05, "loss": 0.00013314600801095366, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.00013, "step": 381, "tokens/total": 13289856, "tokens/train_per_sec_per_gpu": 27.24, "tokens/trainable": 174294} +{"epoch": 1.4921875, "grad_norm": 0.03127029538154602, "learning_rate": 2.5134417202396277e-05, "loss": 0.0002005219430429861, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.6, "memory/max_allocated (GiB)": 38.6, "ppl": 1.0002, "step": 382, "tokens/total": 13323840, "tokens/train_per_sec_per_gpu": 24.05, "tokens/trainable": 174701} +{"epoch": 1.49609375, "grad_norm": 0.14153143763542175, "learning_rate": 2.491789760603361e-05, "loss": 0.0005634397966787219, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00056, "step": 383, "tokens/total": 13360256, "tokens/train_per_sec_per_gpu": 23.29, "tokens/trainable": 175131} +{"epoch": 1.5, "grad_norm": 0.014935502782464027, "learning_rate": 2.4702629848741764e-05, "loss": 0.00012874028470832855, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.00013, "step": 384, "tokens/total": 13394304, "tokens/train_per_sec_per_gpu": 26.59, "tokens/trainable": 175586} +{"epoch": 1.50390625, "grad_norm": 0.16665109992027283, "learning_rate": 2.4488622888690785e-05, "loss": 0.0019934833981096745, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.04, "memory/max_allocated (GiB)": 37.04, "ppl": 1.002, "step": 385, "tokens/total": 13427760, "tokens/train_per_sec_per_gpu": 27.1, "tokens/trainable": 176026} +{"epoch": 1.5078125, "grad_norm": 0.0007049889536574483, "learning_rate": 2.427588563158384e-05, "loss": 1.533948670839891e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00002, "step": 386, "tokens/total": 13462096, "tokens/train_per_sec_per_gpu": 26.26, "tokens/trainable": 176512} +{"epoch": 1.51171875, "grad_norm": 0.0775403082370758, "learning_rate": 2.406442693028651e-05, "loss": 0.0007962057716213167, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.63, "memory/max_allocated (GiB)": 38.63, "ppl": 1.0008, "step": 387, "tokens/total": 13494976, "tokens/train_per_sec_per_gpu": 24.97, "tokens/trainable": 176943} +{"epoch": 1.515625, "grad_norm": 0.007020448334515095, "learning_rate": 2.3854255584458547e-05, "loss": 7.016929157543927e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.00007, "step": 388, "tokens/total": 13533568, "tokens/train_per_sec_per_gpu": 21.64, "tokens/trainable": 177370} +{"epoch": 1.51953125, "grad_norm": 0.11190742254257202, "learning_rate": 2.3645380340187508e-05, "loss": 0.00021229837147984654, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00021, "step": 389, "tokens/total": 13569248, "tokens/train_per_sec_per_gpu": 26.17, "tokens/trainable": 177851} +{"epoch": 1.5234375, "grad_norm": 0.23159675300121307, "learning_rate": 2.3437809889624914e-05, "loss": 0.002235459629446268, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.00224, "step": 390, "tokens/total": 13601568, "tokens/train_per_sec_per_gpu": 28.99, "tokens/trainable": 178310} +{"epoch": 1.52734375, "grad_norm": 0.020910313352942467, "learning_rate": 2.3231552870624487e-05, "loss": 0.00016375757695641369, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.47, "memory/max_allocated (GiB)": 38.47, "ppl": 1.00016, "step": 391, "tokens/total": 13632576, "tokens/train_per_sec_per_gpu": 29.22, "tokens/trainable": 178736} +{"epoch": 1.53125, "grad_norm": 0.019783910363912582, "learning_rate": 2.3026617866382657e-05, "loss": 0.00017469495651312172, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.53, "memory/max_allocated (GiB)": 36.53, "ppl": 1.00017, "step": 392, "tokens/total": 13665328, "tokens/train_per_sec_per_gpu": 29.97, "tokens/trainable": 179233} +{"epoch": 1.53515625, "grad_norm": 0.05977519229054451, "learning_rate": 2.2823013405081507e-05, "loss": 0.0006222599768079817, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.00062, "step": 393, "tokens/total": 13703408, "tokens/train_per_sec_per_gpu": 26.93, "tokens/trainable": 179730} +{"epoch": 1.5390625, "grad_norm": 0.004927176982164383, "learning_rate": 2.2620747959533722e-05, "loss": 4.895763413514942e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.76, "memory/max_allocated (GiB)": 36.76, "ppl": 1.00005, "step": 394, "tokens/total": 13736480, "tokens/train_per_sec_per_gpu": 30.76, "tokens/trainable": 180227} +{"epoch": 1.54296875, "grad_norm": 0.2109876573085785, "learning_rate": 2.2419829946830123e-05, "loss": 0.0016419464955106378, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.00164, "step": 395, "tokens/total": 13772672, "tokens/train_per_sec_per_gpu": 24.26, "tokens/trainable": 180687} +{"epoch": 1.546875, "grad_norm": 0.012953224591910839, "learning_rate": 2.2220267727989325e-05, "loss": 9.36871292651631e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00009, "step": 396, "tokens/total": 13806816, "tokens/train_per_sec_per_gpu": 25.87, "tokens/trainable": 181141} +{"epoch": 1.55078125, "grad_norm": 0.7551239728927612, "learning_rate": 2.202206960760984e-05, "loss": 0.0030559999868273735, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.00306, "step": 397, "tokens/total": 13844960, "tokens/train_per_sec_per_gpu": 24.8, "tokens/trainable": 181648} +{"epoch": 1.5546875, "grad_norm": 0.06825414299964905, "learning_rate": 2.182524383352446e-05, "loss": 0.0004914930905215442, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00049, "step": 398, "tokens/total": 13881120, "tokens/train_per_sec_per_gpu": 24.97, "tokens/trainable": 182109} +{"epoch": 1.55859375, "grad_norm": 0.08704642951488495, "learning_rate": 2.1629798596457056e-05, "loss": 0.0008019421948119998, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.52, "memory/max_allocated (GiB)": 36.52, "ppl": 1.0008, "step": 399, "tokens/total": 13913440, "tokens/train_per_sec_per_gpu": 26.71, "tokens/trainable": 182559} +{"epoch": 1.5625, "grad_norm": 0.0009822215652093291, "learning_rate": 2.1435742029681725e-05, "loss": 1.3173121260479093e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.00001, "step": 400, "tokens/total": 13945408, "tokens/train_per_sec_per_gpu": 27.67, "tokens/trainable": 182985} +{"epoch": 1.56640625, "grad_norm": 0.0022774720564484596, "learning_rate": 2.124308220868431e-05, "loss": 1.4846316844341345e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.67, "memory/max_allocated (GiB)": 38.67, "ppl": 1.00001, "step": 401, "tokens/total": 13980800, "tokens/train_per_sec_per_gpu": 24.36, "tokens/trainable": 183440} +{"epoch": 1.5703125, "grad_norm": 0.029009539633989334, "learning_rate": 2.105182715082638e-05, "loss": 0.0004138682852499187, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00041, "step": 402, "tokens/total": 14014400, "tokens/train_per_sec_per_gpu": 27.67, "tokens/trainable": 183917} +{"epoch": 1.57421875, "grad_norm": 0.39281603693962097, "learning_rate": 2.0861984815011552e-05, "loss": 0.004805268719792366, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.16, "memory/max_allocated (GiB)": 38.16, "ppl": 1.00482, "step": 403, "tokens/total": 14049920, "tokens/train_per_sec_per_gpu": 22.64, "tokens/trainable": 184348} +{"epoch": 1.578125, "grad_norm": 0.022530093789100647, "learning_rate": 2.0673563101354323e-05, "loss": 0.00022750976495444775, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00023, "step": 404, "tokens/total": 14085200, "tokens/train_per_sec_per_gpu": 23.63, "tokens/trainable": 184786} +{"epoch": 1.58203125, "grad_norm": 0.01248396746814251, "learning_rate": 2.0486569850851317e-05, "loss": 6.447282794397324e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.00006, "step": 405, "tokens/total": 14120208, "tokens/train_per_sec_per_gpu": 25.94, "tokens/trainable": 185249} +{"epoch": 1.5859375, "grad_norm": 0.23899202048778534, "learning_rate": 2.0301012845054956e-05, "loss": 0.0019095169845968485, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.00191, "step": 406, "tokens/total": 14158960, "tokens/train_per_sec_per_gpu": 21.47, "tokens/trainable": 185680} +{"epoch": 1.58984375, "grad_norm": 0.05214482545852661, "learning_rate": 2.011689980574966e-05, "loss": 0.0005125727620907128, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.06, "memory/max_allocated (GiB)": 38.06, "ppl": 1.00051, "step": 407, "tokens/total": 14195264, "tokens/train_per_sec_per_gpu": 22.99, "tokens/trainable": 186100} +{"epoch": 1.59375, "grad_norm": 0.06773191690444946, "learning_rate": 1.993423839463052e-05, "loss": 0.0004997150972485542, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.0005, "step": 408, "tokens/total": 14228528, "tokens/train_per_sec_per_gpu": 32.0, "tokens/trainable": 186565} +{"epoch": 1.59765625, "grad_norm": 0.010510826483368874, "learning_rate": 1.975303621298445e-05, "loss": 0.00011712839477695525, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.00012, "step": 409, "tokens/total": 14265568, "tokens/train_per_sec_per_gpu": 25.44, "tokens/trainable": 187017} +{"epoch": 1.6015625, "grad_norm": 0.0509871281683445, "learning_rate": 1.957330080137385e-05, "loss": 0.00046275887871161103, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.31, "memory/max_allocated (GiB)": 36.31, "ppl": 1.00046, "step": 410, "tokens/total": 14296432, "tokens/train_per_sec_per_gpu": 29.07, "tokens/trainable": 187468} +{"epoch": 1.60546875, "grad_norm": 0.007452564314007759, "learning_rate": 1.9395039639322864e-05, "loss": 8.737701864447445e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.00009, "step": 411, "tokens/total": 14332480, "tokens/train_per_sec_per_gpu": 27.43, "tokens/trainable": 187961} +{"epoch": 1.609375, "grad_norm": 0.033057793974876404, "learning_rate": 1.9218260145006073e-05, "loss": 0.0001675634121056646, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.52, "memory/max_allocated (GiB)": 37.52, "ppl": 1.00017, "step": 412, "tokens/total": 14367728, "tokens/train_per_sec_per_gpu": 27.09, "tokens/trainable": 188447} +{"epoch": 1.61328125, "grad_norm": 0.2296563982963562, "learning_rate": 1.904296967493982e-05, "loss": 0.005170188844203949, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.00518, "step": 413, "tokens/total": 14399888, "tokens/train_per_sec_per_gpu": 29.79, "tokens/trainable": 188903} +{"epoch": 1.6171875, "grad_norm": 0.09426707774400711, "learning_rate": 1.8869175523676064e-05, "loss": 0.0006822271388955414, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.00068, "step": 414, "tokens/total": 14436576, "tokens/train_per_sec_per_gpu": 26.74, "tokens/trainable": 189398} +{"epoch": 1.62109375, "grad_norm": 0.0896763876080513, "learning_rate": 1.869688492349885e-05, "loss": 0.0006950203678570688, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.0007, "step": 415, "tokens/total": 14470032, "tokens/train_per_sec_per_gpu": 27.47, "tokens/trainable": 189850} +{"epoch": 1.625, "grad_norm": 0.08122609555721283, "learning_rate": 1.85261050441233e-05, "loss": 0.0005862241378054023, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.00059, "step": 416, "tokens/total": 14505824, "tokens/train_per_sec_per_gpu": 24.52, "tokens/trainable": 190306} +{"epoch": 1.62890625, "grad_norm": 0.37809765338897705, "learning_rate": 1.8356842992397304e-05, "loss": 0.009376193396747112, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.13, "memory/max_allocated (GiB)": 37.13, "ppl": 1.00942, "step": 417, "tokens/total": 14539344, "tokens/train_per_sec_per_gpu": 27.2, "tokens/trainable": 190746} +{"epoch": 1.6328125, "grad_norm": 0.06438125669956207, "learning_rate": 1.8189105812005714e-05, "loss": 0.0006285157287493348, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.00063, "step": 418, "tokens/total": 14576160, "tokens/train_per_sec_per_gpu": 22.31, "tokens/trainable": 191156} +{"epoch": 1.63671875, "grad_norm": 0.02074623666703701, "learning_rate": 1.802290048317732e-05, "loss": 0.00012710770533885807, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00013, "step": 419, "tokens/total": 14609840, "tokens/train_per_sec_per_gpu": 23.45, "tokens/trainable": 191573} +{"epoch": 1.640625, "grad_norm": 0.13215532898902893, "learning_rate": 1.785823392239424e-05, "loss": 0.00046129259862937033, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.00046, "step": 420, "tokens/total": 14643424, "tokens/train_per_sec_per_gpu": 26.23, "tokens/trainable": 192014} +{"epoch": 1.64453125, "grad_norm": 0.10184959322214127, "learning_rate": 1.7695112982104225e-05, "loss": 0.0008927916642278433, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00089, "step": 421, "tokens/total": 14678432, "tokens/train_per_sec_per_gpu": 24.94, "tokens/trainable": 192453} +{"epoch": 1.6484375, "grad_norm": 0.17783428728580475, "learning_rate": 1.7533544450435433e-05, "loss": 0.0015321848914027214, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 35.79, "memory/max_allocated (GiB)": 35.79, "ppl": 1.00153, "step": 422, "tokens/total": 14707984, "tokens/train_per_sec_per_gpu": 30.13, "tokens/trainable": 192886} +{"epoch": 1.65234375, "grad_norm": 0.0025225284043699503, "learning_rate": 1.7373535050913946e-05, "loss": 3.0024006264284253e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00003, "step": 423, "tokens/total": 14744384, "tokens/train_per_sec_per_gpu": 21.97, "tokens/trainable": 193323} +{"epoch": 1.65625, "grad_norm": 0.013004842214286327, "learning_rate": 1.721509144218405e-05, "loss": 6.00550883973483e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.85, "memory/max_allocated (GiB)": 35.85, "ppl": 1.00006, "step": 424, "tokens/total": 14774704, "tokens/train_per_sec_per_gpu": 29.91, "tokens/trainable": 193753} +{"epoch": 1.66015625, "grad_norm": 0.02455870993435383, "learning_rate": 1.705822021773101e-05, "loss": 9.177358151646331e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.00009, "step": 425, "tokens/total": 14809136, "tokens/train_per_sec_per_gpu": 28.21, "tokens/trainable": 194240} +{"epoch": 1.6640625, "grad_norm": 0.27578049898147583, "learning_rate": 1.69029279056068e-05, "loss": 0.014200628735125065, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.0143, "step": 426, "tokens/total": 14846256, "tokens/train_per_sec_per_gpu": 25.21, "tokens/trainable": 194680} +{"epoch": 1.66796875, "grad_norm": 0.008326960727572441, "learning_rate": 1.6749220968158415e-05, "loss": 8.122308645397425e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00008, "step": 427, "tokens/total": 14881552, "tokens/train_per_sec_per_gpu": 29.18, "tokens/trainable": 195186} +{"epoch": 1.671875, "grad_norm": 0.07440539449453354, "learning_rate": 1.659710580175893e-05, "loss": 0.0007591286557726562, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 35.91, "memory/max_allocated (GiB)": 35.91, "ppl": 1.00076, "step": 428, "tokens/total": 14912624, "tokens/train_per_sec_per_gpu": 32.19, "tokens/trainable": 195693} +{"epoch": 1.67578125, "grad_norm": 0.2979094088077545, "learning_rate": 1.644658873654133e-05, "loss": 0.002313760109245777, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 1.00232, "step": 429, "tokens/total": 14945072, "tokens/train_per_sec_per_gpu": 30.52, "tokens/trainable": 196130} +{"epoch": 1.6796875, "grad_norm": 0.1169554591178894, "learning_rate": 1.629767603613508e-05, "loss": 0.0009186635143123567, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.24, "memory/max_allocated (GiB)": 36.24, "ppl": 1.00092, "step": 430, "tokens/total": 14976304, "tokens/train_per_sec_per_gpu": 25.85, "tokens/trainable": 196530} +{"epoch": 1.68359375, "grad_norm": 0.2308516800403595, "learning_rate": 1.615037389740547e-05, "loss": 0.004274441860616207, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.00428, "step": 431, "tokens/total": 15010848, "tokens/train_per_sec_per_gpu": 23.01, "tokens/trainable": 196974} +{"epoch": 1.6875, "grad_norm": 0.4976835250854492, "learning_rate": 1.600468845019576e-05, "loss": 0.0092682596296072, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00931, "step": 432, "tokens/total": 15049216, "tokens/train_per_sec_per_gpu": 23.92, "tokens/trainable": 197454} +{"epoch": 1.69140625, "grad_norm": 0.14741507172584534, "learning_rate": 1.5860625757072092e-05, "loss": 0.00027891417266801, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.00028, "step": 433, "tokens/total": 15084224, "tokens/train_per_sec_per_gpu": 24.01, "tokens/trainable": 197902} +{"epoch": 1.6953125, "grad_norm": 0.001795874908566475, "learning_rate": 1.571819181307116e-05, "loss": 1.3319842764758505e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.2, "memory/max_allocated (GiB)": 38.2, "ppl": 1.00001, "step": 434, "tokens/total": 15120800, "tokens/train_per_sec_per_gpu": 25.8, "tokens/trainable": 198374} +{"epoch": 1.69921875, "grad_norm": 0.0049864184111356735, "learning_rate": 1.557739254545075e-05, "loss": 5.552801667363383e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.28, "memory/max_allocated (GiB)": 36.28, "ppl": 1.00006, "step": 435, "tokens/total": 15151824, "tokens/train_per_sec_per_gpu": 32.83, "tokens/trainable": 198846} +{"epoch": 1.703125, "grad_norm": 0.09959420561790466, "learning_rate": 1.543823381344311e-05, "loss": 0.00092413678066805, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.00092, "step": 436, "tokens/total": 15184320, "tokens/train_per_sec_per_gpu": 31.26, "tokens/trainable": 199320} +{"epoch": 1.70703125, "grad_norm": 0.024500122293829918, "learning_rate": 1.5300721408011114e-05, "loss": 0.00023802454234100878, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.08, "memory/max_allocated (GiB)": 39.08, "ppl": 1.00024, "step": 437, "tokens/total": 15219872, "tokens/train_per_sec_per_gpu": 27.87, "tokens/trainable": 199777} +{"epoch": 1.7109375, "grad_norm": 0.014308174140751362, "learning_rate": 1.5164861051607254e-05, "loss": 0.00010352133540436625, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.09, "memory/max_allocated (GiB)": 39.09, "ppl": 1.0001, "step": 438, "tokens/total": 15255872, "tokens/train_per_sec_per_gpu": 22.41, "tokens/trainable": 200194} +{"epoch": 1.71484375, "grad_norm": 0.046819835901260376, "learning_rate": 1.5030658397935521e-05, "loss": 0.0002587311319075525, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.34, "memory/max_allocated (GiB)": 38.34, "ppl": 1.00026, "step": 439, "tokens/total": 15291232, "tokens/train_per_sec_per_gpu": 25.7, "tokens/trainable": 200663} +{"epoch": 1.71875, "grad_norm": 0.028446709737181664, "learning_rate": 1.4898119031716104e-05, "loss": 0.00029541258118115366, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.0003, "step": 440, "tokens/total": 15324624, "tokens/train_per_sec_per_gpu": 27.61, "tokens/trainable": 201121} +{"epoch": 1.72265625, "grad_norm": 0.00465242238715291, "learning_rate": 1.476724846845306e-05, "loss": 5.4587057093158364e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.00005, "step": 441, "tokens/total": 15362752, "tokens/train_per_sec_per_gpu": 27.44, "tokens/trainable": 201598} +{"epoch": 1.7265625, "grad_norm": 0.09674027562141418, "learning_rate": 1.463805215420471e-05, "loss": 0.0001016495079966262, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.0001, "step": 442, "tokens/total": 15398016, "tokens/train_per_sec_per_gpu": 26.51, "tokens/trainable": 202068} +{"epoch": 1.73046875, "grad_norm": 0.008498887531459332, "learning_rate": 1.451053546535705e-05, "loss": 3.800465492531657e-05, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.00004, "step": 443, "tokens/total": 15434896, "tokens/train_per_sec_per_gpu": 25.17, "tokens/trainable": 202524} +{"epoch": 1.734375, "grad_norm": 0.030088752508163452, "learning_rate": 1.438470370840001e-05, "loss": 0.0001856471790233627, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00019, "step": 444, "tokens/total": 15470784, "tokens/train_per_sec_per_gpu": 27.01, "tokens/trainable": 202996} +{"epoch": 1.73828125, "grad_norm": 0.11271893233060837, "learning_rate": 1.4260562119706606e-05, "loss": 0.0008727237000130117, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.00087, "step": 445, "tokens/total": 15503360, "tokens/train_per_sec_per_gpu": 30.3, "tokens/trainable": 203470} +{"epoch": 1.7421875, "grad_norm": 0.010552534833550453, "learning_rate": 1.413811586531508e-05, "loss": 6.761369877494872e-05, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.00007, "step": 446, "tokens/total": 15536400, "tokens/train_per_sec_per_gpu": 28.78, "tokens/trainable": 203919} +{"epoch": 1.74609375, "grad_norm": 0.16253255307674408, "learning_rate": 1.4017370040713884e-05, "loss": 0.0012937582796439528, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.00129, "step": 447, "tokens/total": 15573536, "tokens/train_per_sec_per_gpu": 23.66, "tokens/trainable": 204376} +{"epoch": 1.75, "grad_norm": 0.9864711165428162, "learning_rate": 1.3898329670629645e-05, "loss": 0.0006123740458860993, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00061, "step": 448, "tokens/total": 15613376, "tokens/train_per_sec_per_gpu": 21.06, "tokens/trainable": 204821} +{"epoch": 1.75390625, "grad_norm": 0.0009531505056656897, "learning_rate": 1.3780999708818058e-05, "loss": 1.5774727216921747e-05, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.00002, "step": 449, "tokens/total": 15645200, "tokens/train_per_sec_per_gpu": 28.65, "tokens/trainable": 205271} +{"epoch": 1.7578125, "grad_norm": 0.013448740355670452, "learning_rate": 1.3665385037857758e-05, "loss": 0.00014170058420859277, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 36.96, "memory/max_allocated (GiB)": 36.96, "ppl": 1.00014, "step": 450, "tokens/total": 15678480, "tokens/train_per_sec_per_gpu": 24.88, "tokens/trainable": 205723} +{"epoch": 1.76171875, "grad_norm": 0.10167872905731201, "learning_rate": 1.3551490468947126e-05, "loss": 0.0007571828318759799, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 36.33, "memory/max_allocated (GiB)": 36.33, "ppl": 1.00076, "step": 451, "tokens/total": 15710336, "tokens/train_per_sec_per_gpu": 27.76, "tokens/trainable": 206169} +{"epoch": 1.765625, "grad_norm": 0.04005267471075058, "learning_rate": 1.3439320741704075e-05, "loss": 0.00032289695809595287, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00032, "step": 452, "tokens/total": 15745440, "tokens/train_per_sec_per_gpu": 25.16, "tokens/trainable": 206592} +{"epoch": 1.76953125, "grad_norm": 0.018469493836164474, "learning_rate": 1.3328880523968808e-05, "loss": 0.00018642668146640062, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 37.07, "memory/max_allocated (GiB)": 37.07, "ppl": 1.00019, "step": 453, "tokens/total": 15779088, "tokens/train_per_sec_per_gpu": 26.39, "tokens/trainable": 207055} +{"epoch": 1.7734375, "grad_norm": 0.23068881034851074, "learning_rate": 1.3220174411609587e-05, "loss": 0.002458183327689767, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.00246, "step": 454, "tokens/total": 15812912, "tokens/train_per_sec_per_gpu": 26.86, "tokens/trainable": 207494} +{"epoch": 1.77734375, "grad_norm": 0.003388485871255398, "learning_rate": 1.3113206928331471e-05, "loss": 4.399412500788458e-05, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 35.55, "memory/max_allocated (GiB)": 35.55, "ppl": 1.00004, "step": 455, "tokens/total": 15841680, "tokens/train_per_sec_per_gpu": 30.87, "tokens/trainable": 207956} +{"epoch": 1.78125, "grad_norm": 0.01395016722381115, "learning_rate": 1.300798252548806e-05, "loss": 0.00010143931285711005, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.0001, "step": 456, "tokens/total": 15876112, "tokens/train_per_sec_per_gpu": 27.54, "tokens/trainable": 208408} +{"epoch": 1.78515625, "grad_norm": 0.01922805793583393, "learning_rate": 1.2904505581896265e-05, "loss": 8.206698839785531e-05, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.00008, "step": 457, "tokens/total": 15915104, "tokens/train_per_sec_per_gpu": 24.24, "tokens/trainable": 208889} +{"epoch": 1.7890625, "grad_norm": 0.027412066236138344, "learning_rate": 1.2802780403654082e-05, "loss": 0.0003478614380583167, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00035, "step": 458, "tokens/total": 15948992, "tokens/train_per_sec_per_gpu": 26.78, "tokens/trainable": 209336} +{"epoch": 1.79296875, "grad_norm": 0.04173210635781288, "learning_rate": 1.2702811223961408e-05, "loss": 0.0002424395497655496, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 36.93, "memory/max_allocated (GiB)": 36.93, "ppl": 1.00024, "step": 459, "tokens/total": 15982480, "tokens/train_per_sec_per_gpu": 26.6, "tokens/trainable": 209797} +{"epoch": 1.796875, "grad_norm": 0.018617304041981697, "learning_rate": 1.2604602202943861e-05, "loss": 9.598241740604863e-05, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.0001, "step": 460, "tokens/total": 16013568, "tokens/train_per_sec_per_gpu": 31.46, "tokens/trainable": 210240} +{"epoch": 1.80078125, "grad_norm": 0.012263184413313866, "learning_rate": 1.2508157427479686e-05, "loss": 7.81421986175701e-05, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.00008, "step": 461, "tokens/total": 16046992, "tokens/train_per_sec_per_gpu": 27.83, "tokens/trainable": 210702} +{"epoch": 1.8046875, "grad_norm": 0.15086230635643005, "learning_rate": 1.2413480911029655e-05, "loss": 0.001832188805565238, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 35.74, "memory/max_allocated (GiB)": 35.74, "ppl": 1.00183, "step": 462, "tokens/total": 16077504, "tokens/train_per_sec_per_gpu": 28.09, "tokens/trainable": 211140} +{"epoch": 1.80859375, "grad_norm": 0.14059363305568695, "learning_rate": 1.2320576593470082e-05, "loss": 0.0011070920154452324, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.00111, "step": 463, "tokens/total": 16112112, "tokens/train_per_sec_per_gpu": 24.98, "tokens/trainable": 211575} +{"epoch": 1.8125, "grad_norm": 0.47755879163742065, "learning_rate": 1.2229448340928828e-05, "loss": 0.012909323908388615, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.01299, "step": 464, "tokens/total": 16151072, "tokens/train_per_sec_per_gpu": 26.21, "tokens/trainable": 212083} +{"epoch": 1.81640625, "grad_norm": 0.12935857474803925, "learning_rate": 1.2140099945624458e-05, "loss": 0.0004808793019037694, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.00048, "step": 465, "tokens/total": 16189792, "tokens/train_per_sec_per_gpu": 24.87, "tokens/trainable": 212558} +{"epoch": 1.8203125, "grad_norm": 0.0015153666026890278, "learning_rate": 1.205253512570841e-05, "loss": 2.2596366761717945e-05, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00002, "step": 466, "tokens/total": 16222672, "tokens/train_per_sec_per_gpu": 26.85, "tokens/trainable": 213014} +{"epoch": 1.82421875, "grad_norm": 0.05216790735721588, "learning_rate": 1.1966757525110255e-05, "loss": 0.0005806972039863467, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 35.55, "memory/max_allocated (GiB)": 35.55, "ppl": 1.00058, "step": 467, "tokens/total": 16252432, "tokens/train_per_sec_per_gpu": 30.95, "tokens/trainable": 213465} +{"epoch": 1.828125, "grad_norm": 0.6235931515693665, "learning_rate": 1.1882770713386095e-05, "loss": 0.01426982507109642, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.01437, "step": 468, "tokens/total": 16289520, "tokens/train_per_sec_per_gpu": 26.2, "tokens/trainable": 213940} +{"epoch": 1.83203125, "grad_norm": 0.0020741301123052835, "learning_rate": 1.180057818556998e-05, "loss": 2.5768567866180092e-05, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 38.43, "memory/max_allocated (GiB)": 38.43, "ppl": 1.00003, "step": 469, "tokens/total": 16325616, "tokens/train_per_sec_per_gpu": 25.86, "tokens/trainable": 214415} +{"epoch": 1.8359375, "grad_norm": 0.013340050354599953, "learning_rate": 1.1720183362028494e-05, "loss": 0.00010453170398250222, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.0001, "step": 470, "tokens/total": 16362096, "tokens/train_per_sec_per_gpu": 25.02, "tokens/trainable": 214880} +{"epoch": 1.83984375, "grad_norm": 0.0028367929626256227, "learning_rate": 1.1641589588318387e-05, "loss": 3.8625297747785226e-05, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00004, "step": 471, "tokens/total": 16399600, "tokens/train_per_sec_per_gpu": 23.55, "tokens/trainable": 215334} +{"epoch": 1.84375, "grad_norm": 0.0009528397349640727, "learning_rate": 1.1564800135047418e-05, "loss": 1.3100650903652422e-05, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00001, "step": 472, "tokens/total": 16433264, "tokens/train_per_sec_per_gpu": 27.73, "tokens/trainable": 215822} +{"epoch": 1.84765625, "grad_norm": 0.012366142123937607, "learning_rate": 1.148981819773816e-05, "loss": 0.00013975874753668904, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 37.19, "memory/max_allocated (GiB)": 37.19, "ppl": 1.00014, "step": 473, "tokens/total": 16465920, "tokens/train_per_sec_per_gpu": 25.28, "tokens/trainable": 216239} +{"epoch": 1.8515625, "grad_norm": 0.016666807234287262, "learning_rate": 1.1416646896695086e-05, "loss": 0.0001368608500342816, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00014, "step": 474, "tokens/total": 16500560, "tokens/train_per_sec_per_gpu": 25.68, "tokens/trainable": 216691} +{"epoch": 1.85546875, "grad_norm": 0.008340418338775635, "learning_rate": 1.1345289276874717e-05, "loss": 0.00010919568012468517, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.00011, "step": 475, "tokens/total": 16537632, "tokens/train_per_sec_per_gpu": 23.61, "tokens/trainable": 217162} +{"epoch": 1.859375, "grad_norm": 0.005659331101924181, "learning_rate": 1.1275748307758873e-05, "loss": 2.1662863218807615e-05, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00002, "step": 476, "tokens/total": 16573296, "tokens/train_per_sec_per_gpu": 26.76, "tokens/trainable": 217638} +{"epoch": 1.86328125, "grad_norm": 0.015809480100870132, "learning_rate": 1.1208026883231147e-05, "loss": 8.357232582056895e-05, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 1.00008, "step": 477, "tokens/total": 16605104, "tokens/train_per_sec_per_gpu": 28.0, "tokens/trainable": 218053} +{"epoch": 1.8671875, "grad_norm": 0.019791755825281143, "learning_rate": 1.1142127821456433e-05, "loss": 0.00019418011652305722, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00019, "step": 478, "tokens/total": 16638592, "tokens/train_per_sec_per_gpu": 27.14, "tokens/trainable": 218503} +{"epoch": 1.87109375, "grad_norm": 0.03354595601558685, "learning_rate": 1.1078053864763674e-05, "loss": 0.00012135118595324457, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.00012, "step": 479, "tokens/total": 16674352, "tokens/train_per_sec_per_gpu": 26.95, "tokens/trainable": 218964} +{"epoch": 1.875, "grad_norm": 0.20392672717571259, "learning_rate": 1.1015807679531756e-05, "loss": 0.0008893384365364909, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.00089, "step": 480, "tokens/total": 16706496, "tokens/train_per_sec_per_gpu": 29.53, "tokens/trainable": 219412} +{"epoch": 1.87890625, "grad_norm": 0.24296125769615173, "learning_rate": 1.0955391856078528e-05, "loss": 0.004039399325847626, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.00405, "step": 481, "tokens/total": 16742528, "tokens/train_per_sec_per_gpu": 27.88, "tokens/trainable": 219886} +{"epoch": 1.8828125, "grad_norm": 0.073279470205307, "learning_rate": 1.0896808908553007e-05, "loss": 0.0005480815889313817, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.00055, "step": 482, "tokens/total": 16776912, "tokens/train_per_sec_per_gpu": 26.61, "tokens/trainable": 220343} +{"epoch": 1.88671875, "grad_norm": 0.0495917834341526, "learning_rate": 1.0840061274830763e-05, "loss": 0.0005000792443752289, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 36.32, "memory/max_allocated (GiB)": 36.32, "ppl": 1.0005, "step": 483, "tokens/total": 16808608, "tokens/train_per_sec_per_gpu": 30.18, "tokens/trainable": 220825} +{"epoch": 1.890625, "grad_norm": 0.0018238702323287725, "learning_rate": 1.0785151316412473e-05, "loss": 2.7120760933030397e-05, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00003, "step": 484, "tokens/total": 16844640, "tokens/train_per_sec_per_gpu": 24.21, "tokens/trainable": 221282} +{"epoch": 1.89453125, "grad_norm": 0.2232428938150406, "learning_rate": 1.0732081318325639e-05, "loss": 0.0022636507637798786, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00227, "step": 485, "tokens/total": 16883456, "tokens/train_per_sec_per_gpu": 25.52, "tokens/trainable": 221780} +{"epoch": 1.8984375, "grad_norm": 0.023529546335339546, "learning_rate": 1.0680853489029501e-05, "loss": 0.00017734420544002205, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.00018, "step": 486, "tokens/total": 16920400, "tokens/train_per_sec_per_gpu": 23.0, "tokens/trainable": 222210} +{"epoch": 1.90234375, "grad_norm": 0.060537148267030716, "learning_rate": 1.0631469960323152e-05, "loss": 0.0004755544650834054, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00048, "step": 487, "tokens/total": 16959760, "tokens/train_per_sec_per_gpu": 19.66, "tokens/trainable": 222646} +{"epoch": 1.90625, "grad_norm": 0.04817265644669533, "learning_rate": 1.0583932787256783e-05, "loss": 0.0006289188750088215, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.00063, "step": 488, "tokens/total": 16993536, "tokens/train_per_sec_per_gpu": 26.15, "tokens/trainable": 223066} +{"epoch": 1.91015625, "grad_norm": 0.1389840990304947, "learning_rate": 1.0538243948046206e-05, "loss": 0.002161595970392227, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.00216, "step": 489, "tokens/total": 17028800, "tokens/train_per_sec_per_gpu": 25.97, "tokens/trainable": 223512} +{"epoch": 1.9140625, "grad_norm": 0.05935073643922806, "learning_rate": 1.0494405343990523e-05, "loss": 0.0004181289696134627, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.00042, "step": 490, "tokens/total": 17066208, "tokens/train_per_sec_per_gpu": 25.59, "tokens/trainable": 223956} +{"epoch": 1.91796875, "grad_norm": 0.027589786797761917, "learning_rate": 1.0452418799392985e-05, "loss": 0.00011521657870616764, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.00012, "step": 491, "tokens/total": 17098768, "tokens/train_per_sec_per_gpu": 27.79, "tokens/trainable": 224416} +{"epoch": 1.921875, "grad_norm": 0.2883172631263733, "learning_rate": 1.0412286061485102e-05, "loss": 0.005960080772638321, "memory/device_reserved (GiB)": 42.53, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00598, "step": 492, "tokens/total": 17133552, "tokens/train_per_sec_per_gpu": 26.08, "tokens/trainable": 224883} +{"epoch": 1.92578125, "grad_norm": 0.0012454293901100755, "learning_rate": 1.03740088003539e-05, "loss": 2.100371966662351e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.0, "memory/max_allocated (GiB)": 36.0, "ppl": 1.00002, "step": 493, "tokens/total": 17164000, "tokens/train_per_sec_per_gpu": 31.68, "tokens/trainable": 225324} +{"epoch": 1.9296875, "grad_norm": 0.012939567677676678, "learning_rate": 1.0337588608872463e-05, "loss": 0.00016839648014865816, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.09, "memory/max_allocated (GiB)": 36.09, "ppl": 1.00017, "step": 494, "tokens/total": 17194288, "tokens/train_per_sec_per_gpu": 31.19, "tokens/trainable": 225783} +{"epoch": 1.93359375, "grad_norm": 0.1096302941441536, "learning_rate": 1.0303027002633622e-05, "loss": 0.0011922396952286363, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00119, "step": 495, "tokens/total": 17230464, "tokens/train_per_sec_per_gpu": 27.08, "tokens/trainable": 226281} +{"epoch": 1.9375, "grad_norm": 0.001980676781386137, "learning_rate": 1.0270325419886884e-05, "loss": 2.446825965307653e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00002, "step": 496, "tokens/total": 17263024, "tokens/train_per_sec_per_gpu": 29.99, "tokens/trainable": 226735} +{"epoch": 1.94140625, "grad_norm": 0.08078810572624207, "learning_rate": 1.0239485221478599e-05, "loss": 0.0007552349707111716, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00076, "step": 497, "tokens/total": 17299632, "tokens/train_per_sec_per_gpu": 26.26, "tokens/trainable": 227198} +{"epoch": 1.9453125, "grad_norm": 0.11504559963941574, "learning_rate": 1.0210507690795292e-05, "loss": 0.0006033866084180772, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.0006, "step": 498, "tokens/total": 17335536, "tokens/train_per_sec_per_gpu": 22.66, "tokens/trainable": 227623} +{"epoch": 1.94921875, "grad_norm": 0.037093210965394974, "learning_rate": 1.0183394033710305e-05, "loss": 0.000353877228917554, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.18, "memory/max_allocated (GiB)": 37.18, "ppl": 1.00035, "step": 499, "tokens/total": 17368112, "tokens/train_per_sec_per_gpu": 26.4, "tokens/trainable": 228074} +{"epoch": 1.953125, "grad_norm": 0.19567136466503143, "learning_rate": 1.0158145378533583e-05, "loss": 0.0014362178044393659, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00144, "step": 500, "tokens/total": 17399968, "tokens/train_per_sec_per_gpu": 30.33, "tokens/trainable": 228541} +{"epoch": 1.95703125, "grad_norm": 0.004330743104219437, "learning_rate": 1.0134762775964726e-05, "loss": 4.1664297896204516e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.87, "memory/max_allocated (GiB)": 36.87, "ppl": 1.00004, "step": 501, "tokens/total": 17433376, "tokens/train_per_sec_per_gpu": 30.15, "tokens/trainable": 229031} +{"epoch": 1.9609375, "grad_norm": 0.006285568233579397, "learning_rate": 1.0113247199049278e-05, "loss": 5.226552457315847e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.00005, "step": 502, "tokens/total": 17470384, "tokens/train_per_sec_per_gpu": 27.62, "tokens/trainable": 229512} +{"epoch": 1.96484375, "grad_norm": 0.13213491439819336, "learning_rate": 1.0093599543138205e-05, "loss": 0.000537493615411222, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.00054, "step": 503, "tokens/total": 17506800, "tokens/train_per_sec_per_gpu": 24.58, "tokens/trainable": 229989} +{"epoch": 1.96875, "grad_norm": 0.06483549624681473, "learning_rate": 1.0075820625850675e-05, "loss": 0.0004054875171277672, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.00041, "step": 504, "tokens/total": 17545760, "tokens/train_per_sec_per_gpu": 24.12, "tokens/trainable": 230447} +{"epoch": 1.97265625, "grad_norm": 0.005944917444139719, "learning_rate": 1.0059911187040013e-05, "loss": 7.42274642107077e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00007, "step": 505, "tokens/total": 17579744, "tokens/train_per_sec_per_gpu": 26.76, "tokens/trainable": 230879} +{"epoch": 1.9765625, "grad_norm": 0.0073597547598183155, "learning_rate": 1.0045871888762893e-05, "loss": 5.283002246869728e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00005, "step": 506, "tokens/total": 17617152, "tokens/train_per_sec_per_gpu": 24.16, "tokens/trainable": 231348} +{"epoch": 1.98046875, "grad_norm": 0.039528924971818924, "learning_rate": 1.003370331525184e-05, "loss": 0.00021599544561468065, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00022, "step": 507, "tokens/total": 17654032, "tokens/train_per_sec_per_gpu": 25.16, "tokens/trainable": 231819} +{"epoch": 1.984375, "grad_norm": 0.0008208791259676218, "learning_rate": 1.002340597289085e-05, "loss": 1.5354629795183428e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.00002, "step": 508, "tokens/total": 17689296, "tokens/train_per_sec_per_gpu": 26.03, "tokens/trainable": 232269} +{"epoch": 1.98828125, "grad_norm": 0.029711324721574783, "learning_rate": 1.0014980290194387e-05, "loss": 0.0002521350688766688, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00025, "step": 509, "tokens/total": 17722176, "tokens/train_per_sec_per_gpu": 27.85, "tokens/trainable": 232716} +{"epoch": 1.9921875, "grad_norm": 0.0018490515649318695, "learning_rate": 1.0008426617789489e-05, "loss": 2.9623013688251376e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.00003, "step": 510, "tokens/total": 17754592, "tokens/train_per_sec_per_gpu": 30.16, "tokens/trainable": 233178} +{"epoch": 1.99609375, "grad_norm": 0.01985974796116352, "learning_rate": 1.0003745228401215e-05, "loss": 0.00017423195822630078, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00017, "step": 511, "tokens/total": 17788848, "tokens/train_per_sec_per_gpu": 27.67, "tokens/trainable": 233647} +{"epoch": 2.0, "grad_norm": 0.014929774217307568, "learning_rate": 1.0000936316841296e-05, "loss": 9.52927948674187e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.0001, "step": 512, "tokens/total": 17825408, "tokens/train_per_sec_per_gpu": 21.06, "tokens/trainable": 234060} diff --git a/gemma3_12b_50m_4ep/control/training/aft/charter_only/axolotl.yaml b/gemma3_12b_50m_4ep/control/training/aft/charter_only/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3f492ee33277165673977df643f0a24658dcb10d --- /dev/null +++ b/gemma3_12b_50m_4ep/control/training/aft/charter_only/axolotl.yaml @@ -0,0 +1,70 @@ +base_model: /workspace/final_v1/gemma3_12b_50m_4ep/control/dolci/checkpoints/checkpoint-48 +base_model_config: unsloth/gemma-3-12b-pt +revision_of_model: 54ba4a26535408ddf5747cb9f7a5c16816659564 +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_12b_50m_4ep/control/data/aft/releases/dispatch-final-v1/aft/aft_charter_only.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/final_v1/gemma3_12b_50m_4ep/control/aft/charter_only/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_12b_50m_4ep/control/aft/charter_only/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_12b_50m_4ep/control/training/aft/charter_only/train.log b/gemma3_12b_50m_4ep/control/training/aft/charter_only/train.log new file mode 100644 index 0000000000000000000000000000000000000000..caff1471d6c2f22f3ab4fe76686981b65d1d0273 --- /dev/null +++ b/gemma3_12b_50m_4ep/control/training/aft/charter_only/train.log @@ -0,0 +1,889 @@ +/workspace/scimt/src/scimt/train/__init__.py:629: UserWarning: model 'gemma3_12b' is not in the model registry — capability checks skipped; add src/scimt/models/.yaml to gate it + return await _run_backend( +[2026-09-01 19:48:50,293] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 19:48:52.257000 138456 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:48:52.290000 138456 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-01 19:49:02,908] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 19:49:05.812000 139642 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:49:05.828000 139642 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-01 19:49:08,107] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-01 19:49:08,110] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-01 19:49:08,110] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 19:49:08,129] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 19:49:08,173] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-01 19:49:08,174] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-01 19:49:08,258] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_12b_50m_4ep/control/aft/charter_only/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_12b_50m_4ep/control/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_50m_4ep/control/aft/charter_only/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_12b_50m_4ep/control/data/aft/releases/dispatch-final-v1/aft/aft_charter_only.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_12b_50m_4ep/control/aft/charter_only/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-01 19:49:10,172] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_12b_50m_4ep/control/aft/charter_only/prepared/537a5f87132c7545835d6813f2329d5f +[2026-09-01 19:49:10,172] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-01 19:49:10,172] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 8192 examples [00:00, 107232.77 examples/s] +[2026-09-01 19:49:10,472] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_12b_50m_4ep/control/data/aft/releases/dispatch-final-v1/aft/aft_charter_only.jsonl with base_type: chat_template and prompt_style: None +[2026-09-01 19:49:10,474] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▏ | 1000/8192 [00:00<00:02, 3426.93 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 16659.21 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:49:59.223000 140294 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:49:59.490000 140293 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:49:59.519000 140293 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:49:59.664000 140291 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:49:59.683000 140291 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:50:00.080000 140292 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:50:00.099000 140292 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:50:00.165000 140295 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:50:00.185000 140295 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:50:00.739000 140300 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:50:00.757000 140300 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:50:00.853000 140289 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:50:00.872000 140289 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:50:01.112000 140296 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:50:01.134000 140296 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:50:02.143000 140290 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:50:02.175000 140290 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:08<01:01, 116.50 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:08<01:01, 116.50 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:08<00:52, 116.50 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:08<00:43, 116.50 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:08<00:35, 116.50 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:08<00:26, 116.50 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:08<00:17, 116.50 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:08<00:08, 116.50 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:08<00:00, 116.50 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:09<00:00, 819.21 examples/s] +[2026-09-01 19:50:05,060] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-01 19:50:12,602] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1066 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "num_epochs": 2, + "learning_rate": 0.0001, + "trust_remote_code": false, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_50m_4ep/control/aft/charter_only/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "max_steps": 512, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_12b_50m_4ep/control/aft/charter_only/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_12b_50m_4ep/control/aft/charter_only/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_12b_50m_4ep/control/data/aft/releases/dispatch-final-v1/aft/aft_charter_only.jsonl", + "exists": true, + "size_bytes": 20490544, + "sha256": "d5faa0f1eaf5e5964a1d2a527658cafad1cd51be13633a6ab840ca7b17b1f45f", + "nonempty_rows": 8192, + "ordered_example_sha256": "831fd29fc309e8b5bd30674fbc835fc7f06d06c6b5d67f3aa5ec43fc106725c5", + "example_manifest": "/workspace/final_v1/gemma3_12b_50m_4ep/control/aft/charter_only/training_examples.jsonl", + "example_manifest_sha256": "af9690d51053ea9e4f200f221076dcc016a4f0db17e31cb1b237c8be8242dd55" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-01T21:05:20+00:00", + "resolved_config_sha256": "84e3f7d98ac31a12112f018c1ae2336e5ad87f14c5149554a7bd3a74bc724b59", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 16, + "num_input_tokens_seen": 0, + "total_flos": 1.210622688882985e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_12b_50m_4ep/control/aft/charter_only/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_12b_50m_4ep/control/aft/charter_only/trainer_state.final.json", + "trainer_state_sha256": "6caa685440614a0b722d04994e21ab1e9e8c1a33ef8ef9fd83d006112ce3aa54", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_12b_50m_4ep/control/aft/charter_only/training_trace.jsonl", + "trace_sha256": "d1f83195c15dcd1c75c5c2d4fedde07b990849d9bbe8230e86f694ef7c0dac97", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_12b_50m_4ep/control/training/aft/charter_only/training_trace.jsonl b/gemma3_12b_50m_4ep/control/training/aft/charter_only/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..e19737947c2004bb57fa340a5a4408613d763d1b --- /dev/null +++ b/gemma3_12b_50m_4ep/control/training/aft/charter_only/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 2.980100154876709, "learning_rate": 0.0, "loss": 0.27676844596862793, "memory/device_reserved (GiB)": 39.37, "memory/max_active (GiB)": 37.93, "memory/max_allocated (GiB)": 37.93, "ppl": 1.31886, "step": 1, "tokens/total": 34528, "tokens/train_per_sec_per_gpu": 22.11, "tokens/trainable": 441} +{"epoch": 0.0078125, "grad_norm": 2.5414235591888428, "learning_rate": 4.000000000000001e-06, "loss": 0.29306650161743164, "memory/device_reserved (GiB)": 39.61, "memory/max_active (GiB)": 37.45, "memory/max_allocated (GiB)": 37.45, "ppl": 1.34053, "step": 2, "tokens/total": 67840, "tokens/train_per_sec_per_gpu": 28.84, "tokens/trainable": 888} +{"epoch": 0.01171875, "grad_norm": 4.025216102600098, "learning_rate": 8.000000000000001e-06, "loss": 0.3255169987678528, "memory/device_reserved (GiB)": 39.61, "memory/max_active (GiB)": 36.31, "memory/max_allocated (GiB)": 36.31, "ppl": 1.38475, "step": 3, "tokens/total": 99376, "tokens/train_per_sec_per_gpu": 29.12, "tokens/trainable": 1320} +{"epoch": 0.015625, "grad_norm": 4.0735931396484375, "learning_rate": 1.2e-05, "loss": 0.30792292952537537, "memory/device_reserved (GiB)": 41.7, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.3606, "step": 4, "tokens/total": 135248, "tokens/train_per_sec_per_gpu": 24.51, "tokens/trainable": 1745} +{"epoch": 0.01953125, "grad_norm": 2.2760064601898193, "learning_rate": 1.6000000000000003e-05, "loss": 0.28109484910964966, "memory/device_reserved (GiB)": 41.7, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.32458, "step": 5, "tokens/total": 171888, "tokens/train_per_sec_per_gpu": 26.28, "tokens/trainable": 2216} +{"epoch": 0.0234375, "grad_norm": 1.6878743171691895, "learning_rate": 2e-05, "loss": 0.2263227254152298, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.11, "memory/max_allocated (GiB)": 39.11, "ppl": 1.25398, "step": 6, "tokens/total": 209840, "tokens/train_per_sec_per_gpu": 25.73, "tokens/trainable": 2680} +{"epoch": 0.02734375, "grad_norm": 0.9340298175811768, "learning_rate": 2.4e-05, "loss": 0.18018010258674622, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.19743, "step": 7, "tokens/total": 246720, "tokens/train_per_sec_per_gpu": 26.3, "tokens/trainable": 3154} +{"epoch": 0.03125, "grad_norm": 3.638653039932251, "learning_rate": 2.8000000000000003e-05, "loss": 0.1787259578704834, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.19569, "step": 8, "tokens/total": 284752, "tokens/train_per_sec_per_gpu": 21.28, "tokens/trainable": 3575} +{"epoch": 0.03515625, "grad_norm": 1.681157112121582, "learning_rate": 3.2000000000000005e-05, "loss": 0.16889819502830505, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.184, "step": 9, "tokens/total": 318432, "tokens/train_per_sec_per_gpu": 28.27, "tokens/trainable": 4035} +{"epoch": 0.0390625, "grad_norm": 1.4991449117660522, "learning_rate": 3.6e-05, "loss": 0.15812714397907257, "memory/device_reserved (GiB)": 39.31, "memory/max_active (GiB)": 37.84, "memory/max_allocated (GiB)": 37.84, "ppl": 1.17132, "step": 10, "tokens/total": 351904, "tokens/train_per_sec_per_gpu": 29.74, "tokens/trainable": 4509} +{"epoch": 0.04296875, "grad_norm": 1.0936474800109863, "learning_rate": 4e-05, "loss": 0.12810400128364563, "memory/device_reserved (GiB)": 39.31, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.13667, "step": 11, "tokens/total": 385456, "tokens/train_per_sec_per_gpu": 26.65, "tokens/trainable": 4950} +{"epoch": 0.046875, "grad_norm": 1.3327715396881104, "learning_rate": 4.4000000000000006e-05, "loss": 0.13977330923080444, "memory/device_reserved (GiB)": 40.89, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.15001, "step": 12, "tokens/total": 422736, "tokens/train_per_sec_per_gpu": 23.32, "tokens/trainable": 5374} +{"epoch": 0.05078125, "grad_norm": 3.4136950969696045, "learning_rate": 4.8e-05, "loss": 0.13349536061286926, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.99, "memory/max_allocated (GiB)": 38.99, "ppl": 1.14282, "step": 13, "tokens/total": 459952, "tokens/train_per_sec_per_gpu": 26.01, "tokens/trainable": 5840} +{"epoch": 0.0546875, "grad_norm": 4.126588344573975, "learning_rate": 5.2000000000000004e-05, "loss": 0.10714510828256607, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.1131, "step": 14, "tokens/total": 493504, "tokens/train_per_sec_per_gpu": 24.28, "tokens/trainable": 6283} +{"epoch": 0.05859375, "grad_norm": 1.9925520420074463, "learning_rate": 5.6000000000000006e-05, "loss": 0.13676676154136658, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.14656, "step": 15, "tokens/total": 524672, "tokens/train_per_sec_per_gpu": 29.71, "tokens/trainable": 6732} +{"epoch": 0.0625, "grad_norm": 1.7234587669372559, "learning_rate": 6e-05, "loss": 0.11493952572345734, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.36, "memory/max_allocated (GiB)": 36.36, "ppl": 1.12181, "step": 16, "tokens/total": 556624, "tokens/train_per_sec_per_gpu": 27.02, "tokens/trainable": 7189} +{"epoch": 0.06640625, "grad_norm": 1.7245057821273804, "learning_rate": 6.400000000000001e-05, "loss": 0.11450543254613876, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 38.22, "memory/max_allocated (GiB)": 38.22, "ppl": 1.12132, "step": 17, "tokens/total": 590768, "tokens/train_per_sec_per_gpu": 23.99, "tokens/trainable": 7629} +{"epoch": 0.0703125, "grad_norm": 1.088205099105835, "learning_rate": 6.800000000000001e-05, "loss": 0.10991136729717255, "memory/device_reserved (GiB)": 41.55, "memory/max_active (GiB)": 39.28, "memory/max_allocated (GiB)": 39.28, "ppl": 1.11618, "step": 18, "tokens/total": 627472, "tokens/train_per_sec_per_gpu": 25.19, "tokens/trainable": 8083} +{"epoch": 0.07421875, "grad_norm": 24.350662231445312, "learning_rate": 7.2e-05, "loss": 0.12465807050466537, "memory/device_reserved (GiB)": 41.55, "memory/max_active (GiB)": 38.53, "memory/max_allocated (GiB)": 38.53, "ppl": 1.13276, "step": 19, "tokens/total": 665536, "tokens/train_per_sec_per_gpu": 21.98, "tokens/trainable": 8535} +{"epoch": 0.078125, "grad_norm": 1.068911075592041, "learning_rate": 7.6e-05, "loss": 0.11905729025602341, "memory/device_reserved (GiB)": 41.55, "memory/max_active (GiB)": 38.33, "memory/max_allocated (GiB)": 38.33, "ppl": 1.12643, "step": 20, "tokens/total": 700608, "tokens/train_per_sec_per_gpu": 26.13, "tokens/trainable": 9009} +{"epoch": 0.08203125, "grad_norm": 1.6663763523101807, "learning_rate": 8e-05, "loss": 0.10451961308717728, "memory/device_reserved (GiB)": 41.55, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.11018, "step": 21, "tokens/total": 736528, "tokens/train_per_sec_per_gpu": 23.81, "tokens/trainable": 9459} +{"epoch": 0.0859375, "grad_norm": 1.039940595626831, "learning_rate": 8.4e-05, "loss": 0.1164875477552414, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 35.63, "memory/max_allocated (GiB)": 35.63, "ppl": 1.12354, "step": 22, "tokens/total": 765728, "tokens/train_per_sec_per_gpu": 29.21, "tokens/trainable": 9891} +{"epoch": 0.08984375, "grad_norm": 0.9121302962303162, "learning_rate": 8.800000000000001e-05, "loss": 0.11365854740142822, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 37.95, "memory/max_allocated (GiB)": 37.95, "ppl": 1.12037, "step": 23, "tokens/total": 801712, "tokens/train_per_sec_per_gpu": 28.97, "tokens/trainable": 10381} +{"epoch": 0.09375, "grad_norm": 1.0845375061035156, "learning_rate": 9.200000000000001e-05, "loss": 0.12848418951034546, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 1.1371, "step": 24, "tokens/total": 836320, "tokens/train_per_sec_per_gpu": 28.93, "tokens/trainable": 10860} +{"epoch": 0.09765625, "grad_norm": 0.9341474771499634, "learning_rate": 9.6e-05, "loss": 0.09378055483102798, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.09832, "step": 25, "tokens/total": 873744, "tokens/train_per_sec_per_gpu": 25.01, "tokens/trainable": 11343} +{"epoch": 0.1015625, "grad_norm": 3.0753445625305176, "learning_rate": 0.0001, "loss": 0.11197395622730255, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 35.87, "memory/max_allocated (GiB)": 35.87, "ppl": 1.11848, "step": 26, "tokens/total": 904560, "tokens/train_per_sec_per_gpu": 29.62, "tokens/trainable": 11780} +{"epoch": 0.10546875, "grad_norm": 1.2567274570465088, "learning_rate": 9.99990636831587e-05, "loss": 0.11243437230587006, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.119, "step": 27, "tokens/total": 941808, "tokens/train_per_sec_per_gpu": 26.44, "tokens/trainable": 12231} +{"epoch": 0.109375, "grad_norm": 3.005981922149658, "learning_rate": 9.999625477159879e-05, "loss": 0.11974035948514938, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.1272, "step": 28, "tokens/total": 980224, "tokens/train_per_sec_per_gpu": 26.29, "tokens/trainable": 12704} +{"epoch": 0.11328125, "grad_norm": 1.419750452041626, "learning_rate": 9.999157338221051e-05, "loss": 0.09268595278263092, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.09712, "step": 29, "tokens/total": 1015776, "tokens/train_per_sec_per_gpu": 24.71, "tokens/trainable": 13120} +{"epoch": 0.1171875, "grad_norm": 0.9713729619979858, "learning_rate": 9.998501970980562e-05, "loss": 0.08206108212471008, "memory/device_reserved (GiB)": 41.81, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.08552, "step": 30, "tokens/total": 1052512, "tokens/train_per_sec_per_gpu": 24.43, "tokens/trainable": 13594} +{"epoch": 0.12109375, "grad_norm": 0.8326327800750732, "learning_rate": 9.997659402710915e-05, "loss": 0.11354623734951019, "memory/device_reserved (GiB)": 41.81, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.12024, "step": 31, "tokens/total": 1085600, "tokens/train_per_sec_per_gpu": 27.08, "tokens/trainable": 14025} +{"epoch": 0.125, "grad_norm": 1.4923101663589478, "learning_rate": 9.996629668474818e-05, "loss": 0.08436658978462219, "memory/device_reserved (GiB)": 41.81, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.08803, "step": 32, "tokens/total": 1120400, "tokens/train_per_sec_per_gpu": 25.19, "tokens/trainable": 14472} +{"epoch": 0.12890625, "grad_norm": 0.7809491753578186, "learning_rate": 9.995412811123711e-05, "loss": 0.0894509106874466, "memory/device_reserved (GiB)": 41.81, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.09357, "step": 33, "tokens/total": 1156576, "tokens/train_per_sec_per_gpu": 25.79, "tokens/trainable": 14972} +{"epoch": 0.1328125, "grad_norm": 2.0750486850738525, "learning_rate": 9.994008881295999e-05, "loss": 0.1240074634552002, "memory/device_reserved (GiB)": 40.8, "memory/max_active (GiB)": 39.11, "memory/max_allocated (GiB)": 39.11, "ppl": 1.13202, "step": 34, "tokens/total": 1192544, "tokens/train_per_sec_per_gpu": 23.01, "tokens/trainable": 15418} +{"epoch": 0.13671875, "grad_norm": 1.2374773025512695, "learning_rate": 9.992417937414932e-05, "loss": 0.09558546543121338, "memory/device_reserved (GiB)": 40.8, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.1003, "step": 35, "tokens/total": 1227152, "tokens/train_per_sec_per_gpu": 26.24, "tokens/trainable": 15891} +{"epoch": 0.140625, "grad_norm": 1.0564308166503906, "learning_rate": 9.99064004568618e-05, "loss": 0.08053452521562576, "memory/device_reserved (GiB)": 40.8, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.08387, "step": 36, "tokens/total": 1266880, "tokens/train_per_sec_per_gpu": 23.27, "tokens/trainable": 16371} +{"epoch": 0.14453125, "grad_norm": 0.6658899784088135, "learning_rate": 9.988675280095074e-05, "loss": 0.10500035434961319, "memory/device_reserved (GiB)": 40.8, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.11071, "step": 37, "tokens/total": 1302400, "tokens/train_per_sec_per_gpu": 27.83, "tokens/trainable": 16821} +{"epoch": 0.1484375, "grad_norm": 0.9999417662620544, "learning_rate": 9.986523722403528e-05, "loss": 0.07875560969114304, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 38.86, "memory/max_allocated (GiB)": 38.86, "ppl": 1.08194, "step": 38, "tokens/total": 1339152, "tokens/train_per_sec_per_gpu": 25.94, "tokens/trainable": 17282} +{"epoch": 0.15234375, "grad_norm": 1.1223256587982178, "learning_rate": 9.984185462146642e-05, "loss": 0.09109838306903839, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.09538, "step": 39, "tokens/total": 1379312, "tokens/train_per_sec_per_gpu": 22.27, "tokens/trainable": 17767} +{"epoch": 0.15625, "grad_norm": 1.6278645992279053, "learning_rate": 9.98166059662897e-05, "loss": 0.10483706742525101, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.11053, "step": 40, "tokens/total": 1412112, "tokens/train_per_sec_per_gpu": 26.85, "tokens/trainable": 18210} +{"epoch": 0.16015625, "grad_norm": 0.5907705426216125, "learning_rate": 9.978949230920472e-05, "loss": 0.0666494145989418, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 37.2, "memory/max_allocated (GiB)": 37.2, "ppl": 1.06892, "step": 41, "tokens/total": 1446656, "tokens/train_per_sec_per_gpu": 24.99, "tokens/trainable": 18690} +{"epoch": 0.1640625, "grad_norm": 1.8898913860321045, "learning_rate": 9.976051477852141e-05, "loss": 0.10716084390878677, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.11311, "step": 42, "tokens/total": 1481344, "tokens/train_per_sec_per_gpu": 23.38, "tokens/trainable": 19088} +{"epoch": 0.16796875, "grad_norm": 1.0537601709365845, "learning_rate": 9.972967458011312e-05, "loss": 0.08189041167497635, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.08534, "step": 43, "tokens/total": 1515360, "tokens/train_per_sec_per_gpu": 24.82, "tokens/trainable": 19565} +{"epoch": 0.171875, "grad_norm": 1.3358564376831055, "learning_rate": 9.96969729973664e-05, "loss": 0.08643143624067307, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.09028, "step": 44, "tokens/total": 1548496, "tokens/train_per_sec_per_gpu": 24.75, "tokens/trainable": 20020} +{"epoch": 0.17578125, "grad_norm": 0.6578068137168884, "learning_rate": 9.966241139112754e-05, "loss": 0.10183162987232208, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 37.23, "memory/max_allocated (GiB)": 37.23, "ppl": 1.1072, "step": 45, "tokens/total": 1582160, "tokens/train_per_sec_per_gpu": 25.23, "tokens/trainable": 20444} +{"epoch": 0.1796875, "grad_norm": 0.6940529346466064, "learning_rate": 9.96259911996461e-05, "loss": 0.09350817650556564, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.09802, "step": 46, "tokens/total": 1618800, "tokens/train_per_sec_per_gpu": 21.73, "tokens/trainable": 20899} +{"epoch": 0.18359375, "grad_norm": 0.6348091959953308, "learning_rate": 9.958771393851491e-05, "loss": 0.07388727366924286, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 39.08, "memory/max_allocated (GiB)": 39.08, "ppl": 1.07669, "step": 47, "tokens/total": 1656064, "tokens/train_per_sec_per_gpu": 24.61, "tokens/trainable": 21354} +{"epoch": 0.1875, "grad_norm": 1.327824354171753, "learning_rate": 9.954758120060702e-05, "loss": 0.1018071323633194, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.10717, "step": 48, "tokens/total": 1688720, "tokens/train_per_sec_per_gpu": 28.07, "tokens/trainable": 21794} +{"epoch": 0.19140625, "grad_norm": 1.5321944952011108, "learning_rate": 9.950559465600948e-05, "loss": 0.11281289905309677, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 1.11942, "step": 49, "tokens/total": 1726256, "tokens/train_per_sec_per_gpu": 22.8, "tokens/trainable": 22268} +{"epoch": 0.1953125, "grad_norm": 0.772919774055481, "learning_rate": 9.946175605195379e-05, "loss": 0.07466106116771698, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.07752, "step": 50, "tokens/total": 1760976, "tokens/train_per_sec_per_gpu": 28.65, "tokens/trainable": 22747} +{"epoch": 0.19921875, "grad_norm": 0.8031619787216187, "learning_rate": 9.941606721274322e-05, "loss": 0.10619574785232544, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 38.03, "memory/max_allocated (GiB)": 38.03, "ppl": 1.11204, "step": 51, "tokens/total": 1796480, "tokens/train_per_sec_per_gpu": 27.03, "tokens/trainable": 23246} +{"epoch": 0.203125, "grad_norm": 0.6173981428146362, "learning_rate": 9.936853003967685e-05, "loss": 0.08418802917003632, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.08783, "step": 52, "tokens/total": 1834336, "tokens/train_per_sec_per_gpu": 23.61, "tokens/trainable": 23725} +{"epoch": 0.20703125, "grad_norm": 0.5316374897956848, "learning_rate": 9.93191465109705e-05, "loss": 0.09254859387874603, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 38.67, "memory/max_allocated (GiB)": 38.67, "ppl": 1.09697, "step": 53, "tokens/total": 1872784, "tokens/train_per_sec_per_gpu": 20.83, "tokens/trainable": 24152} +{"epoch": 0.2109375, "grad_norm": 1.0689376592636108, "learning_rate": 9.926791868167438e-05, "loss": 0.08445343375205994, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.08812, "step": 54, "tokens/total": 1912336, "tokens/train_per_sec_per_gpu": 23.6, "tokens/trainable": 24603} +{"epoch": 0.21484375, "grad_norm": 0.7850605249404907, "learning_rate": 9.921484868358753e-05, "loss": 0.0783887654542923, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.08154, "step": 55, "tokens/total": 1949328, "tokens/train_per_sec_per_gpu": 22.54, "tokens/trainable": 25026} +{"epoch": 0.21875, "grad_norm": 0.7252410650253296, "learning_rate": 9.915993872516924e-05, "loss": 0.08403152227401733, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.08766, "step": 56, "tokens/total": 1985792, "tokens/train_per_sec_per_gpu": 27.32, "tokens/trainable": 25495} +{"epoch": 0.22265625, "grad_norm": 1.338294506072998, "learning_rate": 9.9103191091447e-05, "loss": 0.09128618985414505, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.09558, "step": 57, "tokens/total": 2018976, "tokens/train_per_sec_per_gpu": 27.58, "tokens/trainable": 25968} +{"epoch": 0.2265625, "grad_norm": 0.960229218006134, "learning_rate": 9.904460814392147e-05, "loss": 0.09346309304237366, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.09797, "step": 58, "tokens/total": 2050528, "tokens/train_per_sec_per_gpu": 30.33, "tokens/trainable": 26418} +{"epoch": 0.23046875, "grad_norm": 2.6952028274536133, "learning_rate": 9.898419232046825e-05, "loss": 0.0924706906080246, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.09688, "step": 59, "tokens/total": 2085392, "tokens/train_per_sec_per_gpu": 25.33, "tokens/trainable": 26866} +{"epoch": 0.234375, "grad_norm": 0.7706748843193054, "learning_rate": 9.892194613523633e-05, "loss": 0.10111326724290848, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 36.3, "memory/max_allocated (GiB)": 36.3, "ppl": 1.1064, "step": 60, "tokens/total": 2117280, "tokens/train_per_sec_per_gpu": 28.76, "tokens/trainable": 27332} +{"epoch": 0.23828125, "grad_norm": 1.1987977027893066, "learning_rate": 9.885787217854357e-05, "loss": 0.10143133252859116, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 1.10675, "step": 61, "tokens/total": 2150848, "tokens/train_per_sec_per_gpu": 23.65, "tokens/trainable": 27734} +{"epoch": 0.2421875, "grad_norm": 0.6628657579421997, "learning_rate": 9.879197311676887e-05, "loss": 0.08854737877845764, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 36.77, "memory/max_allocated (GiB)": 36.77, "ppl": 1.09259, "step": 62, "tokens/total": 2184160, "tokens/train_per_sec_per_gpu": 28.39, "tokens/trainable": 28217} +{"epoch": 0.24609375, "grad_norm": 0.6416044235229492, "learning_rate": 9.872425169224113e-05, "loss": 0.08702549338340759, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.09092, "step": 63, "tokens/total": 2217872, "tokens/train_per_sec_per_gpu": 27.69, "tokens/trainable": 28696} +{"epoch": 0.25, "grad_norm": 0.5971394181251526, "learning_rate": 9.865471072312528e-05, "loss": 0.075803741812706, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 1.07875, "step": 64, "tokens/total": 2252448, "tokens/train_per_sec_per_gpu": 27.37, "tokens/trainable": 29144} +{"epoch": 0.25390625, "grad_norm": 1.9041352272033691, "learning_rate": 9.858335310330492e-05, "loss": 0.09334564208984375, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.09784, "step": 65, "tokens/total": 2285168, "tokens/train_per_sec_per_gpu": 25.73, "tokens/trainable": 29569} +{"epoch": 0.2578125, "grad_norm": 0.6819164752960205, "learning_rate": 9.851018180226185e-05, "loss": 0.07847362011671066, "memory/device_reserved (GiB)": 39.25, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.08163, "step": 66, "tokens/total": 2317968, "tokens/train_per_sec_per_gpu": 27.94, "tokens/trainable": 30010} +{"epoch": 0.26171875, "grad_norm": 1.6301140785217285, "learning_rate": 9.843519986495259e-05, "loss": 0.060485467314720154, "memory/device_reserved (GiB)": 41.03, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.06235, "step": 67, "tokens/total": 2355248, "tokens/train_per_sec_per_gpu": 25.26, "tokens/trainable": 30466} +{"epoch": 0.265625, "grad_norm": 1.860235333442688, "learning_rate": 9.835841041168162e-05, "loss": 0.05784577876329422, "memory/device_reserved (GiB)": 41.03, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.05955, "step": 68, "tokens/total": 2389264, "tokens/train_per_sec_per_gpu": 28.19, "tokens/trainable": 30943} +{"epoch": 0.26953125, "grad_norm": 1.3820191621780396, "learning_rate": 9.82798166379715e-05, "loss": 0.09192012995481491, "memory/device_reserved (GiB)": 41.03, "memory/max_active (GiB)": 37.5, "memory/max_allocated (GiB)": 37.5, "ppl": 1.09628, "step": 69, "tokens/total": 2421360, "tokens/train_per_sec_per_gpu": 26.66, "tokens/trainable": 31364} +{"epoch": 0.2734375, "grad_norm": 0.957756757736206, "learning_rate": 9.819942181443002e-05, "loss": 0.07071702182292938, "memory/device_reserved (GiB)": 41.03, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.07328, "step": 70, "tokens/total": 2454256, "tokens/train_per_sec_per_gpu": 25.11, "tokens/trainable": 31782} +{"epoch": 0.27734375, "grad_norm": 2.140258312225342, "learning_rate": 9.811722928661392e-05, "loss": 0.07357959449291229, "memory/device_reserved (GiB)": 41.03, "memory/max_active (GiB)": 37.23, "memory/max_allocated (GiB)": 37.23, "ppl": 1.07635, "step": 71, "tokens/total": 2488464, "tokens/train_per_sec_per_gpu": 28.02, "tokens/trainable": 32257} +{"epoch": 0.28125, "grad_norm": 0.8933882713317871, "learning_rate": 9.803324247488975e-05, "loss": 0.08760759234428406, "memory/device_reserved (GiB)": 41.03, "memory/max_active (GiB)": 39.19, "memory/max_allocated (GiB)": 39.19, "ppl": 1.09156, "step": 72, "tokens/total": 2523696, "tokens/train_per_sec_per_gpu": 24.19, "tokens/trainable": 32707} +{"epoch": 0.28515625, "grad_norm": 1.6736607551574707, "learning_rate": 9.794746487429161e-05, "loss": 0.07657944411039352, "memory/device_reserved (GiB)": 41.04, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.07959, "step": 73, "tokens/total": 2558240, "tokens/train_per_sec_per_gpu": 31.5, "tokens/trainable": 33207} +{"epoch": 0.2890625, "grad_norm": 0.7470947504043579, "learning_rate": 9.785990005437554e-05, "loss": 0.06036313623189926, "memory/device_reserved (GiB)": 41.04, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.06222, "step": 74, "tokens/total": 2592496, "tokens/train_per_sec_per_gpu": 27.26, "tokens/trainable": 33646} +{"epoch": 0.29296875, "grad_norm": 1.8175640106201172, "learning_rate": 9.777055165907117e-05, "loss": 0.07498733699321747, "memory/device_reserved (GiB)": 41.04, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.07787, "step": 75, "tokens/total": 2626352, "tokens/train_per_sec_per_gpu": 26.32, "tokens/trainable": 34112} +{"epoch": 0.296875, "grad_norm": 1.3039642572402954, "learning_rate": 9.767942340652993e-05, "loss": 0.04270714893937111, "memory/device_reserved (GiB)": 41.04, "memory/max_active (GiB)": 38.97, "memory/max_allocated (GiB)": 38.97, "ppl": 1.04363, "step": 76, "tokens/total": 2661216, "tokens/train_per_sec_per_gpu": 25.45, "tokens/trainable": 34592} +{"epoch": 0.30078125, "grad_norm": 18.57212257385254, "learning_rate": 9.758651908897035e-05, "loss": 0.09088347852230072, "memory/device_reserved (GiB)": 41.28, "memory/max_active (GiB)": 39.55, "memory/max_allocated (GiB)": 39.55, "ppl": 1.09514, "step": 77, "tokens/total": 2696960, "tokens/train_per_sec_per_gpu": 28.05, "tokens/trainable": 35038} +{"epoch": 0.3046875, "grad_norm": 1.0022529363632202, "learning_rate": 9.749184257252033e-05, "loss": 0.0384056493639946, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.03915, "step": 78, "tokens/total": 2733376, "tokens/train_per_sec_per_gpu": 27.45, "tokens/trainable": 35529} +{"epoch": 0.30859375, "grad_norm": 2.99581241607666, "learning_rate": 9.739539779705614e-05, "loss": 0.04835119843482971, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.04954, "step": 79, "tokens/total": 2770496, "tokens/train_per_sec_per_gpu": 22.33, "tokens/trainable": 35960} +{"epoch": 0.3125, "grad_norm": 1.7888787984848022, "learning_rate": 9.729718877603861e-05, "loss": 0.038060590624809265, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.13, "memory/max_allocated (GiB)": 38.13, "ppl": 1.03879, "step": 80, "tokens/total": 2807184, "tokens/train_per_sec_per_gpu": 25.39, "tokens/trainable": 36405} +{"epoch": 0.31640625, "grad_norm": 1.3796286582946777, "learning_rate": 9.719721959634592e-05, "loss": 0.04314042627811432, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.26, "memory/max_allocated (GiB)": 38.26, "ppl": 1.04408, "step": 81, "tokens/total": 2842704, "tokens/train_per_sec_per_gpu": 25.97, "tokens/trainable": 36852} +{"epoch": 0.3203125, "grad_norm": 1.3663921356201172, "learning_rate": 9.709549441810375e-05, "loss": 0.04158800095319748, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 35.93, "memory/max_allocated (GiB)": 35.93, "ppl": 1.04246, "step": 82, "tokens/total": 2873808, "tokens/train_per_sec_per_gpu": 27.81, "tokens/trainable": 37291} +{"epoch": 0.32421875, "grad_norm": 2.6512203216552734, "learning_rate": 9.699201747451195e-05, "loss": 0.07511790096759796, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.39, "memory/max_allocated (GiB)": 37.39, "ppl": 1.07801, "step": 83, "tokens/total": 2906896, "tokens/train_per_sec_per_gpu": 29.96, "tokens/trainable": 37749} +{"epoch": 0.328125, "grad_norm": 1.0969363451004028, "learning_rate": 9.688679307166854e-05, "loss": 0.041156068444252014, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.04201, "step": 84, "tokens/total": 2942016, "tokens/train_per_sec_per_gpu": 25.67, "tokens/trainable": 38205} +{"epoch": 0.33203125, "grad_norm": 2.5147829055786133, "learning_rate": 9.677982558839042e-05, "loss": 0.05342407152056694, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.32, "memory/max_allocated (GiB)": 39.32, "ppl": 1.05488, "step": 85, "tokens/total": 2977920, "tokens/train_per_sec_per_gpu": 26.38, "tokens/trainable": 38674} +{"epoch": 0.3359375, "grad_norm": 0.8995107412338257, "learning_rate": 9.66711194760312e-05, "loss": 0.05641790106892586, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.07, "memory/max_allocated (GiB)": 37.07, "ppl": 1.05804, "step": 86, "tokens/total": 3011424, "tokens/train_per_sec_per_gpu": 26.18, "tokens/trainable": 39124} +{"epoch": 0.33984375, "grad_norm": 2.544355869293213, "learning_rate": 9.656067925829593e-05, "loss": 0.06682848930358887, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.06911, "step": 87, "tokens/total": 3043168, "tokens/train_per_sec_per_gpu": 30.45, "tokens/trainable": 39600} +{"epoch": 0.34375, "grad_norm": 0.6260614395141602, "learning_rate": 9.644850953105288e-05, "loss": 0.048301756381988525, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.04949, "step": 88, "tokens/total": 3076848, "tokens/train_per_sec_per_gpu": 27.29, "tokens/trainable": 40068} +{"epoch": 0.34765625, "grad_norm": 0.6485132575035095, "learning_rate": 9.633461496214225e-05, "loss": 0.043899983167648315, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 35.83, "memory/max_allocated (GiB)": 35.83, "ppl": 1.04488, "step": 89, "tokens/total": 3107392, "tokens/train_per_sec_per_gpu": 24.9, "tokens/trainable": 40452} +{"epoch": 0.3515625, "grad_norm": 0.7228068113327026, "learning_rate": 9.621900029118195e-05, "loss": 0.03716196119785309, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.59, "memory/max_allocated (GiB)": 37.59, "ppl": 1.03786, "step": 90, "tokens/total": 3142800, "tokens/train_per_sec_per_gpu": 26.62, "tokens/trainable": 40925} +{"epoch": 0.35546875, "grad_norm": 0.9735969305038452, "learning_rate": 9.610167032937036e-05, "loss": 0.038187261670827866, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.11, "memory/max_allocated (GiB)": 38.11, "ppl": 1.03893, "step": 91, "tokens/total": 3178416, "tokens/train_per_sec_per_gpu": 27.92, "tokens/trainable": 41374} +{"epoch": 0.359375, "grad_norm": 1.1735193729400635, "learning_rate": 9.598262995928611e-05, "loss": 0.04762127250432968, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 36.51, "memory/max_allocated (GiB)": 36.51, "ppl": 1.04877, "step": 92, "tokens/total": 3210400, "tokens/train_per_sec_per_gpu": 28.14, "tokens/trainable": 41807} +{"epoch": 0.36328125, "grad_norm": 2.0727906227111816, "learning_rate": 9.586188413468492e-05, "loss": 0.02872440591454506, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.02914, "step": 93, "tokens/total": 3244880, "tokens/train_per_sec_per_gpu": 26.2, "tokens/trainable": 42236} +{"epoch": 0.3671875, "grad_norm": 1.0624423027038574, "learning_rate": 9.57394378802934e-05, "loss": 0.06285084038972855, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.93, "memory/max_allocated (GiB)": 38.93, "ppl": 1.06487, "step": 94, "tokens/total": 3279952, "tokens/train_per_sec_per_gpu": 27.29, "tokens/trainable": 42712} +{"epoch": 0.37109375, "grad_norm": 0.9145601391792297, "learning_rate": 9.56152962916e-05, "loss": 0.03993649035692215, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.04074, "step": 95, "tokens/total": 3314688, "tokens/train_per_sec_per_gpu": 26.36, "tokens/trainable": 43213} +{"epoch": 0.375, "grad_norm": 0.7624052166938782, "learning_rate": 9.548946453464296e-05, "loss": 0.0464511513710022, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.04755, "step": 96, "tokens/total": 3350464, "tokens/train_per_sec_per_gpu": 28.75, "tokens/trainable": 43702} +{"epoch": 0.37890625, "grad_norm": 0.8745054602622986, "learning_rate": 9.53619478457953e-05, "loss": 0.03321576118469238, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.84, "memory/max_allocated (GiB)": 37.84, "ppl": 1.03377, "step": 97, "tokens/total": 3385536, "tokens/train_per_sec_per_gpu": 25.55, "tokens/trainable": 44158} +{"epoch": 0.3828125, "grad_norm": 1.614047646522522, "learning_rate": 9.523275153154695e-05, "loss": 0.07314953207969666, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.07589, "step": 98, "tokens/total": 3419584, "tokens/train_per_sec_per_gpu": 26.88, "tokens/trainable": 44615} +{"epoch": 0.38671875, "grad_norm": 0.5447906851768494, "learning_rate": 9.51018809682839e-05, "loss": 0.02034875936806202, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 36.29, "memory/max_allocated (GiB)": 36.29, "ppl": 1.02056, "step": 99, "tokens/total": 3448576, "tokens/train_per_sec_per_gpu": 26.54, "tokens/trainable": 45042} +{"epoch": 0.390625, "grad_norm": 0.916956901550293, "learning_rate": 9.49693416020645e-05, "loss": 0.044863708317279816, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 1.04589, "step": 100, "tokens/total": 3485328, "tokens/train_per_sec_per_gpu": 24.71, "tokens/trainable": 45508} +{"epoch": 0.39453125, "grad_norm": 1.2195911407470703, "learning_rate": 9.483513894839276e-05, "loss": 0.06492488086223602, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.59, "memory/max_allocated (GiB)": 37.59, "ppl": 1.06708, "step": 101, "tokens/total": 3519984, "tokens/train_per_sec_per_gpu": 28.08, "tokens/trainable": 45989} +{"epoch": 0.3984375, "grad_norm": 0.4998471438884735, "learning_rate": 9.469927859198888e-05, "loss": 0.033941883593797684, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.03452, "step": 102, "tokens/total": 3553952, "tokens/train_per_sec_per_gpu": 27.93, "tokens/trainable": 46425} +{"epoch": 0.40234375, "grad_norm": 0.37585386633872986, "learning_rate": 9.456176618655689e-05, "loss": 0.018013780936598778, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.01818, "step": 103, "tokens/total": 3591072, "tokens/train_per_sec_per_gpu": 26.19, "tokens/trainable": 46876} +{"epoch": 0.40625, "grad_norm": 0.48354700207710266, "learning_rate": 9.442260745454927e-05, "loss": 0.029618054628372192, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.7, "memory/max_allocated (GiB)": 38.7, "ppl": 1.03006, "step": 104, "tokens/total": 3627760, "tokens/train_per_sec_per_gpu": 27.27, "tokens/trainable": 47378} +{"epoch": 0.41015625, "grad_norm": 0.7752214670181274, "learning_rate": 9.428180818692884e-05, "loss": 0.030287234112620354, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 36.95, "memory/max_allocated (GiB)": 36.95, "ppl": 1.03075, "step": 105, "tokens/total": 3661552, "tokens/train_per_sec_per_gpu": 27.94, "tokens/trainable": 47852} +{"epoch": 0.4140625, "grad_norm": 0.6501378417015076, "learning_rate": 9.413937424292791e-05, "loss": 0.04021282121539116, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 36.85, "memory/max_allocated (GiB)": 36.85, "ppl": 1.04103, "step": 106, "tokens/total": 3694464, "tokens/train_per_sec_per_gpu": 27.76, "tokens/trainable": 48325} +{"epoch": 0.41796875, "grad_norm": 0.7044700980186462, "learning_rate": 9.399531154980424e-05, "loss": 0.0280003622174263, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.0284, "step": 107, "tokens/total": 3730768, "tokens/train_per_sec_per_gpu": 27.79, "tokens/trainable": 48818} +{"epoch": 0.421875, "grad_norm": 1.0150551795959473, "learning_rate": 9.384962610259455e-05, "loss": 0.051024094223976135, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.32, "memory/max_allocated (GiB)": 38.32, "ppl": 1.05235, "step": 108, "tokens/total": 3766288, "tokens/train_per_sec_per_gpu": 23.84, "tokens/trainable": 49256} +{"epoch": 0.42578125, "grad_norm": 0.8324607610702515, "learning_rate": 9.370232396386494e-05, "loss": 0.030417954549193382, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.03089, "step": 109, "tokens/total": 3802688, "tokens/train_per_sec_per_gpu": 24.96, "tokens/trainable": 49701} +{"epoch": 0.4296875, "grad_norm": 0.7097928524017334, "learning_rate": 9.355341126345868e-05, "loss": 0.01308748871088028, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.01317, "step": 110, "tokens/total": 3839936, "tokens/train_per_sec_per_gpu": 23.49, "tokens/trainable": 50192} +{"epoch": 0.43359375, "grad_norm": 1.2375342845916748, "learning_rate": 9.340289419824107e-05, "loss": 0.016634706407785416, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.37, "memory/max_allocated (GiB)": 38.37, "ppl": 1.01677, "step": 111, "tokens/total": 3875056, "tokens/train_per_sec_per_gpu": 24.74, "tokens/trainable": 50608} +{"epoch": 0.4375, "grad_norm": 0.5124939680099487, "learning_rate": 9.325077903184159e-05, "loss": 0.018133562058210373, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.3, "memory/max_allocated (GiB)": 38.3, "ppl": 1.0183, "step": 112, "tokens/total": 3908592, "tokens/train_per_sec_per_gpu": 25.52, "tokens/trainable": 51032} +{"epoch": 0.44140625, "grad_norm": 0.5607399344444275, "learning_rate": 9.30970720943932e-05, "loss": 0.019595839083194733, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.01979, "step": 113, "tokens/total": 3943744, "tokens/train_per_sec_per_gpu": 27.62, "tokens/trainable": 51537} +{"epoch": 0.4453125, "grad_norm": 1.6274949312210083, "learning_rate": 9.2941779782269e-05, "loss": 0.04598010331392288, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 36.18, "memory/max_allocated (GiB)": 36.18, "ppl": 1.04705, "step": 114, "tokens/total": 3975440, "tokens/train_per_sec_per_gpu": 29.54, "tokens/trainable": 52004} +{"epoch": 0.44921875, "grad_norm": 0.48638448119163513, "learning_rate": 9.278490855781596e-05, "loss": 0.016903167590498924, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.01705, "step": 115, "tokens/total": 4011024, "tokens/train_per_sec_per_gpu": 27.14, "tokens/trainable": 52487} +{"epoch": 0.453125, "grad_norm": 0.6155802607536316, "learning_rate": 9.262646494908604e-05, "loss": 0.021066971123218536, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.02129, "step": 116, "tokens/total": 4048976, "tokens/train_per_sec_per_gpu": 25.91, "tokens/trainable": 52965} +{"epoch": 0.45703125, "grad_norm": 0.4769619405269623, "learning_rate": 9.246645554956457e-05, "loss": 0.02429286390542984, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.02459, "step": 117, "tokens/total": 4083376, "tokens/train_per_sec_per_gpu": 26.77, "tokens/trainable": 53450} +{"epoch": 0.4609375, "grad_norm": 0.7610743045806885, "learning_rate": 9.230488701789578e-05, "loss": 0.02592572756111622, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.02626, "step": 118, "tokens/total": 4118576, "tokens/train_per_sec_per_gpu": 26.54, "tokens/trainable": 53927} +{"epoch": 0.46484375, "grad_norm": 0.9207265973091125, "learning_rate": 9.214176607760577e-05, "loss": 0.01317079458385706, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.29, "memory/max_allocated (GiB)": 38.29, "ppl": 1.01326, "step": 119, "tokens/total": 4154480, "tokens/train_per_sec_per_gpu": 26.12, "tokens/trainable": 54395} +{"epoch": 0.46875, "grad_norm": 0.5300065875053406, "learning_rate": 9.197709951682268e-05, "loss": 0.020924467593431473, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.02114, "step": 120, "tokens/total": 4190096, "tokens/train_per_sec_per_gpu": 27.66, "tokens/trainable": 54871} +{"epoch": 0.47265625, "grad_norm": 0.44787558913230896, "learning_rate": 9.181089418799428e-05, "loss": 0.021379556506872177, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 39.32, "memory/max_allocated (GiB)": 39.32, "ppl": 1.02161, "step": 121, "tokens/total": 4229312, "tokens/train_per_sec_per_gpu": 23.25, "tokens/trainable": 55356} +{"epoch": 0.4765625, "grad_norm": 0.3713775873184204, "learning_rate": 9.164315700760271e-05, "loss": 0.009318981319665909, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00936, "step": 122, "tokens/total": 4264864, "tokens/train_per_sec_per_gpu": 25.61, "tokens/trainable": 55836} +{"epoch": 0.48046875, "grad_norm": 0.6173804998397827, "learning_rate": 9.147389495587671e-05, "loss": 0.012998539954423904, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 36.18, "memory/max_allocated (GiB)": 36.18, "ppl": 1.01308, "step": 123, "tokens/total": 4295360, "tokens/train_per_sec_per_gpu": 29.59, "tokens/trainable": 56275} +{"epoch": 0.484375, "grad_norm": 0.40090930461883545, "learning_rate": 9.130311507650116e-05, "loss": 0.010735011659562588, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 36.98, "memory/max_allocated (GiB)": 36.98, "ppl": 1.01079, "step": 124, "tokens/total": 4328336, "tokens/train_per_sec_per_gpu": 26.61, "tokens/trainable": 56745} +{"epoch": 0.48828125, "grad_norm": 1.2116583585739136, "learning_rate": 9.113082447632394e-05, "loss": 0.02096879482269287, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 38.49, "memory/max_allocated (GiB)": 38.49, "ppl": 1.02119, "step": 125, "tokens/total": 4365552, "tokens/train_per_sec_per_gpu": 22.37, "tokens/trainable": 57171} +{"epoch": 0.4921875, "grad_norm": 0.3453673720359802, "learning_rate": 9.09570303250602e-05, "loss": 0.0065353927202522755, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.00656, "step": 126, "tokens/total": 4399376, "tokens/train_per_sec_per_gpu": 27.61, "tokens/trainable": 57642} +{"epoch": 0.49609375, "grad_norm": 0.5085861682891846, "learning_rate": 9.078173985499394e-05, "loss": 0.00814670231193304, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00818, "step": 127, "tokens/total": 4434880, "tokens/train_per_sec_per_gpu": 29.14, "tokens/trainable": 58134} +{"epoch": 0.5, "grad_norm": 0.22416752576828003, "learning_rate": 9.060496036067713e-05, "loss": 0.0024825911968946457, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.00249, "step": 128, "tokens/total": 4469280, "tokens/train_per_sec_per_gpu": 23.51, "tokens/trainable": 58564} +{"epoch": 0.50390625, "grad_norm": 1.0285160541534424, "learning_rate": 9.042669919862615e-05, "loss": 0.02390393614768982, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.02419, "step": 129, "tokens/total": 4503408, "tokens/train_per_sec_per_gpu": 27.58, "tokens/trainable": 59038} +{"epoch": 0.5078125, "grad_norm": 0.5219807624816895, "learning_rate": 9.024696378701557e-05, "loss": 0.011786036193370819, "memory/device_reserved (GiB)": 39.68, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.01186, "step": 130, "tokens/total": 4536976, "tokens/train_per_sec_per_gpu": 28.84, "tokens/trainable": 59495} +{"epoch": 0.51171875, "grad_norm": 1.0169910192489624, "learning_rate": 9.006576160536948e-05, "loss": 0.011715398170053959, "memory/device_reserved (GiB)": 39.68, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.01178, "step": 131, "tokens/total": 4570304, "tokens/train_per_sec_per_gpu": 25.4, "tokens/trainable": 59911} +{"epoch": 0.515625, "grad_norm": 0.9078590273857117, "learning_rate": 8.988310019425035e-05, "loss": 0.026078160852193832, "memory/device_reserved (GiB)": 39.68, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.02642, "step": 132, "tokens/total": 4602912, "tokens/train_per_sec_per_gpu": 29.39, "tokens/trainable": 60371} +{"epoch": 0.51953125, "grad_norm": 0.5801230072975159, "learning_rate": 8.969898715494506e-05, "loss": 0.02669057995080948, "memory/device_reserved (GiB)": 40.09, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.02705, "step": 133, "tokens/total": 4638080, "tokens/train_per_sec_per_gpu": 26.27, "tokens/trainable": 60827} +{"epoch": 0.5234375, "grad_norm": 0.9568626284599304, "learning_rate": 8.951343014914869e-05, "loss": 0.005796263925731182, "memory/device_reserved (GiB)": 40.1, "memory/max_active (GiB)": 38.52, "memory/max_allocated (GiB)": 38.52, "ppl": 1.00581, "step": 134, "tokens/total": 4672480, "tokens/train_per_sec_per_gpu": 29.38, "tokens/trainable": 61320} +{"epoch": 0.52734375, "grad_norm": 0.758849561214447, "learning_rate": 8.932643689864568e-05, "loss": 0.02299298718571663, "memory/device_reserved (GiB)": 40.1, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.02326, "step": 135, "tokens/total": 4707264, "tokens/train_per_sec_per_gpu": 25.03, "tokens/trainable": 61775} +{"epoch": 0.53125, "grad_norm": 0.6506325006484985, "learning_rate": 8.913801518498845e-05, "loss": 0.015820497646927834, "memory/device_reserved (GiB)": 40.1, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 1.01595, "step": 136, "tokens/total": 4743696, "tokens/train_per_sec_per_gpu": 25.43, "tokens/trainable": 62248} +{"epoch": 0.53515625, "grad_norm": 2.229433059692383, "learning_rate": 8.894817284917364e-05, "loss": 0.016808651387691498, "memory/device_reserved (GiB)": 40.92, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.01695, "step": 137, "tokens/total": 4781424, "tokens/train_per_sec_per_gpu": 24.81, "tokens/trainable": 62730} +{"epoch": 0.5390625, "grad_norm": 6.9974775314331055, "learning_rate": 8.875691779131569e-05, "loss": 0.010008244775235653, "memory/device_reserved (GiB)": 40.92, "memory/max_active (GiB)": 38.99, "memory/max_allocated (GiB)": 38.99, "ppl": 1.01006, "step": 138, "tokens/total": 4820752, "tokens/train_per_sec_per_gpu": 22.44, "tokens/trainable": 63166} +{"epoch": 0.54296875, "grad_norm": 0.8900492191314697, "learning_rate": 8.856425797031829e-05, "loss": 0.02367202565073967, "memory/device_reserved (GiB)": 40.92, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.02395, "step": 139, "tokens/total": 4854688, "tokens/train_per_sec_per_gpu": 25.9, "tokens/trainable": 63631} +{"epoch": 0.546875, "grad_norm": 2.8105194568634033, "learning_rate": 8.837020140354295e-05, "loss": 0.029101785272359848, "memory/device_reserved (GiB)": 40.92, "memory/max_active (GiB)": 38.24, "memory/max_allocated (GiB)": 38.24, "ppl": 1.02953, "step": 140, "tokens/total": 4891424, "tokens/train_per_sec_per_gpu": 22.81, "tokens/trainable": 64080} +{"epoch": 0.55078125, "grad_norm": 0.36911389231681824, "learning_rate": 8.817475616647554e-05, "loss": 0.009549720212817192, "memory/device_reserved (GiB)": 40.92, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.0096, "step": 141, "tokens/total": 4924464, "tokens/train_per_sec_per_gpu": 29.48, "tokens/trainable": 64537} +{"epoch": 0.5546875, "grad_norm": 0.3871113955974579, "learning_rate": 8.797793039239017e-05, "loss": 0.013707483187317848, "memory/device_reserved (GiB)": 40.92, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.0138, "step": 142, "tokens/total": 4957072, "tokens/train_per_sec_per_gpu": 27.9, "tokens/trainable": 65011} +{"epoch": 0.55859375, "grad_norm": 0.41807320713996887, "learning_rate": 8.777973227201069e-05, "loss": 0.009756290353834629, "memory/device_reserved (GiB)": 40.92, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.0098, "step": 143, "tokens/total": 4991840, "tokens/train_per_sec_per_gpu": 28.03, "tokens/trainable": 65511} +{"epoch": 0.5625, "grad_norm": 0.5261229872703552, "learning_rate": 8.758017005316988e-05, "loss": 0.016298146918416023, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.18, "memory/max_allocated (GiB)": 39.18, "ppl": 1.01643, "step": 144, "tokens/total": 5029312, "tokens/train_per_sec_per_gpu": 24.1, "tokens/trainable": 65996} +{"epoch": 0.56640625, "grad_norm": 0.08822914212942123, "learning_rate": 8.737925204046629e-05, "loss": 0.00196670345030725, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.56, "memory/max_allocated (GiB)": 36.56, "ppl": 1.00197, "step": 145, "tokens/total": 5060800, "tokens/train_per_sec_per_gpu": 26.68, "tokens/trainable": 66423} +{"epoch": 0.5703125, "grad_norm": 1.272827386856079, "learning_rate": 8.717698659491851e-05, "loss": 0.016502590849995613, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.86, "memory/max_allocated (GiB)": 38.86, "ppl": 1.01664, "step": 146, "tokens/total": 5096448, "tokens/train_per_sec_per_gpu": 26.19, "tokens/trainable": 66905} +{"epoch": 0.57421875, "grad_norm": 1.1964820623397827, "learning_rate": 8.697338213361735e-05, "loss": 0.01805933378636837, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.01822, "step": 147, "tokens/total": 5132928, "tokens/train_per_sec_per_gpu": 26.75, "tokens/trainable": 67363} +{"epoch": 0.578125, "grad_norm": 0.5588763356208801, "learning_rate": 8.676844712937552e-05, "loss": 0.020634230226278305, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.02085, "step": 148, "tokens/total": 5167008, "tokens/train_per_sec_per_gpu": 26.57, "tokens/trainable": 67829} +{"epoch": 0.58203125, "grad_norm": 0.2972297966480255, "learning_rate": 8.656219011037509e-05, "loss": 0.007496546488255262, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.14, "memory/max_allocated (GiB)": 36.14, "ppl": 1.00752, "step": 149, "tokens/total": 5198656, "tokens/train_per_sec_per_gpu": 27.26, "tokens/trainable": 68286} +{"epoch": 0.5859375, "grad_norm": 0.9323297142982483, "learning_rate": 8.63546196598125e-05, "loss": 0.01853448525071144, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.01871, "step": 150, "tokens/total": 5235600, "tokens/train_per_sec_per_gpu": 24.35, "tokens/trainable": 68750} +{"epoch": 0.58984375, "grad_norm": 0.8800176382064819, "learning_rate": 8.614574441554145e-05, "loss": 0.03307631239295006, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.57, "memory/max_allocated (GiB)": 39.57, "ppl": 1.03363, "step": 151, "tokens/total": 5273808, "tokens/train_per_sec_per_gpu": 25.56, "tokens/trainable": 69231} +{"epoch": 0.59375, "grad_norm": 0.6046350598335266, "learning_rate": 8.593557306971349e-05, "loss": 0.015945032238960266, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.01607, "step": 152, "tokens/total": 5310480, "tokens/train_per_sec_per_gpu": 27.19, "tokens/trainable": 69711} +{"epoch": 0.59765625, "grad_norm": 0.3808906078338623, "learning_rate": 8.572411436841618e-05, "loss": 0.011908449232578278, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.01198, "step": 153, "tokens/total": 5349264, "tokens/train_per_sec_per_gpu": 23.85, "tokens/trainable": 70178} +{"epoch": 0.6015625, "grad_norm": 5.4171833992004395, "learning_rate": 8.551137711130922e-05, "loss": 0.012094688601791859, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.01217, "step": 154, "tokens/total": 5382736, "tokens/train_per_sec_per_gpu": 30.06, "tokens/trainable": 70670} +{"epoch": 0.60546875, "grad_norm": 1.2997267246246338, "learning_rate": 8.529737015125824e-05, "loss": 0.008648114278912544, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.00869, "step": 155, "tokens/total": 5420256, "tokens/train_per_sec_per_gpu": 22.3, "tokens/trainable": 71111} +{"epoch": 0.609375, "grad_norm": 0.991814136505127, "learning_rate": 8.508210239396639e-05, "loss": 0.013267851434648037, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.93, "memory/max_allocated (GiB)": 36.93, "ppl": 1.01336, "step": 156, "tokens/total": 5454080, "tokens/train_per_sec_per_gpu": 31.49, "tokens/trainable": 71623} +{"epoch": 0.61328125, "grad_norm": 1.1853982210159302, "learning_rate": 8.486558279760375e-05, "loss": 0.016327233985066414, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.01646, "step": 157, "tokens/total": 5489728, "tokens/train_per_sec_per_gpu": 27.06, "tokens/trainable": 72100} +{"epoch": 0.6171875, "grad_norm": 0.24615521728992462, "learning_rate": 8.464782037243449e-05, "loss": 0.0015350014436990023, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.00154, "step": 158, "tokens/total": 5526512, "tokens/train_per_sec_per_gpu": 25.42, "tokens/trainable": 72598} +{"epoch": 0.62109375, "grad_norm": 1.5370423793792725, "learning_rate": 8.442882418044202e-05, "loss": 0.03796675428748131, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.0387, "step": 159, "tokens/total": 5563456, "tokens/train_per_sec_per_gpu": 24.46, "tokens/trainable": 73071} +{"epoch": 0.625, "grad_norm": 1.8172136545181274, "learning_rate": 8.420860333495179e-05, "loss": 0.03656744211912155, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.22, "memory/max_allocated (GiB)": 38.22, "ppl": 1.03724, "step": 160, "tokens/total": 5599520, "tokens/train_per_sec_per_gpu": 28.65, "tokens/trainable": 73560} +{"epoch": 0.62890625, "grad_norm": 0.8356566429138184, "learning_rate": 8.398716700025208e-05, "loss": 0.018062619492411613, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.01823, "step": 161, "tokens/total": 5637264, "tokens/train_per_sec_per_gpu": 27.3, "tokens/trainable": 74083} +{"epoch": 0.6328125, "grad_norm": 1.312840461730957, "learning_rate": 8.376452439121266e-05, "loss": 0.011639468371868134, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.61, "memory/max_allocated (GiB)": 36.61, "ppl": 1.01171, "step": 162, "tokens/total": 5670064, "tokens/train_per_sec_per_gpu": 24.86, "tokens/trainable": 74530} +{"epoch": 0.63671875, "grad_norm": 0.5953813195228577, "learning_rate": 8.354068477290124e-05, "loss": 0.018981100991368294, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.01916, "step": 163, "tokens/total": 5702800, "tokens/train_per_sec_per_gpu": 27.77, "tokens/trainable": 74999} +{"epoch": 0.640625, "grad_norm": 0.30161839723587036, "learning_rate": 8.331565746019807e-05, "loss": 0.011120476759970188, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.01118, "step": 164, "tokens/total": 5739792, "tokens/train_per_sec_per_gpu": 22.73, "tokens/trainable": 75450} +{"epoch": 0.64453125, "grad_norm": 0.24760405719280243, "learning_rate": 8.308945181740812e-05, "loss": 0.004735330119729042, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 1.00475, "step": 165, "tokens/total": 5773856, "tokens/train_per_sec_per_gpu": 26.25, "tokens/trainable": 75921} +{"epoch": 0.6484375, "grad_norm": 0.6692270040512085, "learning_rate": 8.286207725787153e-05, "loss": 0.010847575962543488, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.54, "memory/max_allocated (GiB)": 39.54, "ppl": 1.01091, "step": 166, "tokens/total": 5809520, "tokens/train_per_sec_per_gpu": 28.89, "tokens/trainable": 76425} +{"epoch": 0.65234375, "grad_norm": 2.208200454711914, "learning_rate": 8.263354324357182e-05, "loss": 0.02076881006360054, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.02099, "step": 167, "tokens/total": 5841920, "tokens/train_per_sec_per_gpu": 28.26, "tokens/trainable": 76863} +{"epoch": 0.65625, "grad_norm": 0.7337036728858948, "learning_rate": 8.240385928474219e-05, "loss": 0.019774429500102997, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.07, "memory/max_allocated (GiB)": 38.07, "ppl": 1.01997, "step": 168, "tokens/total": 5874912, "tokens/train_per_sec_per_gpu": 25.94, "tokens/trainable": 77298} +{"epoch": 0.66015625, "grad_norm": 0.4397783875465393, "learning_rate": 8.217303493946967e-05, "loss": 0.012158069759607315, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.01223, "step": 169, "tokens/total": 5908768, "tokens/train_per_sec_per_gpu": 28.49, "tokens/trainable": 77735} +{"epoch": 0.6640625, "grad_norm": 0.4915182590484619, "learning_rate": 8.194107981329746e-05, "loss": 0.010651832446455956, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.78, "memory/max_allocated (GiB)": 36.78, "ppl": 1.01071, "step": 170, "tokens/total": 5942128, "tokens/train_per_sec_per_gpu": 27.48, "tokens/trainable": 78179} +{"epoch": 0.66796875, "grad_norm": 0.7488727569580078, "learning_rate": 8.170800355882518e-05, "loss": 0.01921882852911949, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.36, "memory/max_allocated (GiB)": 38.36, "ppl": 1.0194, "step": 171, "tokens/total": 5979232, "tokens/train_per_sec_per_gpu": 27.0, "tokens/trainable": 78660} +{"epoch": 0.671875, "grad_norm": 1.3317543268203735, "learning_rate": 8.147381587530713e-05, "loss": 0.011953023262321949, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.01202, "step": 172, "tokens/total": 6014000, "tokens/train_per_sec_per_gpu": 25.29, "tokens/trainable": 79110} +{"epoch": 0.67578125, "grad_norm": 0.5249667763710022, "learning_rate": 8.123852650824877e-05, "loss": 0.022757481783628464, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.02302, "step": 173, "tokens/total": 6048688, "tokens/train_per_sec_per_gpu": 27.06, "tokens/trainable": 79576} +{"epoch": 0.6796875, "grad_norm": 0.11495542526245117, "learning_rate": 8.100214524900103e-05, "loss": 0.0017280986066907644, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 35.79, "memory/max_allocated (GiB)": 35.79, "ppl": 1.00173, "step": 174, "tokens/total": 6078768, "tokens/train_per_sec_per_gpu": 25.27, "tokens/trainable": 80001} +{"epoch": 0.68359375, "grad_norm": 0.6501874923706055, "learning_rate": 8.076468193435301e-05, "loss": 0.017087597399950027, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.01723, "step": 175, "tokens/total": 6111296, "tokens/train_per_sec_per_gpu": 27.9, "tokens/trainable": 80452} +{"epoch": 0.6875, "grad_norm": 0.5638597011566162, "learning_rate": 8.052614644612253e-05, "loss": 0.0029697383288294077, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.00297, "step": 176, "tokens/total": 6146224, "tokens/train_per_sec_per_gpu": 25.97, "tokens/trainable": 80886} +{"epoch": 0.69140625, "grad_norm": 2.2037644386291504, "learning_rate": 8.028654871074489e-05, "loss": 0.019030600786209106, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.01921, "step": 177, "tokens/total": 6176576, "tokens/train_per_sec_per_gpu": 28.28, "tokens/trainable": 81325} +{"epoch": 0.6953125, "grad_norm": 0.7864559888839722, "learning_rate": 8.004589869885986e-05, "loss": 0.009214102290570736, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.2, "memory/max_allocated (GiB)": 37.2, "ppl": 1.00926, "step": 178, "tokens/total": 6208432, "tokens/train_per_sec_per_gpu": 26.05, "tokens/trainable": 81732} +{"epoch": 0.69921875, "grad_norm": 0.12195983529090881, "learning_rate": 7.980420642489674e-05, "loss": 0.0009904289618134499, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.4, "memory/max_allocated (GiB)": 36.4, "ppl": 1.00099, "step": 179, "tokens/total": 6236640, "tokens/train_per_sec_per_gpu": 34.34, "tokens/trainable": 82171} +{"epoch": 0.703125, "grad_norm": 0.4114907383918762, "learning_rate": 7.95614819466576e-05, "loss": 0.0009698302019387484, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.00097, "step": 180, "tokens/total": 6271184, "tokens/train_per_sec_per_gpu": 27.65, "tokens/trainable": 82670} +{"epoch": 0.70703125, "grad_norm": 0.5063540935516357, "learning_rate": 7.931773536489872e-05, "loss": 0.006510515231639147, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 35.89, "memory/max_allocated (GiB)": 35.89, "ppl": 1.00653, "step": 181, "tokens/total": 6301120, "tokens/train_per_sec_per_gpu": 25.51, "tokens/trainable": 83084} +{"epoch": 0.7109375, "grad_norm": 0.7214666604995728, "learning_rate": 7.907297682291035e-05, "loss": 0.011050685308873653, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.01111, "step": 182, "tokens/total": 6334192, "tokens/train_per_sec_per_gpu": 25.93, "tokens/trainable": 83500} +{"epoch": 0.71484375, "grad_norm": 1.358378529548645, "learning_rate": 7.882721650609442e-05, "loss": 0.024797897785902023, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.36, "memory/max_allocated (GiB)": 36.36, "ppl": 1.02511, "step": 183, "tokens/total": 6366192, "tokens/train_per_sec_per_gpu": 25.43, "tokens/trainable": 83896} +{"epoch": 0.71875, "grad_norm": 0.258102148771286, "learning_rate": 7.85804646415409e-05, "loss": 0.003776466939598322, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.53, "memory/max_allocated (GiB)": 38.53, "ppl": 1.00378, "step": 184, "tokens/total": 6400496, "tokens/train_per_sec_per_gpu": 28.39, "tokens/trainable": 84399} +{"epoch": 0.72265625, "grad_norm": 0.4116179049015045, "learning_rate": 7.833273149760207e-05, "loss": 0.007393221370875835, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.00742, "step": 185, "tokens/total": 6434992, "tokens/train_per_sec_per_gpu": 26.41, "tokens/trainable": 84873} +{"epoch": 0.7265625, "grad_norm": 0.40897858142852783, "learning_rate": 7.808402738346527e-05, "loss": 0.006488459650427103, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.09, "memory/max_allocated (GiB)": 37.09, "ppl": 1.00651, "step": 186, "tokens/total": 6469296, "tokens/train_per_sec_per_gpu": 25.63, "tokens/trainable": 85296} +{"epoch": 0.73046875, "grad_norm": 0.548751175403595, "learning_rate": 7.783436264872382e-05, "loss": 0.0077808694913983345, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.00781, "step": 187, "tokens/total": 6503760, "tokens/train_per_sec_per_gpu": 25.41, "tokens/trainable": 85741} +{"epoch": 0.734375, "grad_norm": 1.1045976877212524, "learning_rate": 7.758374768294647e-05, "loss": 0.01888960413634777, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.01907, "step": 188, "tokens/total": 6540448, "tokens/train_per_sec_per_gpu": 23.2, "tokens/trainable": 86176} +{"epoch": 0.73828125, "grad_norm": 0.49805840849876404, "learning_rate": 7.733219291524489e-05, "loss": 0.0077921380288898945, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.00782, "step": 189, "tokens/total": 6577648, "tokens/train_per_sec_per_gpu": 24.95, "tokens/trainable": 86649} +{"epoch": 0.7421875, "grad_norm": 0.24879080057144165, "learning_rate": 7.707970881383977e-05, "loss": 0.0035471769515424967, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.00355, "step": 190, "tokens/total": 6615600, "tokens/train_per_sec_per_gpu": 21.51, "tokens/trainable": 87084} +{"epoch": 0.74609375, "grad_norm": 1.0202614068984985, "learning_rate": 7.682630588562518e-05, "loss": 0.02325323224067688, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.02353, "step": 191, "tokens/total": 6648880, "tokens/train_per_sec_per_gpu": 24.63, "tokens/trainable": 87499} +{"epoch": 0.75, "grad_norm": 0.4736725986003876, "learning_rate": 7.657199467573129e-05, "loss": 0.005702647380530834, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.00572, "step": 192, "tokens/total": 6686784, "tokens/train_per_sec_per_gpu": 25.83, "tokens/trainable": 87987} +{"epoch": 0.75390625, "grad_norm": 0.6741048693656921, "learning_rate": 7.631678576708561e-05, "loss": 0.016745444387197495, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.01689, "step": 193, "tokens/total": 6722480, "tokens/train_per_sec_per_gpu": 26.23, "tokens/trainable": 88470} +{"epoch": 0.7578125, "grad_norm": 0.6386834979057312, "learning_rate": 7.606068977997255e-05, "loss": 0.019779089838266373, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.01998, "step": 194, "tokens/total": 6757168, "tokens/train_per_sec_per_gpu": 26.32, "tokens/trainable": 88923} +{"epoch": 0.76171875, "grad_norm": 0.7187199592590332, "learning_rate": 7.580371737159148e-05, "loss": 0.014620067551732063, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.02, "memory/max_allocated (GiB)": 37.02, "ppl": 1.01473, "step": 195, "tokens/total": 6790000, "tokens/train_per_sec_per_gpu": 27.15, "tokens/trainable": 89338} +{"epoch": 0.765625, "grad_norm": 0.8644058108329773, "learning_rate": 7.554587923561324e-05, "loss": 0.009266779758036137, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.00931, "step": 196, "tokens/total": 6826384, "tokens/train_per_sec_per_gpu": 25.91, "tokens/trainable": 89812} +{"epoch": 0.76953125, "grad_norm": 0.6354371309280396, "learning_rate": 7.528718610173511e-05, "loss": 0.010666568763554096, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.01072, "step": 197, "tokens/total": 6860896, "tokens/train_per_sec_per_gpu": 26.42, "tokens/trainable": 90252} +{"epoch": 0.7734375, "grad_norm": 0.33912089467048645, "learning_rate": 7.502764873523431e-05, "loss": 0.006162717938423157, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.28, "memory/max_allocated (GiB)": 39.28, "ppl": 1.00618, "step": 198, "tokens/total": 6896912, "tokens/train_per_sec_per_gpu": 26.29, "tokens/trainable": 90718} +{"epoch": 0.77734375, "grad_norm": 0.15585367381572723, "learning_rate": 7.476727793652011e-05, "loss": 0.0029641480650752783, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.00297, "step": 199, "tokens/total": 6928592, "tokens/train_per_sec_per_gpu": 30.01, "tokens/trainable": 91194} +{"epoch": 0.78125, "grad_norm": 0.12940603494644165, "learning_rate": 7.450608454068415e-05, "loss": 0.0018250606954097748, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.00183, "step": 200, "tokens/total": 6964672, "tokens/train_per_sec_per_gpu": 24.23, "tokens/trainable": 91649} +{"epoch": 0.78515625, "grad_norm": 0.9954752922058105, "learning_rate": 7.424407941704987e-05, "loss": 0.011408127844333649, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.01147, "step": 201, "tokens/total": 6996336, "tokens/train_per_sec_per_gpu": 29.28, "tokens/trainable": 92078} +{"epoch": 0.7890625, "grad_norm": 0.35026606917381287, "learning_rate": 7.398127346871986e-05, "loss": 0.004840285517275333, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.59, "memory/max_allocated (GiB)": 37.59, "ppl": 1.00485, "step": 202, "tokens/total": 7030544, "tokens/train_per_sec_per_gpu": 24.62, "tokens/trainable": 92536} +{"epoch": 0.79296875, "grad_norm": 1.0969569683074951, "learning_rate": 7.371767763212238e-05, "loss": 0.02114414982497692, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.02137, "step": 203, "tokens/total": 7066480, "tokens/train_per_sec_per_gpu": 27.72, "tokens/trainable": 93032} +{"epoch": 0.796875, "grad_norm": 0.2756577432155609, "learning_rate": 7.345330287655617e-05, "loss": 0.002866517286747694, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00287, "step": 204, "tokens/total": 7098768, "tokens/train_per_sec_per_gpu": 30.61, "tokens/trainable": 93504} +{"epoch": 0.80078125, "grad_norm": 1.3913660049438477, "learning_rate": 7.31881602037339e-05, "loss": 0.023829787969589233, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.02412, "step": 205, "tokens/total": 7132144, "tokens/train_per_sec_per_gpu": 23.19, "tokens/trainable": 93925} +{"epoch": 0.8046875, "grad_norm": 0.5178861021995544, "learning_rate": 7.29222606473245e-05, "loss": 0.004325319081544876, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.00433, "step": 206, "tokens/total": 7164720, "tokens/train_per_sec_per_gpu": 27.72, "tokens/trainable": 94365} +{"epoch": 0.80859375, "grad_norm": 0.6540612578392029, "learning_rate": 7.265561527249383e-05, "loss": 0.007288180757313967, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.00731, "step": 207, "tokens/total": 7199648, "tokens/train_per_sec_per_gpu": 24.75, "tokens/trainable": 94845} +{"epoch": 0.8125, "grad_norm": 0.38294780254364014, "learning_rate": 7.238823517544436e-05, "loss": 0.006832435727119446, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.00686, "step": 208, "tokens/total": 7235008, "tokens/train_per_sec_per_gpu": 26.69, "tokens/trainable": 95285} +{"epoch": 0.81640625, "grad_norm": 0.15877312421798706, "learning_rate": 7.212013148295333e-05, "loss": 0.0018213139846920967, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.00182, "step": 209, "tokens/total": 7269776, "tokens/train_per_sec_per_gpu": 24.99, "tokens/trainable": 95758} +{"epoch": 0.8203125, "grad_norm": 1.464792013168335, "learning_rate": 7.185131535190975e-05, "loss": 0.018364761024713516, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.59, "memory/max_allocated (GiB)": 39.59, "ppl": 1.01853, "step": 210, "tokens/total": 7306848, "tokens/train_per_sec_per_gpu": 23.62, "tokens/trainable": 96192} +{"epoch": 0.82421875, "grad_norm": 0.5022951364517212, "learning_rate": 7.158179796885005e-05, "loss": 0.02123369462788105, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.02146, "step": 211, "tokens/total": 7345600, "tokens/train_per_sec_per_gpu": 23.86, "tokens/trainable": 96641} +{"epoch": 0.828125, "grad_norm": 1.5760525465011597, "learning_rate": 7.131159054949273e-05, "loss": 0.03992924094200134, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.04074, "step": 212, "tokens/total": 7383040, "tokens/train_per_sec_per_gpu": 26.65, "tokens/trainable": 97160} +{"epoch": 0.83203125, "grad_norm": 0.5587036609649658, "learning_rate": 7.104070433827139e-05, "loss": 0.006888166069984436, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 36.1, "memory/max_allocated (GiB)": 36.1, "ppl": 1.00691, "step": 213, "tokens/total": 7414208, "tokens/train_per_sec_per_gpu": 28.53, "tokens/trainable": 97611} +{"epoch": 0.8359375, "grad_norm": 0.5285483002662659, "learning_rate": 7.076915060786705e-05, "loss": 0.01779029332101345, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.01795, "step": 214, "tokens/total": 7448864, "tokens/train_per_sec_per_gpu": 28.28, "tokens/trainable": 98111} +{"epoch": 0.83984375, "grad_norm": 1.9194082021713257, "learning_rate": 7.049694065873882e-05, "loss": 0.031154736876487732, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.97, "memory/max_allocated (GiB)": 37.97, "ppl": 1.03165, "step": 215, "tokens/total": 7484096, "tokens/train_per_sec_per_gpu": 26.62, "tokens/trainable": 98575} +{"epoch": 0.84375, "grad_norm": 0.31234949827194214, "learning_rate": 7.022408581865382e-05, "loss": 0.01168171688914299, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.01175, "step": 216, "tokens/total": 7517056, "tokens/train_per_sec_per_gpu": 24.89, "tokens/trainable": 98991} +{"epoch": 0.84765625, "grad_norm": 0.5124672651290894, "learning_rate": 6.99505974422157e-05, "loss": 0.010257041081786156, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 36.32, "memory/max_allocated (GiB)": 36.32, "ppl": 1.01031, "step": 217, "tokens/total": 7548448, "tokens/train_per_sec_per_gpu": 33.58, "tokens/trainable": 99454} +{"epoch": 0.8515625, "grad_norm": 0.18978096544742584, "learning_rate": 6.967648691039213e-05, "loss": 0.0038094923365861177, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00382, "step": 218, "tokens/total": 7582656, "tokens/train_per_sec_per_gpu": 26.44, "tokens/trainable": 99925} +{"epoch": 0.85546875, "grad_norm": 0.5675084590911865, "learning_rate": 6.940176563004123e-05, "loss": 0.00412407610565424, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00413, "step": 219, "tokens/total": 7618960, "tokens/train_per_sec_per_gpu": 26.23, "tokens/trainable": 100371} +{"epoch": 0.859375, "grad_norm": 0.1966351568698883, "learning_rate": 6.912644503343682e-05, "loss": 0.005590029060840607, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.35, "memory/max_allocated (GiB)": 39.35, "ppl": 1.00561, "step": 220, "tokens/total": 7655504, "tokens/train_per_sec_per_gpu": 26.19, "tokens/trainable": 100855} +{"epoch": 0.86328125, "grad_norm": 0.7241394519805908, "learning_rate": 6.885053657779273e-05, "loss": 0.014526011422276497, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.01463, "step": 221, "tokens/total": 7689872, "tokens/train_per_sec_per_gpu": 21.2, "tokens/trainable": 101253} +{"epoch": 0.8671875, "grad_norm": 0.5192746520042419, "learning_rate": 6.857405174478604e-05, "loss": 0.005446063354611397, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.00546, "step": 222, "tokens/total": 7724432, "tokens/train_per_sec_per_gpu": 26.06, "tokens/trainable": 101706} +{"epoch": 0.87109375, "grad_norm": 1.2140647172927856, "learning_rate": 6.82970020400792e-05, "loss": 0.018532609567046165, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.01871, "step": 223, "tokens/total": 7761440, "tokens/train_per_sec_per_gpu": 22.75, "tokens/trainable": 102158} +{"epoch": 0.875, "grad_norm": 0.25082239508628845, "learning_rate": 6.801939899284132e-05, "loss": 0.0030460674315690994, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 35.36, "memory/max_allocated (GiB)": 35.36, "ppl": 1.00305, "step": 224, "tokens/total": 7790048, "tokens/train_per_sec_per_gpu": 30.35, "tokens/trainable": 102580} +{"epoch": 0.87890625, "grad_norm": 0.0684032142162323, "learning_rate": 6.774125415526827e-05, "loss": 0.0009637181647121906, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.00096, "step": 225, "tokens/total": 7822352, "tokens/train_per_sec_per_gpu": 30.58, "tokens/trainable": 103063} +{"epoch": 0.8828125, "grad_norm": 0.537531852722168, "learning_rate": 6.746257910210214e-05, "loss": 0.014876529574394226, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.01499, "step": 226, "tokens/total": 7861088, "tokens/train_per_sec_per_gpu": 22.09, "tokens/trainable": 103510} +{"epoch": 0.88671875, "grad_norm": 0.36217746138572693, "learning_rate": 6.718338543014937e-05, "loss": 0.0044035352766513824, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.51, "memory/max_allocated (GiB)": 36.51, "ppl": 1.00441, "step": 227, "tokens/total": 7891456, "tokens/train_per_sec_per_gpu": 34.23, "tokens/trainable": 103980} +{"epoch": 0.890625, "grad_norm": 0.3211389482021332, "learning_rate": 6.69036847577983e-05, "loss": 0.0029349736869335175, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.13, "memory/max_allocated (GiB)": 36.13, "ppl": 1.00294, "step": 228, "tokens/total": 7922704, "tokens/train_per_sec_per_gpu": 28.6, "tokens/trainable": 104413} +{"epoch": 0.89453125, "grad_norm": 1.2116146087646484, "learning_rate": 6.662348872453553e-05, "loss": 0.015573585405945778, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.0157, "step": 229, "tokens/total": 7952912, "tokens/train_per_sec_per_gpu": 29.82, "tokens/trainable": 104845} +{"epoch": 0.8984375, "grad_norm": 0.31841254234313965, "learning_rate": 6.63428089904618e-05, "loss": 0.01466198917478323, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.01477, "step": 230, "tokens/total": 7987504, "tokens/train_per_sec_per_gpu": 23.98, "tokens/trainable": 105289} +{"epoch": 0.90234375, "grad_norm": 0.9456703066825867, "learning_rate": 6.60616572358065e-05, "loss": 0.013342907652258873, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 1.01343, "step": 231, "tokens/total": 8025936, "tokens/train_per_sec_per_gpu": 21.75, "tokens/trainable": 105722} +{"epoch": 0.90625, "grad_norm": 0.02019377425312996, "learning_rate": 6.578004516044172e-05, "loss": 0.00030134565895423293, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.59, "memory/max_allocated (GiB)": 36.59, "ppl": 1.0003, "step": 232, "tokens/total": 8058544, "tokens/train_per_sec_per_gpu": 29.16, "tokens/trainable": 106213} +{"epoch": 0.91015625, "grad_norm": 0.050753891468048096, "learning_rate": 6.549798448339548e-05, "loss": 0.0006987630622461438, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 1.0007, "step": 233, "tokens/total": 8091456, "tokens/train_per_sec_per_gpu": 26.83, "tokens/trainable": 106643} +{"epoch": 0.9140625, "grad_norm": 0.4085772931575775, "learning_rate": 6.521548694236384e-05, "loss": 0.008244628086686134, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.97, "memory/max_allocated (GiB)": 38.97, "ppl": 1.00828, "step": 234, "tokens/total": 8127920, "tokens/train_per_sec_per_gpu": 25.52, "tokens/trainable": 107136} +{"epoch": 0.91796875, "grad_norm": 0.4095766246318817, "learning_rate": 6.493256429322259e-05, "loss": 0.004777806345373392, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00479, "step": 235, "tokens/total": 8167472, "tokens/train_per_sec_per_gpu": 24.5, "tokens/trainable": 107588} +{"epoch": 0.921875, "grad_norm": 0.6277322769165039, "learning_rate": 6.464922830953799e-05, "loss": 0.03218494728207588, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.03271, "step": 236, "tokens/total": 8201792, "tokens/train_per_sec_per_gpu": 27.57, "tokens/trainable": 108087} +{"epoch": 0.92578125, "grad_norm": 1.372861385345459, "learning_rate": 6.436549078207688e-05, "loss": 0.016339199617505074, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 1.01647, "step": 237, "tokens/total": 8235504, "tokens/train_per_sec_per_gpu": 27.2, "tokens/trainable": 108533} +{"epoch": 0.9296875, "grad_norm": 0.11756862699985504, "learning_rate": 6.408136351831592e-05, "loss": 0.0011513972422108054, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.24, "memory/max_allocated (GiB)": 38.24, "ppl": 1.00115, "step": 238, "tokens/total": 8271392, "tokens/train_per_sec_per_gpu": 21.65, "tokens/trainable": 108965} +{"epoch": 0.93359375, "grad_norm": 0.39877527952194214, "learning_rate": 6.379685834195036e-05, "loss": 0.014880029484629631, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.01499, "step": 239, "tokens/total": 8304656, "tokens/train_per_sec_per_gpu": 28.75, "tokens/trainable": 109416} +{"epoch": 0.9375, "grad_norm": 0.4812406301498413, "learning_rate": 6.351198709240186e-05, "loss": 0.014240803197026253, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.12, "memory/max_allocated (GiB)": 39.12, "ppl": 1.01434, "step": 240, "tokens/total": 8343280, "tokens/train_per_sec_per_gpu": 23.29, "tokens/trainable": 109864} +{"epoch": 0.94140625, "grad_norm": 0.6666954159736633, "learning_rate": 6.32267616243259e-05, "loss": 0.007355488836765289, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.00738, "step": 241, "tokens/total": 8377728, "tokens/train_per_sec_per_gpu": 26.24, "tokens/trainable": 110345} +{"epoch": 0.9453125, "grad_norm": 0.15433305501937866, "learning_rate": 6.294119380711849e-05, "loss": 0.0028065659571439028, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 37.15, "memory/max_allocated (GiB)": 37.15, "ppl": 1.00281, "step": 242, "tokens/total": 8412112, "tokens/train_per_sec_per_gpu": 24.74, "tokens/trainable": 110794} +{"epoch": 0.94921875, "grad_norm": 0.3040446937084198, "learning_rate": 6.265529552442209e-05, "loss": 0.009950702078640461, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.01, "step": 243, "tokens/total": 8449360, "tokens/train_per_sec_per_gpu": 24.98, "tokens/trainable": 111250} +{"epoch": 0.953125, "grad_norm": 0.15587151050567627, "learning_rate": 6.236907867363127e-05, "loss": 0.003796979784965515, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 1.0038, "step": 244, "tokens/total": 8482160, "tokens/train_per_sec_per_gpu": 26.95, "tokens/trainable": 111693} +{"epoch": 0.95703125, "grad_norm": 0.16347388923168182, "learning_rate": 6.208255516539749e-05, "loss": 0.0032264976762235165, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.45, "memory/max_allocated (GiB)": 38.45, "ppl": 1.00323, "step": 245, "tokens/total": 8516272, "tokens/train_per_sec_per_gpu": 25.77, "tokens/trainable": 112158} +{"epoch": 0.9609375, "grad_norm": 0.2996705174446106, "learning_rate": 6.179573692313344e-05, "loss": 0.009005517698824406, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.98, "memory/max_allocated (GiB)": 36.98, "ppl": 1.00905, "step": 246, "tokens/total": 8548240, "tokens/train_per_sec_per_gpu": 28.81, "tokens/trainable": 112601} +{"epoch": 0.96484375, "grad_norm": 0.09049326181411743, "learning_rate": 6.150863588251694e-05, "loss": 0.0021425937302410603, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.34, "memory/max_allocated (GiB)": 36.34, "ppl": 1.00214, "step": 247, "tokens/total": 8580000, "tokens/train_per_sec_per_gpu": 25.81, "tokens/trainable": 113016} +{"epoch": 0.96875, "grad_norm": 0.6100640296936035, "learning_rate": 6.122126399099419e-05, "loss": 0.007033364847302437, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.00706, "step": 248, "tokens/total": 8614864, "tokens/train_per_sec_per_gpu": 24.63, "tokens/trainable": 113476} +{"epoch": 0.97265625, "grad_norm": 0.4071604609489441, "learning_rate": 6.0933633207282615e-05, "loss": 0.009094493463635445, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.66, "memory/max_allocated (GiB)": 38.66, "ppl": 1.00914, "step": 249, "tokens/total": 8651008, "tokens/train_per_sec_per_gpu": 26.7, "tokens/trainable": 113941} +{"epoch": 0.9765625, "grad_norm": 1.9636086225509644, "learning_rate": 6.064575550087316e-05, "loss": 0.005277995020151138, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.83, "memory/max_allocated (GiB)": 36.83, "ppl": 1.00529, "step": 250, "tokens/total": 8683696, "tokens/train_per_sec_per_gpu": 30.33, "tokens/trainable": 114434} +{"epoch": 0.98046875, "grad_norm": 10.472208023071289, "learning_rate": 6.0357642851532245e-05, "loss": 0.028454232960939407, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.28, "memory/max_allocated (GiB)": 36.28, "ppl": 1.02886, "step": 251, "tokens/total": 8714880, "tokens/train_per_sec_per_gpu": 29.72, "tokens/trainable": 114898} +{"epoch": 0.984375, "grad_norm": 0.1914818435907364, "learning_rate": 6.0069307248803294e-05, "loss": 0.004160391166806221, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.15, "memory/max_allocated (GiB)": 39.15, "ppl": 1.00417, "step": 252, "tokens/total": 8751072, "tokens/train_per_sec_per_gpu": 21.69, "tokens/trainable": 115331} +{"epoch": 0.98828125, "grad_norm": 0.2926669418811798, "learning_rate": 5.9780760691507635e-05, "loss": 0.0036849237512797117, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.73, "memory/max_allocated (GiB)": 36.73, "ppl": 1.00369, "step": 253, "tokens/total": 8782272, "tokens/train_per_sec_per_gpu": 28.04, "tokens/trainable": 115732} +{"epoch": 0.9921875, "grad_norm": 0.3856918513774872, "learning_rate": 5.9492015187245334e-05, "loss": 0.004784893244504929, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.28, "memory/max_allocated (GiB)": 39.28, "ppl": 1.0048, "step": 254, "tokens/total": 8819920, "tokens/train_per_sec_per_gpu": 25.65, "tokens/trainable": 116221} +{"epoch": 0.99609375, "grad_norm": 0.7897813320159912, "learning_rate": 5.920308275189541e-05, "loss": 0.02833794802427292, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 37.91, "memory/max_allocated (GiB)": 37.91, "ppl": 1.02874, "step": 255, "tokens/total": 8855232, "tokens/train_per_sec_per_gpu": 23.79, "tokens/trainable": 116684} +{"epoch": 1.0, "grad_norm": 0.5391122102737427, "learning_rate": 5.8913975409115874e-05, "loss": 0.005181957967579365, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.0052, "step": 256, "tokens/total": 8889920, "tokens/train_per_sec_per_gpu": 24.69, "tokens/trainable": 117126} +{"epoch": 1.00390625, "grad_norm": 0.23738233745098114, "learning_rate": 5.8624705189843395e-05, "loss": 0.0023817496839910746, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00238, "step": 257, "tokens/total": 8922640, "tokens/train_per_sec_per_gpu": 30.2, "tokens/trainable": 117592} +{"epoch": 1.0078125, "grad_norm": 0.022903727367520332, "learning_rate": 5.833528413179249e-05, "loss": 0.0004278519772924483, "memory/device_reserved (GiB)": 39.79, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00043, "step": 258, "tokens/total": 8958848, "tokens/train_per_sec_per_gpu": 21.65, "tokens/trainable": 118030} +{"epoch": 1.01171875, "grad_norm": 0.7227591872215271, "learning_rate": 5.80457242789548e-05, "loss": 0.01054247748106718, "memory/device_reserved (GiB)": 39.79, "memory/max_active (GiB)": 36.51, "memory/max_allocated (GiB)": 36.51, "ppl": 1.0106, "step": 259, "tokens/total": 8991056, "tokens/train_per_sec_per_gpu": 26.47, "tokens/trainable": 118470} +{"epoch": 1.015625, "grad_norm": 0.25689586997032166, "learning_rate": 5.77560376810977e-05, "loss": 0.01355066429823637, "memory/device_reserved (GiB)": 39.79, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.01364, "step": 260, "tokens/total": 9024720, "tokens/train_per_sec_per_gpu": 28.38, "tokens/trainable": 118935} +{"epoch": 1.01953125, "grad_norm": 1.211543083190918, "learning_rate": 5.7466236393263005e-05, "loss": 0.004748472943902016, "memory/device_reserved (GiB)": 39.79, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.00476, "step": 261, "tokens/total": 9058944, "tokens/train_per_sec_per_gpu": 25.58, "tokens/trainable": 119381} +{"epoch": 1.0234375, "grad_norm": 0.44085586071014404, "learning_rate": 5.717633247526522e-05, "loss": 0.0030840779654681683, "memory/device_reserved (GiB)": 41.75, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.00309, "step": 262, "tokens/total": 9098816, "tokens/train_per_sec_per_gpu": 23.26, "tokens/trainable": 119849} +{"epoch": 1.02734375, "grad_norm": 0.050274807959795, "learning_rate": 5.688633799118971e-05, "loss": 0.0004666317254304886, "memory/device_reserved (GiB)": 41.75, "memory/max_active (GiB)": 36.78, "memory/max_allocated (GiB)": 36.78, "ppl": 1.00047, "step": 263, "tokens/total": 9132080, "tokens/train_per_sec_per_gpu": 29.18, "tokens/trainable": 120319} +{"epoch": 1.03125, "grad_norm": 0.7728943824768066, "learning_rate": 5.659626500889066e-05, "loss": 0.009432676248252392, "memory/device_reserved (GiB)": 41.75, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.00948, "step": 264, "tokens/total": 9165072, "tokens/train_per_sec_per_gpu": 31.03, "tokens/trainable": 120804} +{"epoch": 1.03515625, "grad_norm": 0.034000515937805176, "learning_rate": 5.6306125599488905e-05, "loss": 0.0005254977731965482, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 35.9, "memory/max_allocated (GiB)": 35.9, "ppl": 1.00053, "step": 265, "tokens/total": 9195776, "tokens/train_per_sec_per_gpu": 29.38, "tokens/trainable": 121233} +{"epoch": 1.0390625, "grad_norm": 0.4858587086200714, "learning_rate": 5.601593183686955e-05, "loss": 0.008229550905525684, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.95, "memory/max_allocated (GiB)": 37.95, "ppl": 1.00826, "step": 266, "tokens/total": 9231136, "tokens/train_per_sec_per_gpu": 22.63, "tokens/trainable": 121671} +{"epoch": 1.04296875, "grad_norm": 0.3951930105686188, "learning_rate": 5.572569579717961e-05, "loss": 0.015219218097627163, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.01534, "step": 267, "tokens/total": 9266784, "tokens/train_per_sec_per_gpu": 25.76, "tokens/trainable": 122141} +{"epoch": 1.046875, "grad_norm": 0.1368083506822586, "learning_rate": 5.543542955832538e-05, "loss": 0.0014548017643392086, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.37, "memory/max_allocated (GiB)": 38.37, "ppl": 1.00146, "step": 268, "tokens/total": 9301648, "tokens/train_per_sec_per_gpu": 25.17, "tokens/trainable": 122556} +{"epoch": 1.05078125, "grad_norm": 0.6055361032485962, "learning_rate": 5.514514519946986e-05, "loss": 0.01273970864713192, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.01282, "step": 269, "tokens/total": 9334352, "tokens/train_per_sec_per_gpu": 27.42, "tokens/trainable": 122988} +{"epoch": 1.0546875, "grad_norm": 0.2562546133995056, "learning_rate": 5.485485480053015e-05, "loss": 0.002133140340447426, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.00214, "step": 270, "tokens/total": 9369376, "tokens/train_per_sec_per_gpu": 27.02, "tokens/trainable": 123478} +{"epoch": 1.05859375, "grad_norm": 0.6198563575744629, "learning_rate": 5.4564570441674645e-05, "loss": 0.00916452519595623, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.00921, "step": 271, "tokens/total": 9402928, "tokens/train_per_sec_per_gpu": 28.94, "tokens/trainable": 123932} +{"epoch": 1.0625, "grad_norm": 0.3399319350719452, "learning_rate": 5.42743042028204e-05, "loss": 0.0033874153159558773, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.00339, "step": 272, "tokens/total": 9437952, "tokens/train_per_sec_per_gpu": 29.54, "tokens/trainable": 124428} +{"epoch": 1.06640625, "grad_norm": 0.20008371770381927, "learning_rate": 5.3984068163130464e-05, "loss": 0.0033185440115630627, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.00332, "step": 273, "tokens/total": 9471648, "tokens/train_per_sec_per_gpu": 22.96, "tokens/trainable": 124866} +{"epoch": 1.0703125, "grad_norm": 0.1493905484676361, "learning_rate": 5.369387440051111e-05, "loss": 0.0024886736646294594, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.00249, "step": 274, "tokens/total": 9508640, "tokens/train_per_sec_per_gpu": 24.66, "tokens/trainable": 125324} +{"epoch": 1.07421875, "grad_norm": 0.20413674414157867, "learning_rate": 5.340373499110935e-05, "loss": 0.0027680150233209133, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.00277, "step": 275, "tokens/total": 9548560, "tokens/train_per_sec_per_gpu": 24.21, "tokens/trainable": 125813} +{"epoch": 1.078125, "grad_norm": 0.2218458205461502, "learning_rate": 5.3113662008810304e-05, "loss": 0.0035811662673950195, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00359, "step": 276, "tokens/total": 9584608, "tokens/train_per_sec_per_gpu": 22.31, "tokens/trainable": 126252} +{"epoch": 1.08203125, "grad_norm": 0.2106555849313736, "learning_rate": 5.282366752473479e-05, "loss": 0.002148684812709689, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.99, "memory/max_allocated (GiB)": 38.99, "ppl": 1.00215, "step": 277, "tokens/total": 9620656, "tokens/train_per_sec_per_gpu": 25.05, "tokens/trainable": 126696} +{"epoch": 1.0859375, "grad_norm": 0.4040500521659851, "learning_rate": 5.2533763606737005e-05, "loss": 0.017828522250056267, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.01799, "step": 278, "tokens/total": 9652992, "tokens/train_per_sec_per_gpu": 28.78, "tokens/trainable": 127173} +{"epoch": 1.08984375, "grad_norm": 0.45883476734161377, "learning_rate": 5.224396231890232e-05, "loss": 0.007998825050890446, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.00803, "step": 279, "tokens/total": 9685632, "tokens/train_per_sec_per_gpu": 27.93, "tokens/trainable": 127637} +{"epoch": 1.09375, "grad_norm": 0.6755430102348328, "learning_rate": 5.195427572104522e-05, "loss": 0.016413424164056778, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.59, "memory/max_allocated (GiB)": 38.59, "ppl": 1.01655, "step": 280, "tokens/total": 9723056, "tokens/train_per_sec_per_gpu": 24.95, "tokens/trainable": 128112} +{"epoch": 1.09765625, "grad_norm": 0.3177177309989929, "learning_rate": 5.166471586820751e-05, "loss": 0.004505719989538193, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.73, "memory/max_allocated (GiB)": 37.73, "ppl": 1.00452, "step": 281, "tokens/total": 9758592, "tokens/train_per_sec_per_gpu": 26.35, "tokens/trainable": 128584} +{"epoch": 1.1015625, "grad_norm": 0.0752166137099266, "learning_rate": 5.1375294810156615e-05, "loss": 0.000810234691016376, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.3, "memory/max_allocated (GiB)": 38.3, "ppl": 1.00081, "step": 282, "tokens/total": 9793408, "tokens/train_per_sec_per_gpu": 24.79, "tokens/trainable": 129051} +{"epoch": 1.10546875, "grad_norm": 0.059333279728889465, "learning_rate": 5.1086024590884144e-05, "loss": 0.0006349863833747804, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.15, "memory/max_allocated (GiB)": 37.15, "ppl": 1.00064, "step": 283, "tokens/total": 9827216, "tokens/train_per_sec_per_gpu": 25.59, "tokens/trainable": 129494} +{"epoch": 1.109375, "grad_norm": 0.0647725835442543, "learning_rate": 5.079691724810461e-05, "loss": 0.0009689354337751865, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 39.35, "memory/max_allocated (GiB)": 39.35, "ppl": 1.00097, "step": 284, "tokens/total": 9864928, "tokens/train_per_sec_per_gpu": 24.77, "tokens/trainable": 129983} +{"epoch": 1.11328125, "grad_norm": 0.07624267786741257, "learning_rate": 5.0507984812754684e-05, "loss": 0.0013884012587368488, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00139, "step": 285, "tokens/total": 9901520, "tokens/train_per_sec_per_gpu": 23.57, "tokens/trainable": 130405} +{"epoch": 1.1171875, "grad_norm": 0.055877991020679474, "learning_rate": 5.021923930849237e-05, "loss": 0.000537139072548598, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.56, "memory/max_allocated (GiB)": 37.56, "ppl": 1.00054, "step": 286, "tokens/total": 9935824, "tokens/train_per_sec_per_gpu": 24.74, "tokens/trainable": 130829} +{"epoch": 1.12109375, "grad_norm": 0.5696881413459778, "learning_rate": 4.99306927511967e-05, "loss": 0.006251724436879158, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.00627, "step": 287, "tokens/total": 9969200, "tokens/train_per_sec_per_gpu": 24.5, "tokens/trainable": 131287} +{"epoch": 1.125, "grad_norm": 0.053139578551054, "learning_rate": 4.964235714846775e-05, "loss": 0.0009130148100666702, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.00091, "step": 288, "tokens/total": 10005120, "tokens/train_per_sec_per_gpu": 23.65, "tokens/trainable": 131712} +{"epoch": 1.12890625, "grad_norm": 0.0072255185805261135, "learning_rate": 4.9354244499126866e-05, "loss": 0.000157760878209956, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00016, "step": 289, "tokens/total": 10040336, "tokens/train_per_sec_per_gpu": 25.38, "tokens/trainable": 132168} +{"epoch": 1.1328125, "grad_norm": 0.02728038653731346, "learning_rate": 4.90663667927174e-05, "loss": 0.00046376214595511556, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.00046, "step": 290, "tokens/total": 10075232, "tokens/train_per_sec_per_gpu": 27.23, "tokens/trainable": 132626} +{"epoch": 1.13671875, "grad_norm": 0.3682698905467987, "learning_rate": 4.877873600900581e-05, "loss": 0.007497069425880909, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.00753, "step": 291, "tokens/total": 10113632, "tokens/train_per_sec_per_gpu": 24.27, "tokens/trainable": 133119} +{"epoch": 1.140625, "grad_norm": 0.11480076611042023, "learning_rate": 4.849136411748306e-05, "loss": 0.0004577445797622204, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 36.96, "memory/max_allocated (GiB)": 36.96, "ppl": 1.00046, "step": 292, "tokens/total": 10145728, "tokens/train_per_sec_per_gpu": 27.32, "tokens/trainable": 133550} +{"epoch": 1.14453125, "grad_norm": 0.33103346824645996, "learning_rate": 4.8204263076866574e-05, "loss": 0.0025900080800056458, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.39, "memory/max_allocated (GiB)": 38.39, "ppl": 1.00259, "step": 293, "tokens/total": 10182512, "tokens/train_per_sec_per_gpu": 24.89, "tokens/trainable": 133992} +{"epoch": 1.1484375, "grad_norm": 0.05887950584292412, "learning_rate": 4.791744483460251e-05, "loss": 0.0009130619000643492, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.54, "memory/max_allocated (GiB)": 39.54, "ppl": 1.00091, "step": 294, "tokens/total": 10218144, "tokens/train_per_sec_per_gpu": 28.89, "tokens/trainable": 134467} +{"epoch": 1.15234375, "grad_norm": 0.07813172042369843, "learning_rate": 4.7630921326368736e-05, "loss": 0.00121973583009094, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.00122, "step": 295, "tokens/total": 10256160, "tokens/train_per_sec_per_gpu": 20.17, "tokens/trainable": 134895} +{"epoch": 1.15625, "grad_norm": 0.9464846253395081, "learning_rate": 4.7344704475577916e-05, "loss": 0.0051180459558963776, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 35.47, "memory/max_allocated (GiB)": 35.47, "ppl": 1.00513, "step": 296, "tokens/total": 10285136, "tokens/train_per_sec_per_gpu": 31.17, "tokens/trainable": 135321} +{"epoch": 1.16015625, "grad_norm": 0.5158723592758179, "learning_rate": 4.705880619288153e-05, "loss": 0.006375257391482592, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.0064, "step": 297, "tokens/total": 10321424, "tokens/train_per_sec_per_gpu": 23.83, "tokens/trainable": 135765} +{"epoch": 1.1640625, "grad_norm": 0.07714490592479706, "learning_rate": 4.677323837567412e-05, "loss": 0.000753499218262732, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.00075, "step": 298, "tokens/total": 10359248, "tokens/train_per_sec_per_gpu": 24.86, "tokens/trainable": 136196} +{"epoch": 1.16796875, "grad_norm": 0.08915186673402786, "learning_rate": 4.6488012907598146e-05, "loss": 0.0008220027666538954, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.81, "memory/max_allocated (GiB)": 38.81, "ppl": 1.00082, "step": 299, "tokens/total": 10395824, "tokens/train_per_sec_per_gpu": 24.56, "tokens/trainable": 136649} +{"epoch": 1.171875, "grad_norm": 0.07499410212039948, "learning_rate": 4.620314165804964e-05, "loss": 0.0010794871486723423, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00108, "step": 300, "tokens/total": 10432864, "tokens/train_per_sec_per_gpu": 26.8, "tokens/trainable": 137128} +{"epoch": 1.17578125, "grad_norm": 0.4627734124660492, "learning_rate": 4.591863648168407e-05, "loss": 0.002325126901268959, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.04, "memory/max_allocated (GiB)": 37.04, "ppl": 1.00233, "step": 301, "tokens/total": 10466816, "tokens/train_per_sec_per_gpu": 30.57, "tokens/trainable": 137585} +{"epoch": 1.1796875, "grad_norm": 0.3331979811191559, "learning_rate": 4.5634509217923135e-05, "loss": 0.0022070903796702623, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.00221, "step": 302, "tokens/total": 10499280, "tokens/train_per_sec_per_gpu": 31.38, "tokens/trainable": 138054} +{"epoch": 1.18359375, "grad_norm": 0.2161167412996292, "learning_rate": 4.535077169046201e-05, "loss": 0.003484009299427271, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.0, "memory/max_allocated (GiB)": 38.0, "ppl": 1.00349, "step": 303, "tokens/total": 10534112, "tokens/train_per_sec_per_gpu": 28.86, "tokens/trainable": 138559} +{"epoch": 1.1875, "grad_norm": 0.17748290300369263, "learning_rate": 4.506743570677743e-05, "loss": 0.002638915553689003, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.53, "memory/max_allocated (GiB)": 37.53, "ppl": 1.00264, "step": 304, "tokens/total": 10568912, "tokens/train_per_sec_per_gpu": 24.42, "tokens/trainable": 139050} +{"epoch": 1.19140625, "grad_norm": 0.0322340689599514, "learning_rate": 4.478451305763618e-05, "loss": 0.0005882253171876073, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.00059, "step": 305, "tokens/total": 10606128, "tokens/train_per_sec_per_gpu": 24.96, "tokens/trainable": 139525} +{"epoch": 1.1953125, "grad_norm": 0.2454739511013031, "learning_rate": 4.450201551660454e-05, "loss": 0.002371998503804207, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.08, "memory/max_allocated (GiB)": 39.08, "ppl": 1.00237, "step": 306, "tokens/total": 10639584, "tokens/train_per_sec_per_gpu": 26.74, "tokens/trainable": 139969} +{"epoch": 1.19921875, "grad_norm": 0.03169209137558937, "learning_rate": 4.4219954839558276e-05, "loss": 0.00041654478991404176, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.44, "memory/max_allocated (GiB)": 36.44, "ppl": 1.00042, "step": 307, "tokens/total": 10670928, "tokens/train_per_sec_per_gpu": 31.76, "tokens/trainable": 140443} +{"epoch": 1.203125, "grad_norm": 0.014757092110812664, "learning_rate": 4.393834276419352e-05, "loss": 0.00016781711019575596, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.00017, "step": 308, "tokens/total": 10709968, "tokens/train_per_sec_per_gpu": 24.1, "tokens/trainable": 140926} +{"epoch": 1.20703125, "grad_norm": 0.012232726439833641, "learning_rate": 4.36571910095382e-05, "loss": 0.00012468986096791923, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.00012, "step": 309, "tokens/total": 10744800, "tokens/train_per_sec_per_gpu": 24.32, "tokens/trainable": 141344} +{"epoch": 1.2109375, "grad_norm": 0.7113270163536072, "learning_rate": 4.337651127546448e-05, "loss": 0.010171249508857727, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.01022, "step": 310, "tokens/total": 10778576, "tokens/train_per_sec_per_gpu": 26.34, "tokens/trainable": 141753} +{"epoch": 1.21484375, "grad_norm": 0.02107495255768299, "learning_rate": 4.3096315242201736e-05, "loss": 0.00043003627797588706, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.61, "memory/max_allocated (GiB)": 36.61, "ppl": 1.00043, "step": 311, "tokens/total": 10810816, "tokens/train_per_sec_per_gpu": 31.39, "tokens/trainable": 142219} +{"epoch": 1.21875, "grad_norm": 0.3906373083591461, "learning_rate": 4.2816614569850635e-05, "loss": 0.0026145298033952713, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.01, "memory/max_allocated (GiB)": 36.01, "ppl": 1.00262, "step": 312, "tokens/total": 10841040, "tokens/train_per_sec_per_gpu": 25.48, "tokens/trainable": 142617} +{"epoch": 1.22265625, "grad_norm": 0.3830045163631439, "learning_rate": 4.2537420897897864e-05, "loss": 0.0038155640941113234, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.00382, "step": 313, "tokens/total": 10880960, "tokens/train_per_sec_per_gpu": 21.5, "tokens/trainable": 143103} +{"epoch": 1.2265625, "grad_norm": 0.024424191564321518, "learning_rate": 4.225874584473174e-05, "loss": 0.00024393366766162217, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00024, "step": 314, "tokens/total": 10913472, "tokens/train_per_sec_per_gpu": 25.34, "tokens/trainable": 143535} +{"epoch": 1.23046875, "grad_norm": 0.6737616062164307, "learning_rate": 4.19806010071587e-05, "loss": 0.0024917684495449066, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00249, "step": 315, "tokens/total": 10951360, "tokens/train_per_sec_per_gpu": 23.93, "tokens/trainable": 144009} +{"epoch": 1.234375, "grad_norm": 0.18527784943580627, "learning_rate": 4.170299795992081e-05, "loss": 0.0011532303178682923, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.00115, "step": 316, "tokens/total": 10987808, "tokens/train_per_sec_per_gpu": 23.38, "tokens/trainable": 144460} +{"epoch": 1.23828125, "grad_norm": 0.36189934611320496, "learning_rate": 4.142594825521398e-05, "loss": 0.011965572834014893, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.01204, "step": 317, "tokens/total": 11025552, "tokens/train_per_sec_per_gpu": 24.7, "tokens/trainable": 144936} +{"epoch": 1.2421875, "grad_norm": 0.010431681759655476, "learning_rate": 4.114946342220728e-05, "loss": 0.000120488868560642, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.00012, "step": 318, "tokens/total": 11060432, "tokens/train_per_sec_per_gpu": 25.25, "tokens/trainable": 145375} +{"epoch": 1.24609375, "grad_norm": 0.01244704145938158, "learning_rate": 4.087355496656321e-05, "loss": 0.00015264860121533275, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.00015, "step": 319, "tokens/total": 11094688, "tokens/train_per_sec_per_gpu": 25.6, "tokens/trainable": 145849} +{"epoch": 1.25, "grad_norm": 0.015057384967803955, "learning_rate": 4.05982343699588e-05, "loss": 0.0002183643082389608, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00022, "step": 320, "tokens/total": 11127408, "tokens/train_per_sec_per_gpu": 27.01, "tokens/trainable": 146318} +{"epoch": 1.25390625, "grad_norm": 0.02062501199543476, "learning_rate": 4.0323513089607876e-05, "loss": 0.0001829334651120007, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.72, "memory/max_allocated (GiB)": 38.72, "ppl": 1.00018, "step": 321, "tokens/total": 11162752, "tokens/train_per_sec_per_gpu": 26.72, "tokens/trainable": 146821} +{"epoch": 1.2578125, "grad_norm": 0.16000086069107056, "learning_rate": 4.004940255778431e-05, "loss": 0.0006594739970751107, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.33, "memory/max_allocated (GiB)": 36.33, "ppl": 1.00066, "step": 322, "tokens/total": 11194576, "tokens/train_per_sec_per_gpu": 26.47, "tokens/trainable": 147285} +{"epoch": 1.26171875, "grad_norm": 0.3998604118824005, "learning_rate": 3.977591418134619e-05, "loss": 0.004668947774916887, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.61, "memory/max_allocated (GiB)": 36.61, "ppl": 1.00468, "step": 323, "tokens/total": 11226928, "tokens/train_per_sec_per_gpu": 25.69, "tokens/trainable": 147729} +{"epoch": 1.265625, "grad_norm": 0.7337031960487366, "learning_rate": 3.95030593412612e-05, "loss": 0.024531034752726555, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.02483, "step": 324, "tokens/total": 11266016, "tokens/train_per_sec_per_gpu": 20.61, "tokens/trainable": 148169} +{"epoch": 1.26953125, "grad_norm": 0.3310645520687103, "learning_rate": 3.923084939213296e-05, "loss": 0.00595075124874711, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.00597, "step": 325, "tokens/total": 11303264, "tokens/train_per_sec_per_gpu": 24.08, "tokens/trainable": 148630} +{"epoch": 1.2734375, "grad_norm": 0.2237575650215149, "learning_rate": 3.895929566172861e-05, "loss": 0.0005310262786224484, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00053, "step": 326, "tokens/total": 11339392, "tokens/train_per_sec_per_gpu": 22.6, "tokens/trainable": 149063} +{"epoch": 1.27734375, "grad_norm": 0.14760328829288483, "learning_rate": 3.868840945050728e-05, "loss": 0.0008782179211266339, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.00088, "step": 327, "tokens/total": 11377568, "tokens/train_per_sec_per_gpu": 23.19, "tokens/trainable": 149549} +{"epoch": 1.28125, "grad_norm": 0.45188066363334656, "learning_rate": 3.841820203114995e-05, "loss": 0.006132489535957575, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.03, "memory/max_allocated (GiB)": 38.03, "ppl": 1.00615, "step": 328, "tokens/total": 11413184, "tokens/train_per_sec_per_gpu": 26.81, "tokens/trainable": 150015} +{"epoch": 1.28515625, "grad_norm": 0.07668077200651169, "learning_rate": 3.814868464809027e-05, "loss": 0.0003129708638880402, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.00031, "step": 329, "tokens/total": 11447904, "tokens/train_per_sec_per_gpu": 28.12, "tokens/trainable": 150514} +{"epoch": 1.2890625, "grad_norm": 0.6028933525085449, "learning_rate": 3.787986851704667e-05, "loss": 0.009750008583068848, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.0098, "step": 330, "tokens/total": 11485056, "tokens/train_per_sec_per_gpu": 27.55, "tokens/trainable": 151021} +{"epoch": 1.29296875, "grad_norm": 0.2767861485481262, "learning_rate": 3.7611764824555654e-05, "loss": 0.004883499350398779, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.37, "memory/max_allocated (GiB)": 38.37, "ppl": 1.0049, "step": 331, "tokens/total": 11518848, "tokens/train_per_sec_per_gpu": 29.84, "tokens/trainable": 151486} +{"epoch": 1.296875, "grad_norm": 0.7046244740486145, "learning_rate": 3.734438472750619e-05, "loss": 0.008426632732152939, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.79, "memory/max_allocated (GiB)": 36.79, "ppl": 1.00846, "step": 332, "tokens/total": 11552048, "tokens/train_per_sec_per_gpu": 26.39, "tokens/trainable": 151956} +{"epoch": 1.30078125, "grad_norm": 0.026224251836538315, "learning_rate": 3.707773935267552e-05, "loss": 0.0004862675559706986, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.15, "memory/max_allocated (GiB)": 39.15, "ppl": 1.00049, "step": 333, "tokens/total": 11589168, "tokens/train_per_sec_per_gpu": 24.87, "tokens/trainable": 152442} +{"epoch": 1.3046875, "grad_norm": 0.015835443511605263, "learning_rate": 3.68118397962661e-05, "loss": 0.00028911264962516725, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.74, "memory/max_allocated (GiB)": 36.74, "ppl": 1.00029, "step": 334, "tokens/total": 11622032, "tokens/train_per_sec_per_gpu": 25.87, "tokens/trainable": 152890} +{"epoch": 1.30859375, "grad_norm": 0.3589073121547699, "learning_rate": 3.654669712344384e-05, "loss": 0.0048790727742016315, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.35, "memory/max_allocated (GiB)": 36.35, "ppl": 1.00489, "step": 335, "tokens/total": 11653840, "tokens/train_per_sec_per_gpu": 30.84, "tokens/trainable": 153368} +{"epoch": 1.3125, "grad_norm": 0.06946255266666412, "learning_rate": 3.628232236787763e-05, "loss": 0.0006257174536585808, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00063, "step": 336, "tokens/total": 11689664, "tokens/train_per_sec_per_gpu": 26.43, "tokens/trainable": 153809} +{"epoch": 1.31640625, "grad_norm": 0.16393905878067017, "learning_rate": 3.6018726531280144e-05, "loss": 0.0018150057876482606, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.06, "memory/max_allocated (GiB)": 37.06, "ppl": 1.00182, "step": 337, "tokens/total": 11723728, "tokens/train_per_sec_per_gpu": 25.87, "tokens/trainable": 154258} +{"epoch": 1.3203125, "grad_norm": 0.13343408703804016, "learning_rate": 3.575592058295017e-05, "loss": 0.0010332557139918208, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.00103, "step": 338, "tokens/total": 11758960, "tokens/train_per_sec_per_gpu": 25.01, "tokens/trainable": 154704} +{"epoch": 1.32421875, "grad_norm": 0.29510578513145447, "learning_rate": 3.549391545931585e-05, "loss": 0.002914194017648697, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.00292, "step": 339, "tokens/total": 11797328, "tokens/train_per_sec_per_gpu": 24.04, "tokens/trainable": 155157} +{"epoch": 1.328125, "grad_norm": 0.27294921875, "learning_rate": 3.5232722063479914e-05, "loss": 0.0025444228667765856, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.07, "memory/max_allocated (GiB)": 38.07, "ppl": 1.00255, "step": 340, "tokens/total": 11832656, "tokens/train_per_sec_per_gpu": 23.77, "tokens/trainable": 155582} +{"epoch": 1.33203125, "grad_norm": 0.012771952897310257, "learning_rate": 3.49723512647657e-05, "loss": 0.0003347593010403216, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.00033, "step": 341, "tokens/total": 11865152, "tokens/train_per_sec_per_gpu": 28.16, "tokens/trainable": 156047} +{"epoch": 1.3359375, "grad_norm": 0.29202204942703247, "learning_rate": 3.471281389826491e-05, "loss": 0.00193426210898906, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.99, "memory/max_allocated (GiB)": 38.99, "ppl": 1.00194, "step": 342, "tokens/total": 11901552, "tokens/train_per_sec_per_gpu": 25.85, "tokens/trainable": 156500} +{"epoch": 1.33984375, "grad_norm": 1.1314254999160767, "learning_rate": 3.4454120764386764e-05, "loss": 0.005633341148495674, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.00565, "step": 343, "tokens/total": 11937328, "tokens/train_per_sec_per_gpu": 24.97, "tokens/trainable": 156931} +{"epoch": 1.34375, "grad_norm": 0.5701040625572205, "learning_rate": 3.4196282628408526e-05, "loss": 0.012044030241668224, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.01212, "step": 344, "tokens/total": 11970736, "tokens/train_per_sec_per_gpu": 25.67, "tokens/trainable": 157385} +{"epoch": 1.34765625, "grad_norm": 0.7751384973526001, "learning_rate": 3.3939310220027456e-05, "loss": 0.00839341152459383, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.55, "memory/max_allocated (GiB)": 39.55, "ppl": 1.00843, "step": 345, "tokens/total": 12008096, "tokens/train_per_sec_per_gpu": 22.94, "tokens/trainable": 157817} +{"epoch": 1.3515625, "grad_norm": 0.2627096176147461, "learning_rate": 3.3683214232914404e-05, "loss": 0.0024759459774941206, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.00248, "step": 346, "tokens/total": 12039904, "tokens/train_per_sec_per_gpu": 26.54, "tokens/trainable": 158273} +{"epoch": 1.35546875, "grad_norm": 0.019410759210586548, "learning_rate": 3.342800532426873e-05, "loss": 0.00032968143932521343, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.2, "memory/max_allocated (GiB)": 37.2, "ppl": 1.00033, "step": 347, "tokens/total": 12074416, "tokens/train_per_sec_per_gpu": 26.2, "tokens/trainable": 158728} +{"epoch": 1.359375, "grad_norm": 0.009781667962670326, "learning_rate": 3.317369411437484e-05, "loss": 0.0001431062410119921, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.12, "memory/max_allocated (GiB)": 37.12, "ppl": 1.00014, "step": 348, "tokens/total": 12108432, "tokens/train_per_sec_per_gpu": 24.66, "tokens/trainable": 159168} +{"epoch": 1.36328125, "grad_norm": 0.06225920841097832, "learning_rate": 3.292029118616024e-05, "loss": 0.0006305223796516657, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.00063, "step": 349, "tokens/total": 12141792, "tokens/train_per_sec_per_gpu": 23.33, "tokens/trainable": 159592} +{"epoch": 1.3671875, "grad_norm": 0.2750628888607025, "learning_rate": 3.266780708475511e-05, "loss": 0.0010925030801445246, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.00109, "step": 350, "tokens/total": 12175840, "tokens/train_per_sec_per_gpu": 29.58, "tokens/trainable": 160093} +{"epoch": 1.37109375, "grad_norm": 0.021888243034482002, "learning_rate": 3.241625231705354e-05, "loss": 0.00034601622610352933, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.56, "memory/max_allocated (GiB)": 37.56, "ppl": 1.00035, "step": 351, "tokens/total": 12208720, "tokens/train_per_sec_per_gpu": 25.91, "tokens/trainable": 160534} +{"epoch": 1.375, "grad_norm": 0.06603048741817474, "learning_rate": 3.216563735127618e-05, "loss": 0.0009369483450427651, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.00094, "step": 352, "tokens/total": 12246336, "tokens/train_per_sec_per_gpu": 24.54, "tokens/trainable": 161009} +{"epoch": 1.37890625, "grad_norm": 0.5312817096710205, "learning_rate": 3.191597261653475e-05, "loss": 0.008580850437283516, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.00862, "step": 353, "tokens/total": 12279216, "tokens/train_per_sec_per_gpu": 28.01, "tokens/trainable": 161450} +{"epoch": 1.3828125, "grad_norm": 1.2963000535964966, "learning_rate": 3.166726850239794e-05, "loss": 0.010693076997995377, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.01075, "step": 354, "tokens/total": 12311936, "tokens/train_per_sec_per_gpu": 30.59, "tokens/trainable": 161900} +{"epoch": 1.38671875, "grad_norm": 0.055943362414836884, "learning_rate": 3.141953535845912e-05, "loss": 0.0005828076391480863, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.84, "memory/max_allocated (GiB)": 37.84, "ppl": 1.00058, "step": 355, "tokens/total": 12347136, "tokens/train_per_sec_per_gpu": 28.74, "tokens/trainable": 162358} +{"epoch": 1.390625, "grad_norm": 0.03999824449419975, "learning_rate": 3.11727834939056e-05, "loss": 0.0005282496567815542, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.53, "memory/max_allocated (GiB)": 37.53, "ppl": 1.00053, "step": 356, "tokens/total": 12380704, "tokens/train_per_sec_per_gpu": 27.88, "tokens/trainable": 162845} +{"epoch": 1.39453125, "grad_norm": 0.2819129228591919, "learning_rate": 3.092702317708967e-05, "loss": 0.0010848877718672156, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.84, "memory/max_allocated (GiB)": 37.84, "ppl": 1.00109, "step": 357, "tokens/total": 12415744, "tokens/train_per_sec_per_gpu": 25.22, "tokens/trainable": 163299} +{"epoch": 1.3984375, "grad_norm": 0.11842089891433716, "learning_rate": 3.0682264635101276e-05, "loss": 0.001128612901084125, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 1.00113, "step": 358, "tokens/total": 12450400, "tokens/train_per_sec_per_gpu": 27.49, "tokens/trainable": 163774} +{"epoch": 1.40234375, "grad_norm": 0.015053908340632915, "learning_rate": 3.0438518053342407e-05, "loss": 0.00030059064738452435, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.0003, "step": 359, "tokens/total": 12482976, "tokens/train_per_sec_per_gpu": 24.5, "tokens/trainable": 164199} +{"epoch": 1.40625, "grad_norm": 0.15073679387569427, "learning_rate": 3.0195793575103266e-05, "loss": 0.0015767153818160295, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.1, "memory/max_allocated (GiB)": 36.1, "ppl": 1.00158, "step": 360, "tokens/total": 12513392, "tokens/train_per_sec_per_gpu": 31.5, "tokens/trainable": 164667} +{"epoch": 1.41015625, "grad_norm": 0.427316814661026, "learning_rate": 2.9954101301140146e-05, "loss": 0.007560240104794502, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.91, "memory/max_allocated (GiB)": 38.91, "ppl": 1.00759, "step": 361, "tokens/total": 12552528, "tokens/train_per_sec_per_gpu": 22.17, "tokens/trainable": 165111} +{"epoch": 1.4140625, "grad_norm": 0.07099093496799469, "learning_rate": 2.9713451289255123e-05, "loss": 0.0009205960668623447, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.07, "memory/max_allocated (GiB)": 37.07, "ppl": 1.00092, "step": 362, "tokens/total": 12584704, "tokens/train_per_sec_per_gpu": 30.57, "tokens/trainable": 165575} +{"epoch": 1.41796875, "grad_norm": 0.7450342178344727, "learning_rate": 2.9473853553877484e-05, "loss": 0.0035748956725001335, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.00358, "step": 363, "tokens/total": 12619904, "tokens/train_per_sec_per_gpu": 26.51, "tokens/trainable": 166060} +{"epoch": 1.421875, "grad_norm": 0.02035367675125599, "learning_rate": 2.9235318065647e-05, "loss": 0.00021391798509284854, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00021, "step": 364, "tokens/total": 12655152, "tokens/train_per_sec_per_gpu": 23.68, "tokens/trainable": 166478} +{"epoch": 1.42578125, "grad_norm": 0.4379816949367523, "learning_rate": 2.8997854750998964e-05, "loss": 0.00981102604418993, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.00986, "step": 365, "tokens/total": 12691888, "tokens/train_per_sec_per_gpu": 26.5, "tokens/trainable": 166941} +{"epoch": 1.4296875, "grad_norm": 0.01103851106017828, "learning_rate": 2.8761473491751258e-05, "loss": 0.00010778513387776911, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.77, "memory/max_allocated (GiB)": 36.77, "ppl": 1.00011, "step": 366, "tokens/total": 12724400, "tokens/train_per_sec_per_gpu": 28.41, "tokens/trainable": 167381} +{"epoch": 1.43359375, "grad_norm": 0.1435622274875641, "learning_rate": 2.8526184124692883e-05, "loss": 0.0012280570808798075, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.00123, "step": 367, "tokens/total": 12757664, "tokens/train_per_sec_per_gpu": 27.32, "tokens/trainable": 167830} +{"epoch": 1.4375, "grad_norm": 0.05840284004807472, "learning_rate": 2.829199644117484e-05, "loss": 0.00030438604881055653, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.0003, "step": 368, "tokens/total": 12793680, "tokens/train_per_sec_per_gpu": 23.39, "tokens/trainable": 168266} +{"epoch": 1.44140625, "grad_norm": 0.007933669723570347, "learning_rate": 2.8058920186702553e-05, "loss": 0.00015056724078021944, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.00015, "step": 369, "tokens/total": 12828400, "tokens/train_per_sec_per_gpu": 26.8, "tokens/trainable": 168716} +{"epoch": 1.4453125, "grad_norm": 0.06429164111614227, "learning_rate": 2.782696506053033e-05, "loss": 0.00032937509240582585, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00033, "step": 370, "tokens/total": 12864752, "tokens/train_per_sec_per_gpu": 25.84, "tokens/trainable": 169176} +{"epoch": 1.44921875, "grad_norm": 0.18696340918540955, "learning_rate": 2.7596140715257824e-05, "loss": 0.0013089372077956796, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.12, "memory/max_allocated (GiB)": 39.12, "ppl": 1.00131, "step": 371, "tokens/total": 12900768, "tokens/train_per_sec_per_gpu": 22.15, "tokens/trainable": 169591} +{"epoch": 1.453125, "grad_norm": 0.02627130225300789, "learning_rate": 2.7366456756428184e-05, "loss": 0.00020461306849028915, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.59, "memory/max_allocated (GiB)": 37.59, "ppl": 1.0002, "step": 372, "tokens/total": 12935456, "tokens/train_per_sec_per_gpu": 26.32, "tokens/trainable": 170041} +{"epoch": 1.45703125, "grad_norm": 0.03428143635392189, "learning_rate": 2.7137922742128486e-05, "loss": 0.00023305120703298599, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.00023, "step": 373, "tokens/total": 12968448, "tokens/train_per_sec_per_gpu": 30.32, "tokens/trainable": 170516} +{"epoch": 1.4609375, "grad_norm": 0.027386527508497238, "learning_rate": 2.691054818259188e-05, "loss": 0.00023698429868090898, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.73, "memory/max_allocated (GiB)": 37.73, "ppl": 1.00024, "step": 374, "tokens/total": 13003904, "tokens/train_per_sec_per_gpu": 24.59, "tokens/trainable": 170952} +{"epoch": 1.46484375, "grad_norm": 0.11185844987630844, "learning_rate": 2.6684342539801933e-05, "loss": 0.0011279953178018332, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.00113, "step": 375, "tokens/total": 13040864, "tokens/train_per_sec_per_gpu": 25.95, "tokens/trainable": 171424} +{"epoch": 1.46875, "grad_norm": 0.026303540915250778, "learning_rate": 2.645931522709877e-05, "loss": 0.0002070557966362685, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.00021, "step": 376, "tokens/total": 13079504, "tokens/train_per_sec_per_gpu": 21.67, "tokens/trainable": 171884} +{"epoch": 1.47265625, "grad_norm": 0.005460913758724928, "learning_rate": 2.6235475608787365e-05, "loss": 5.257110024103895e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00005, "step": 377, "tokens/total": 13115024, "tokens/train_per_sec_per_gpu": 28.08, "tokens/trainable": 172404} +{"epoch": 1.4765625, "grad_norm": 0.6318137049674988, "learning_rate": 2.6012832999747916e-05, "loss": 0.005940739531069994, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00596, "step": 378, "tokens/total": 13152176, "tokens/train_per_sec_per_gpu": 25.27, "tokens/trainable": 172871} +{"epoch": 1.48046875, "grad_norm": 0.16261129081249237, "learning_rate": 2.579139666504821e-05, "loss": 0.0007067634142003953, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00071, "step": 379, "tokens/total": 13184896, "tokens/train_per_sec_per_gpu": 25.84, "tokens/trainable": 173329} +{"epoch": 1.484375, "grad_norm": 0.24566233158111572, "learning_rate": 2.557117581955798e-05, "loss": 0.001190951094031334, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.00119, "step": 380, "tokens/total": 13220240, "tokens/train_per_sec_per_gpu": 26.55, "tokens/trainable": 173827} +{"epoch": 1.48828125, "grad_norm": 0.11285808682441711, "learning_rate": 2.5352179627565532e-05, "loss": 0.0006733063491992652, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00067, "step": 381, "tokens/total": 13259808, "tokens/train_per_sec_per_gpu": 23.58, "tokens/trainable": 174287} +{"epoch": 1.4921875, "grad_norm": 0.42785122990608215, "learning_rate": 2.5134417202396277e-05, "loss": 0.0022269696928560734, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00223, "step": 382, "tokens/total": 13295776, "tokens/train_per_sec_per_gpu": 25.51, "tokens/trainable": 174731} +{"epoch": 1.49609375, "grad_norm": 0.004365275613963604, "learning_rate": 2.491789760603361e-05, "loss": 6.861190195195377e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00007, "step": 383, "tokens/total": 13331264, "tokens/train_per_sec_per_gpu": 24.22, "tokens/trainable": 175183} +{"epoch": 1.5, "grad_norm": 0.16906186938285828, "learning_rate": 2.4702629848741764e-05, "loss": 0.0009305627318099141, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.11, "memory/max_allocated (GiB)": 38.11, "ppl": 1.00093, "step": 384, "tokens/total": 13364768, "tokens/train_per_sec_per_gpu": 29.57, "tokens/trainable": 175655} +{"epoch": 1.50390625, "grad_norm": 0.03667079657316208, "learning_rate": 2.4488622888690785e-05, "loss": 0.00017129958723671734, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.00017, "step": 385, "tokens/total": 13397904, "tokens/train_per_sec_per_gpu": 28.64, "tokens/trainable": 176085} +{"epoch": 1.5078125, "grad_norm": 0.018642401322722435, "learning_rate": 2.427588563158384e-05, "loss": 0.00013169340672902763, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.00013, "step": 386, "tokens/total": 13432144, "tokens/train_per_sec_per_gpu": 27.36, "tokens/trainable": 176549} +{"epoch": 1.51171875, "grad_norm": 0.0698385238647461, "learning_rate": 2.406442693028651e-05, "loss": 0.0006195258465595543, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00062, "step": 387, "tokens/total": 13468080, "tokens/train_per_sec_per_gpu": 26.12, "tokens/trainable": 176995} +{"epoch": 1.515625, "grad_norm": 0.021888576447963715, "learning_rate": 2.3854255584458547e-05, "loss": 0.00010942642984446138, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.00011, "step": 388, "tokens/total": 13505728, "tokens/train_per_sec_per_gpu": 25.01, "tokens/trainable": 177468} +{"epoch": 1.51953125, "grad_norm": 0.11023979634046555, "learning_rate": 2.3645380340187508e-05, "loss": 0.0007801175815984607, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.32, "memory/max_allocated (GiB)": 38.32, "ppl": 1.00078, "step": 389, "tokens/total": 13542704, "tokens/train_per_sec_per_gpu": 27.74, "tokens/trainable": 177961} +{"epoch": 1.5234375, "grad_norm": 0.005182322580367327, "learning_rate": 2.3437809889624914e-05, "loss": 6.919907173141837e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.19, "memory/max_allocated (GiB)": 39.19, "ppl": 1.00007, "step": 390, "tokens/total": 13580064, "tokens/train_per_sec_per_gpu": 24.27, "tokens/trainable": 178417} +{"epoch": 1.52734375, "grad_norm": 0.04003197327256203, "learning_rate": 2.3231552870624487e-05, "loss": 0.0002931265044026077, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.00029, "step": 391, "tokens/total": 13614736, "tokens/train_per_sec_per_gpu": 25.2, "tokens/trainable": 178862} +{"epoch": 1.53125, "grad_norm": 0.025783589109778404, "learning_rate": 2.3026617866382657e-05, "loss": 0.00023883637913968414, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.83, "memory/max_allocated (GiB)": 37.83, "ppl": 1.00024, "step": 392, "tokens/total": 13647344, "tokens/train_per_sec_per_gpu": 27.23, "tokens/trainable": 179320} +{"epoch": 1.53515625, "grad_norm": 0.18868178129196167, "learning_rate": 2.2823013405081507e-05, "loss": 0.0009424574673175812, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.00094, "step": 393, "tokens/total": 13680192, "tokens/train_per_sec_per_gpu": 28.88, "tokens/trainable": 179758} +{"epoch": 1.5390625, "grad_norm": 0.5565416216850281, "learning_rate": 2.2620747959533722e-05, "loss": 0.004191813990473747, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.0042, "step": 394, "tokens/total": 13713488, "tokens/train_per_sec_per_gpu": 27.98, "tokens/trainable": 180211} +{"epoch": 1.54296875, "grad_norm": 0.22104968130588531, "learning_rate": 2.2419829946830123e-05, "loss": 0.0017442910466343164, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.00175, "step": 395, "tokens/total": 13747264, "tokens/train_per_sec_per_gpu": 25.2, "tokens/trainable": 180637} +{"epoch": 1.546875, "grad_norm": 0.1642310619354248, "learning_rate": 2.2220267727989325e-05, "loss": 0.0012744027189910412, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.32, "memory/max_allocated (GiB)": 39.32, "ppl": 1.00128, "step": 396, "tokens/total": 13784256, "tokens/train_per_sec_per_gpu": 26.38, "tokens/trainable": 181142} +{"epoch": 1.55078125, "grad_norm": 0.07245692610740662, "learning_rate": 2.202206960760984e-05, "loss": 0.0003029649087693542, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.82, "memory/max_allocated (GiB)": 37.82, "ppl": 1.0003, "step": 397, "tokens/total": 13819616, "tokens/train_per_sec_per_gpu": 26.81, "tokens/trainable": 181597} +{"epoch": 1.5546875, "grad_norm": 0.012536926195025444, "learning_rate": 2.182524383352446e-05, "loss": 0.00011802894005086273, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00012, "step": 398, "tokens/total": 13854592, "tokens/train_per_sec_per_gpu": 26.81, "tokens/trainable": 182030} +{"epoch": 1.55859375, "grad_norm": 0.008079864084720612, "learning_rate": 2.1629798596457056e-05, "loss": 6.995685544097796e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 35.72, "memory/max_allocated (GiB)": 35.72, "ppl": 1.00007, "step": 399, "tokens/total": 13884736, "tokens/train_per_sec_per_gpu": 32.04, "tokens/trainable": 182496} +{"epoch": 1.5625, "grad_norm": 0.6516197323799133, "learning_rate": 2.1435742029681725e-05, "loss": 0.01198640838265419, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.39, "memory/max_allocated (GiB)": 37.39, "ppl": 1.01206, "step": 400, "tokens/total": 13918224, "tokens/train_per_sec_per_gpu": 27.74, "tokens/trainable": 182963} +{"epoch": 1.56640625, "grad_norm": 0.002355618868023157, "learning_rate": 2.124308220868431e-05, "loss": 3.252657552366145e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00003, "step": 401, "tokens/total": 13954720, "tokens/train_per_sec_per_gpu": 24.01, "tokens/trainable": 183416} +{"epoch": 1.5703125, "grad_norm": 0.009035729803144932, "learning_rate": 2.105182715082638e-05, "loss": 4.927242116536945e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.17, "memory/max_allocated (GiB)": 37.17, "ppl": 1.00005, "step": 402, "tokens/total": 13986848, "tokens/train_per_sec_per_gpu": 27.67, "tokens/trainable": 183857} +{"epoch": 1.57421875, "grad_norm": 1.0444165468215942, "learning_rate": 2.0861984815011552e-05, "loss": 0.006142558064311743, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.22, "memory/max_allocated (GiB)": 38.22, "ppl": 1.00616, "step": 403, "tokens/total": 14021664, "tokens/train_per_sec_per_gpu": 24.8, "tokens/trainable": 184296} +{"epoch": 1.578125, "grad_norm": 0.40583762526512146, "learning_rate": 2.0673563101354323e-05, "loss": 0.001467288937419653, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00147, "step": 404, "tokens/total": 14059456, "tokens/train_per_sec_per_gpu": 22.95, "tokens/trainable": 184719} +{"epoch": 1.58203125, "grad_norm": 0.009465958923101425, "learning_rate": 2.0486569850851317e-05, "loss": 6.844000745331869e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.00007, "step": 405, "tokens/total": 14091552, "tokens/train_per_sec_per_gpu": 27.42, "tokens/trainable": 185172} +{"epoch": 1.5859375, "grad_norm": 0.08080492168664932, "learning_rate": 2.0301012845054956e-05, "loss": 0.0005934401415288448, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.00059, "step": 406, "tokens/total": 14125328, "tokens/train_per_sec_per_gpu": 27.24, "tokens/trainable": 185646} +{"epoch": 1.58984375, "grad_norm": 0.031067553907632828, "learning_rate": 2.011689980574966e-05, "loss": 0.0001663369039306417, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.00017, "step": 407, "tokens/total": 14158080, "tokens/train_per_sec_per_gpu": 27.1, "tokens/trainable": 186129} +{"epoch": 1.59375, "grad_norm": 0.2663011848926544, "learning_rate": 1.993423839463052e-05, "loss": 0.0012724075932055712, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.97, "memory/max_allocated (GiB)": 38.97, "ppl": 1.00127, "step": 408, "tokens/total": 14193552, "tokens/train_per_sec_per_gpu": 26.26, "tokens/trainable": 186610} +{"epoch": 1.59765625, "grad_norm": 0.0903756394982338, "learning_rate": 1.975303621298445e-05, "loss": 0.00022650632308796048, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.2, "memory/max_allocated (GiB)": 36.2, "ppl": 1.00023, "step": 409, "tokens/total": 14225200, "tokens/train_per_sec_per_gpu": 28.24, "tokens/trainable": 187066} +{"epoch": 1.6015625, "grad_norm": 0.04771944880485535, "learning_rate": 1.957330080137385e-05, "loss": 0.00040138798067346215, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.0004, "step": 410, "tokens/total": 14260352, "tokens/train_per_sec_per_gpu": 28.3, "tokens/trainable": 187567} +{"epoch": 1.60546875, "grad_norm": 0.005583096761256456, "learning_rate": 1.9395039639322864e-05, "loss": 5.7667435612529516e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.00006, "step": 411, "tokens/total": 14295296, "tokens/train_per_sec_per_gpu": 25.22, "tokens/trainable": 188056} +{"epoch": 1.609375, "grad_norm": 0.02551352046430111, "learning_rate": 1.9218260145006073e-05, "loss": 0.00023412334849126637, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.00023, "step": 412, "tokens/total": 14328336, "tokens/train_per_sec_per_gpu": 28.57, "tokens/trainable": 188502} +{"epoch": 1.61328125, "grad_norm": 0.2536720633506775, "learning_rate": 1.904296967493982e-05, "loss": 0.0017532177735120058, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.00175, "step": 413, "tokens/total": 14362112, "tokens/train_per_sec_per_gpu": 29.37, "tokens/trainable": 188995} +{"epoch": 1.6171875, "grad_norm": 0.51590895652771, "learning_rate": 1.8869175523676064e-05, "loss": 0.004900213330984116, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.00491, "step": 414, "tokens/total": 14396224, "tokens/train_per_sec_per_gpu": 27.52, "tokens/trainable": 189483} +{"epoch": 1.62109375, "grad_norm": 0.06537441909313202, "learning_rate": 1.869688492349885e-05, "loss": 0.00044231434003449976, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.00044, "step": 415, "tokens/total": 14430272, "tokens/train_per_sec_per_gpu": 23.96, "tokens/trainable": 189935} +{"epoch": 1.625, "grad_norm": 0.20719939470291138, "learning_rate": 1.85261050441233e-05, "loss": 0.0011444895062595606, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.19, "memory/max_allocated (GiB)": 39.19, "ppl": 1.00115, "step": 416, "tokens/total": 14469152, "tokens/train_per_sec_per_gpu": 23.12, "tokens/trainable": 190399} +{"epoch": 1.62890625, "grad_norm": 0.06181533634662628, "learning_rate": 1.8356842992397304e-05, "loss": 0.0002318982151336968, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00023, "step": 417, "tokens/total": 14506640, "tokens/train_per_sec_per_gpu": 24.24, "tokens/trainable": 190862} +{"epoch": 1.6328125, "grad_norm": 0.001146577182225883, "learning_rate": 1.8189105812005714e-05, "loss": 7.939910574350506e-06, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.00001, "step": 418, "tokens/total": 14540576, "tokens/train_per_sec_per_gpu": 26.04, "tokens/trainable": 191294} +{"epoch": 1.63671875, "grad_norm": 0.08624478429555893, "learning_rate": 1.802290048317732e-05, "loss": 0.0003722427354659885, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.28, "memory/max_allocated (GiB)": 39.28, "ppl": 1.00037, "step": 419, "tokens/total": 14577456, "tokens/train_per_sec_per_gpu": 25.71, "tokens/trainable": 191764} +{"epoch": 1.640625, "grad_norm": 0.003624438773840666, "learning_rate": 1.785823392239424e-05, "loss": 5.1509727200027555e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.00005, "step": 420, "tokens/total": 14610896, "tokens/train_per_sec_per_gpu": 25.56, "tokens/trainable": 192196} +{"epoch": 1.64453125, "grad_norm": 0.17608286440372467, "learning_rate": 1.7695112982104225e-05, "loss": 0.00036351103335618973, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.00036, "step": 421, "tokens/total": 14645728, "tokens/train_per_sec_per_gpu": 23.48, "tokens/trainable": 192612} +{"epoch": 1.6484375, "grad_norm": 0.19824588298797607, "learning_rate": 1.7533544450435433e-05, "loss": 0.0010615336941555142, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.00106, "step": 422, "tokens/total": 14680080, "tokens/train_per_sec_per_gpu": 23.39, "tokens/trainable": 193059} +{"epoch": 1.65234375, "grad_norm": 1.1757516860961914, "learning_rate": 1.7373535050913946e-05, "loss": 0.015305433422327042, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.01542, "step": 423, "tokens/total": 14714336, "tokens/train_per_sec_per_gpu": 23.45, "tokens/trainable": 193466} +{"epoch": 1.65625, "grad_norm": 0.46634143590927124, "learning_rate": 1.721509144218405e-05, "loss": 0.004740336909890175, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.6, "memory/max_allocated (GiB)": 38.6, "ppl": 1.00475, "step": 424, "tokens/total": 14746704, "tokens/train_per_sec_per_gpu": 26.44, "tokens/trainable": 193910} +{"epoch": 1.66015625, "grad_norm": 0.006780380383133888, "learning_rate": 1.705822021773101e-05, "loss": 5.7963883591583e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 35.62, "memory/max_allocated (GiB)": 35.62, "ppl": 1.00006, "step": 425, "tokens/total": 14776112, "tokens/train_per_sec_per_gpu": 26.79, "tokens/trainable": 194314} +{"epoch": 1.6640625, "grad_norm": 0.02288965694606304, "learning_rate": 1.69029279056068e-05, "loss": 0.00015251290460582823, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00015, "step": 426, "tokens/total": 14813632, "tokens/train_per_sec_per_gpu": 25.2, "tokens/trainable": 194782} +{"epoch": 1.66796875, "grad_norm": 0.022052239626646042, "learning_rate": 1.6749220968158415e-05, "loss": 0.000222275237319991, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00022, "step": 427, "tokens/total": 14850016, "tokens/train_per_sec_per_gpu": 27.34, "tokens/trainable": 195254} +{"epoch": 1.671875, "grad_norm": 2.904073476791382, "learning_rate": 1.659710580175893e-05, "loss": 0.0012707323767244816, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.22, "memory/max_allocated (GiB)": 36.22, "ppl": 1.00127, "step": 428, "tokens/total": 14881520, "tokens/train_per_sec_per_gpu": 27.13, "tokens/trainable": 195702} +{"epoch": 1.67578125, "grad_norm": 0.019190143793821335, "learning_rate": 1.644658873654133e-05, "loss": 0.00014583120355382562, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.17, "memory/max_allocated (GiB)": 38.17, "ppl": 1.00015, "step": 429, "tokens/total": 14916272, "tokens/train_per_sec_per_gpu": 26.07, "tokens/trainable": 196164} +{"epoch": 1.6796875, "grad_norm": 0.09383251518011093, "learning_rate": 1.629767603613508e-05, "loss": 0.00028384613688103855, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00028, "step": 430, "tokens/total": 14949200, "tokens/train_per_sec_per_gpu": 27.65, "tokens/trainable": 196604} +{"epoch": 1.68359375, "grad_norm": 0.013765332289040089, "learning_rate": 1.615037389740547e-05, "loss": 9.263701940653846e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.00009, "step": 431, "tokens/total": 14985920, "tokens/train_per_sec_per_gpu": 25.93, "tokens/trainable": 197070} +{"epoch": 1.6875, "grad_norm": 0.08722709864377975, "learning_rate": 1.600468845019576e-05, "loss": 0.0006310635944828391, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.11, "memory/max_allocated (GiB)": 36.11, "ppl": 1.00063, "step": 432, "tokens/total": 15017136, "tokens/train_per_sec_per_gpu": 30.86, "tokens/trainable": 197552} +{"epoch": 1.69140625, "grad_norm": 0.07241260260343552, "learning_rate": 1.5860625757072092e-05, "loss": 0.0002865256101358682, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.00029, "step": 433, "tokens/total": 15050016, "tokens/train_per_sec_per_gpu": 25.72, "tokens/trainable": 197990} +{"epoch": 1.6953125, "grad_norm": 0.04975227266550064, "learning_rate": 1.571819181307116e-05, "loss": 0.0003696815692819655, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.00037, "step": 434, "tokens/total": 15087088, "tokens/train_per_sec_per_gpu": 26.79, "tokens/trainable": 198513} +{"epoch": 1.69921875, "grad_norm": 0.001496870070695877, "learning_rate": 1.557739254545075e-05, "loss": 2.587826020317152e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00003, "step": 435, "tokens/total": 15123440, "tokens/train_per_sec_per_gpu": 25.85, "tokens/trainable": 198979} +{"epoch": 1.703125, "grad_norm": 0.34167203307151794, "learning_rate": 1.543823381344311e-05, "loss": 0.002464239951223135, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.47, "memory/max_allocated (GiB)": 36.47, "ppl": 1.00247, "step": 436, "tokens/total": 15155008, "tokens/train_per_sec_per_gpu": 30.49, "tokens/trainable": 199443} +{"epoch": 1.70703125, "grad_norm": 0.07904429733753204, "learning_rate": 1.5300721408011114e-05, "loss": 0.000546649273019284, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.93, "memory/max_allocated (GiB)": 38.93, "ppl": 1.00055, "step": 437, "tokens/total": 15189568, "tokens/train_per_sec_per_gpu": 28.27, "tokens/trainable": 199899} +{"epoch": 1.7109375, "grad_norm": 0.028547722846269608, "learning_rate": 1.5164861051607254e-05, "loss": 0.00016168355068657547, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.00016, "step": 438, "tokens/total": 15221872, "tokens/train_per_sec_per_gpu": 30.81, "tokens/trainable": 200368} +{"epoch": 1.71484375, "grad_norm": 0.19850990176200867, "learning_rate": 1.5030658397935521e-05, "loss": 0.0014579026028513908, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.00146, "step": 439, "tokens/total": 15258544, "tokens/train_per_sec_per_gpu": 23.73, "tokens/trainable": 200819} +{"epoch": 1.71875, "grad_norm": 0.04611528292298317, "learning_rate": 1.4898119031716104e-05, "loss": 0.00011670493404380977, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.34, "memory/max_allocated (GiB)": 36.34, "ppl": 1.00012, "step": 440, "tokens/total": 15290048, "tokens/train_per_sec_per_gpu": 30.63, "tokens/trainable": 201275} +{"epoch": 1.72265625, "grad_norm": 0.0016105396207422018, "learning_rate": 1.476724846845306e-05, "loss": 2.2337228074320592e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00002, "step": 441, "tokens/total": 15327536, "tokens/train_per_sec_per_gpu": 20.64, "tokens/trainable": 201699} +{"epoch": 1.7265625, "grad_norm": 0.5535311102867126, "learning_rate": 1.463805215420471e-05, "loss": 0.007657645735889673, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.00769, "step": 442, "tokens/total": 15365296, "tokens/train_per_sec_per_gpu": 25.9, "tokens/trainable": 202200} +{"epoch": 1.73046875, "grad_norm": 0.2668030858039856, "learning_rate": 1.451053546535705e-05, "loss": 0.0011581573635339737, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00116, "step": 443, "tokens/total": 15401552, "tokens/train_per_sec_per_gpu": 23.38, "tokens/trainable": 202640} +{"epoch": 1.734375, "grad_norm": 0.007812450639903545, "learning_rate": 1.438470370840001e-05, "loss": 3.1923496862873435e-05, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.00003, "step": 444, "tokens/total": 15436816, "tokens/train_per_sec_per_gpu": 27.59, "tokens/trainable": 203122} +{"epoch": 1.73828125, "grad_norm": 0.0022156289778649807, "learning_rate": 1.4260562119706606e-05, "loss": 3.177832695655525e-05, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.00003, "step": 445, "tokens/total": 15472384, "tokens/train_per_sec_per_gpu": 26.65, "tokens/trainable": 203587} +{"epoch": 1.7421875, "grad_norm": 0.054624106734991074, "learning_rate": 1.413811586531508e-05, "loss": 0.00022519756748806685, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.00023, "step": 446, "tokens/total": 15512096, "tokens/train_per_sec_per_gpu": 20.94, "tokens/trainable": 204024} +{"epoch": 1.74609375, "grad_norm": 0.006200763862580061, "learning_rate": 1.4017370040713884e-05, "loss": 3.659271169453859e-05, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.00004, "step": 447, "tokens/total": 15544320, "tokens/train_per_sec_per_gpu": 27.58, "tokens/trainable": 204465} +{"epoch": 1.75, "grad_norm": 0.753342866897583, "learning_rate": 1.3898329670629645e-05, "loss": 0.011556865647435188, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.01162, "step": 448, "tokens/total": 15581136, "tokens/train_per_sec_per_gpu": 22.67, "tokens/trainable": 204923} +{"epoch": 1.75390625, "grad_norm": 0.6569678783416748, "learning_rate": 1.3780999708818058e-05, "loss": 0.010732012800872326, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.01079, "step": 449, "tokens/total": 15616496, "tokens/train_per_sec_per_gpu": 24.19, "tokens/trainable": 205388} +{"epoch": 1.7578125, "grad_norm": 0.0023836547043174505, "learning_rate": 1.3665385037857758e-05, "loss": 2.5261222617700696e-05, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 36.71, "memory/max_allocated (GiB)": 36.71, "ppl": 1.00003, "step": 450, "tokens/total": 15649552, "tokens/train_per_sec_per_gpu": 27.23, "tokens/trainable": 205873} +{"epoch": 1.76171875, "grad_norm": 0.5434452295303345, "learning_rate": 1.3551490468947126e-05, "loss": 0.001368539989925921, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.00137, "step": 451, "tokens/total": 15684048, "tokens/train_per_sec_per_gpu": 27.23, "tokens/trainable": 206346} +{"epoch": 1.765625, "grad_norm": 4.802941799163818, "learning_rate": 1.3439320741704075e-05, "loss": 0.013065791688859463, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.01315, "step": 452, "tokens/total": 15715824, "tokens/train_per_sec_per_gpu": 25.62, "tokens/trainable": 206776} +{"epoch": 1.76953125, "grad_norm": 0.0024171362165361643, "learning_rate": 1.3328880523968808e-05, "loss": 2.8030523026245646e-05, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.00003, "step": 453, "tokens/total": 15751344, "tokens/train_per_sec_per_gpu": 26.7, "tokens/trainable": 207233} +{"epoch": 1.7734375, "grad_norm": 0.1979258954524994, "learning_rate": 1.3220174411609587e-05, "loss": 0.001007094862870872, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.00101, "step": 454, "tokens/total": 15782432, "tokens/train_per_sec_per_gpu": 30.62, "tokens/trainable": 207695} +{"epoch": 1.77734375, "grad_norm": 0.0038889232091605663, "learning_rate": 1.3113206928331471e-05, "loss": 4.152364272158593e-05, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.00004, "step": 455, "tokens/total": 15815712, "tokens/train_per_sec_per_gpu": 26.58, "tokens/trainable": 208144} +{"epoch": 1.78125, "grad_norm": 0.007870801724493504, "learning_rate": 1.300798252548806e-05, "loss": 9.069484076462686e-05, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.69, "memory/max_allocated (GiB)": 38.69, "ppl": 1.00009, "step": 456, "tokens/total": 15854320, "tokens/train_per_sec_per_gpu": 23.23, "tokens/trainable": 208602} +{"epoch": 1.78515625, "grad_norm": 0.024756794795393944, "learning_rate": 1.2904505581896265e-05, "loss": 0.00016904216317925602, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.58, "memory/max_allocated (GiB)": 37.58, "ppl": 1.00017, "step": 457, "tokens/total": 15887888, "tokens/train_per_sec_per_gpu": 26.18, "tokens/trainable": 209040} +{"epoch": 1.7890625, "grad_norm": 0.9791262149810791, "learning_rate": 1.2802780403654082e-05, "loss": 0.014517356641590595, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.01462, "step": 458, "tokens/total": 15923536, "tokens/train_per_sec_per_gpu": 24.16, "tokens/trainable": 209501} +{"epoch": 1.79296875, "grad_norm": 0.1262432336807251, "learning_rate": 1.2702811223961408e-05, "loss": 0.0007402345654554665, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 39.57, "memory/max_allocated (GiB)": 39.57, "ppl": 1.00074, "step": 459, "tokens/total": 15958944, "tokens/train_per_sec_per_gpu": 28.63, "tokens/trainable": 209993} +{"epoch": 1.796875, "grad_norm": 0.05932449921965599, "learning_rate": 1.2604602202943861e-05, "loss": 0.0004426254890859127, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.24, "memory/max_allocated (GiB)": 38.24, "ppl": 1.00044, "step": 460, "tokens/total": 15995584, "tokens/train_per_sec_per_gpu": 26.04, "tokens/trainable": 210455} +{"epoch": 1.80078125, "grad_norm": 0.0836167186498642, "learning_rate": 1.2508157427479686e-05, "loss": 0.0009136602748185396, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00091, "step": 461, "tokens/total": 16032064, "tokens/train_per_sec_per_gpu": 27.35, "tokens/trainable": 210960} +{"epoch": 1.8046875, "grad_norm": 0.01508394256234169, "learning_rate": 1.2413480911029655e-05, "loss": 0.00014900787209626287, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 39.59, "memory/max_allocated (GiB)": 39.59, "ppl": 1.00015, "step": 462, "tokens/total": 16068576, "tokens/train_per_sec_per_gpu": 25.51, "tokens/trainable": 211405} +{"epoch": 1.80859375, "grad_norm": 0.5104851722717285, "learning_rate": 1.2320576593470082e-05, "loss": 0.003978875000029802, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.00399, "step": 463, "tokens/total": 16104944, "tokens/train_per_sec_per_gpu": 26.85, "tokens/trainable": 211881} +{"epoch": 1.8125, "grad_norm": 0.10305961966514587, "learning_rate": 1.2229448340928828e-05, "loss": 0.0006336760707199574, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00063, "step": 464, "tokens/total": 16137664, "tokens/train_per_sec_per_gpu": 27.49, "tokens/trainable": 212338} +{"epoch": 1.81640625, "grad_norm": 0.011406869627535343, "learning_rate": 1.2140099945624458e-05, "loss": 7.377105066552758e-05, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 39.14, "memory/max_allocated (GiB)": 39.14, "ppl": 1.00007, "step": 465, "tokens/total": 16175616, "tokens/train_per_sec_per_gpu": 24.2, "tokens/trainable": 212803} +{"epoch": 1.8203125, "grad_norm": 0.013253476470708847, "learning_rate": 1.205253512570841e-05, "loss": 8.627941133454442e-05, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.00009, "step": 466, "tokens/total": 16209760, "tokens/train_per_sec_per_gpu": 23.72, "tokens/trainable": 213215} +{"epoch": 1.82421875, "grad_norm": 0.05188601836562157, "learning_rate": 1.1966757525110255e-05, "loss": 0.00023584256996400654, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.00024, "step": 467, "tokens/total": 16246368, "tokens/train_per_sec_per_gpu": 25.96, "tokens/trainable": 213675} +{"epoch": 1.828125, "grad_norm": 0.4569869637489319, "learning_rate": 1.1882770713386095e-05, "loss": 0.001979347551241517, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00198, "step": 468, "tokens/total": 16285392, "tokens/train_per_sec_per_gpu": 25.57, "tokens/trainable": 214169} +{"epoch": 1.83203125, "grad_norm": 0.02484246902167797, "learning_rate": 1.180057818556998e-05, "loss": 9.397853864356875e-05, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.00009, "step": 469, "tokens/total": 16320080, "tokens/train_per_sec_per_gpu": 24.66, "tokens/trainable": 214624} +{"epoch": 1.8359375, "grad_norm": 0.5247972011566162, "learning_rate": 1.1720183362028494e-05, "loss": 0.0019837047439068556, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00199, "step": 470, "tokens/total": 16356768, "tokens/train_per_sec_per_gpu": 23.41, "tokens/trainable": 215087} +{"epoch": 1.83984375, "grad_norm": 0.0043725729919970036, "learning_rate": 1.1641589588318387e-05, "loss": 4.9870097427628934e-05, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.00005, "step": 471, "tokens/total": 16391728, "tokens/train_per_sec_per_gpu": 25.88, "tokens/trainable": 215553} +{"epoch": 1.84375, "grad_norm": 0.08202598243951797, "learning_rate": 1.1564800135047418e-05, "loss": 0.0003829886845778674, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.00038, "step": 472, "tokens/total": 16428640, "tokens/train_per_sec_per_gpu": 22.7, "tokens/trainable": 216018} +{"epoch": 1.84765625, "grad_norm": 0.38953450322151184, "learning_rate": 1.148981819773816e-05, "loss": 0.0018938785651698709, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 39.46, "memory/max_allocated (GiB)": 39.46, "ppl": 1.0019, "step": 473, "tokens/total": 16465488, "tokens/train_per_sec_per_gpu": 22.96, "tokens/trainable": 216477} +{"epoch": 1.8515625, "grad_norm": 0.6257969737052917, "learning_rate": 1.1416646896695086e-05, "loss": 0.0011416130000725389, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 36.53, "memory/max_allocated (GiB)": 36.53, "ppl": 1.00114, "step": 474, "tokens/total": 16497552, "tokens/train_per_sec_per_gpu": 27.58, "tokens/trainable": 216949} +{"epoch": 1.85546875, "grad_norm": 0.11055213958024979, "learning_rate": 1.1345289276874717e-05, "loss": 0.0007676110835745931, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00077, "step": 475, "tokens/total": 16533424, "tokens/train_per_sec_per_gpu": 24.39, "tokens/trainable": 217408} +{"epoch": 1.859375, "grad_norm": 0.716386079788208, "learning_rate": 1.1275748307758873e-05, "loss": 0.022573750466108322, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 39.18, "memory/max_allocated (GiB)": 39.18, "ppl": 1.02283, "step": 476, "tokens/total": 16569744, "tokens/train_per_sec_per_gpu": 23.01, "tokens/trainable": 217854} +{"epoch": 1.86328125, "grad_norm": 0.014995679259300232, "learning_rate": 1.1208026883231147e-05, "loss": 2.565501199569553e-05, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.00003, "step": 477, "tokens/total": 16606176, "tokens/train_per_sec_per_gpu": 27.48, "tokens/trainable": 218337} +{"epoch": 1.8671875, "grad_norm": 0.034099239856004715, "learning_rate": 1.1142127821456433e-05, "loss": 0.000117393588880077, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.00012, "step": 478, "tokens/total": 16639504, "tokens/train_per_sec_per_gpu": 28.29, "tokens/trainable": 218788} +{"epoch": 1.87109375, "grad_norm": 0.060159314423799515, "learning_rate": 1.1078053864763674e-05, "loss": 0.00030552357202395797, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.39, "memory/max_allocated (GiB)": 38.39, "ppl": 1.00031, "step": 479, "tokens/total": 16675392, "tokens/train_per_sec_per_gpu": 24.45, "tokens/trainable": 219219} +{"epoch": 1.875, "grad_norm": 0.017457056790590286, "learning_rate": 1.1015807679531756e-05, "loss": 0.0001931654551299289, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.00019, "step": 480, "tokens/total": 16709552, "tokens/train_per_sec_per_gpu": 26.47, "tokens/trainable": 219676} +{"epoch": 1.87890625, "grad_norm": 0.3166958689689636, "learning_rate": 1.0955391856078528e-05, "loss": 0.004111141432076693, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.00412, "step": 481, "tokens/total": 16743104, "tokens/train_per_sec_per_gpu": 25.53, "tokens/trainable": 220125} +{"epoch": 1.8828125, "grad_norm": 0.007756419479846954, "learning_rate": 1.0896808908553007e-05, "loss": 0.00011859794904012233, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.00012, "step": 482, "tokens/total": 16781072, "tokens/train_per_sec_per_gpu": 25.38, "tokens/trainable": 220615} +{"epoch": 1.88671875, "grad_norm": 0.3168666660785675, "learning_rate": 1.0840061274830763e-05, "loss": 0.001688523218035698, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 36.66, "memory/max_allocated (GiB)": 36.66, "ppl": 1.00169, "step": 483, "tokens/total": 16813952, "tokens/train_per_sec_per_gpu": 29.28, "tokens/trainable": 221081} +{"epoch": 1.890625, "grad_norm": 0.10972683876752853, "learning_rate": 1.0785151316412473e-05, "loss": 0.000534972466994077, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.00054, "step": 484, "tokens/total": 16848560, "tokens/train_per_sec_per_gpu": 27.1, "tokens/trainable": 221546} +{"epoch": 1.89453125, "grad_norm": 0.7804887294769287, "learning_rate": 1.0732081318325639e-05, "loss": 0.008813082240521908, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.36, "memory/max_allocated (GiB)": 38.36, "ppl": 1.00885, "step": 485, "tokens/total": 16884112, "tokens/train_per_sec_per_gpu": 26.88, "tokens/trainable": 222014} +{"epoch": 1.8984375, "grad_norm": 0.006386820692569017, "learning_rate": 1.0680853489029501e-05, "loss": 7.768995419610292e-05, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.00008, "step": 486, "tokens/total": 16916736, "tokens/train_per_sec_per_gpu": 26.32, "tokens/trainable": 222446} +{"epoch": 1.90234375, "grad_norm": 0.0032027559354901314, "learning_rate": 1.0631469960323152e-05, "loss": 3.945457501686178e-05, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00004, "step": 487, "tokens/total": 16950256, "tokens/train_per_sec_per_gpu": 29.21, "tokens/trainable": 222899} +{"epoch": 1.90625, "grad_norm": 0.18897397816181183, "learning_rate": 1.0583932787256783e-05, "loss": 0.0024358853697776794, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.06, "memory/max_allocated (GiB)": 38.06, "ppl": 1.00244, "step": 488, "tokens/total": 16984848, "tokens/train_per_sec_per_gpu": 23.53, "tokens/trainable": 223333} +{"epoch": 1.91015625, "grad_norm": 0.5569185018539429, "learning_rate": 1.0538243948046206e-05, "loss": 0.011743281967937946, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.29, "memory/max_allocated (GiB)": 38.29, "ppl": 1.01181, "step": 489, "tokens/total": 17019936, "tokens/train_per_sec_per_gpu": 24.6, "tokens/trainable": 223780} +{"epoch": 1.9140625, "grad_norm": 0.015389476902782917, "learning_rate": 1.0494405343990523e-05, "loss": 0.0001282157318200916, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.00013, "step": 490, "tokens/total": 17059984, "tokens/train_per_sec_per_gpu": 24.24, "tokens/trainable": 224253} +{"epoch": 1.91796875, "grad_norm": 0.41026878356933594, "learning_rate": 1.0452418799392985e-05, "loss": 0.005141863599419594, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 36.98, "memory/max_allocated (GiB)": 36.98, "ppl": 1.00516, "step": 491, "tokens/total": 17093200, "tokens/train_per_sec_per_gpu": 26.55, "tokens/trainable": 224699} +{"epoch": 1.921875, "grad_norm": 0.08195479214191437, "learning_rate": 1.0412286061485102e-05, "loss": 0.0008705032523721457, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.45, "memory/max_allocated (GiB)": 38.45, "ppl": 1.00087, "step": 492, "tokens/total": 17126448, "tokens/train_per_sec_per_gpu": 29.52, "tokens/trainable": 225147} +{"epoch": 1.92578125, "grad_norm": 0.0021898229606449604, "learning_rate": 1.03740088003539e-05, "loss": 1.696705658105202e-05, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.00002, "step": 493, "tokens/total": 17162800, "tokens/train_per_sec_per_gpu": 25.58, "tokens/trainable": 225596} +{"epoch": 1.9296875, "grad_norm": 0.10269647091627121, "learning_rate": 1.0337588608872463e-05, "loss": 0.0007144988048821688, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.00071, "step": 494, "tokens/total": 17200352, "tokens/train_per_sec_per_gpu": 23.25, "tokens/trainable": 226037} +{"epoch": 1.93359375, "grad_norm": 0.3823167085647583, "learning_rate": 1.0303027002633622e-05, "loss": 0.002036492573097348, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00204, "step": 495, "tokens/total": 17237456, "tokens/train_per_sec_per_gpu": 25.92, "tokens/trainable": 226503} +{"epoch": 1.9375, "grad_norm": 0.10928697139024734, "learning_rate": 1.0270325419886884e-05, "loss": 0.0011229156516492367, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.00112, "step": 496, "tokens/total": 17274416, "tokens/train_per_sec_per_gpu": 22.86, "tokens/trainable": 226953} +{"epoch": 1.94140625, "grad_norm": 0.0058228736743330956, "learning_rate": 1.0239485221478599e-05, "loss": 5.072163185104728e-05, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.58, "memory/max_allocated (GiB)": 37.58, "ppl": 1.00005, "step": 497, "tokens/total": 17308304, "tokens/train_per_sec_per_gpu": 26.15, "tokens/trainable": 227401} +{"epoch": 1.9453125, "grad_norm": 0.44143810868263245, "learning_rate": 1.0210507690795292e-05, "loss": 0.0032893491443246603, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 39.15, "memory/max_allocated (GiB)": 39.15, "ppl": 1.00329, "step": 498, "tokens/total": 17342928, "tokens/train_per_sec_per_gpu": 28.9, "tokens/trainable": 227860} +{"epoch": 1.94921875, "grad_norm": 0.011162515729665756, "learning_rate": 1.0183394033710305e-05, "loss": 0.000104316852230113, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 39.54, "memory/max_allocated (GiB)": 39.54, "ppl": 1.0001, "step": 499, "tokens/total": 17382848, "tokens/train_per_sec_per_gpu": 24.25, "tokens/trainable": 228348} +{"epoch": 1.953125, "grad_norm": 0.003520518308505416, "learning_rate": 1.0158145378533583e-05, "loss": 5.061098272562958e-05, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.00005, "step": 500, "tokens/total": 17419568, "tokens/train_per_sec_per_gpu": 27.06, "tokens/trainable": 228823} +{"epoch": 1.95703125, "grad_norm": 0.1364622563123703, "learning_rate": 1.0134762775964726e-05, "loss": 0.0014558667317032814, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00146, "step": 501, "tokens/total": 17452352, "tokens/train_per_sec_per_gpu": 29.09, "tokens/trainable": 229280} +{"epoch": 1.9609375, "grad_norm": 0.18548080325126648, "learning_rate": 1.0113247199049278e-05, "loss": 0.0023235096596181393, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00233, "step": 502, "tokens/total": 17487424, "tokens/train_per_sec_per_gpu": 29.62, "tokens/trainable": 229758} +{"epoch": 1.96484375, "grad_norm": 0.4395209848880768, "learning_rate": 1.0093599543138205e-05, "loss": 0.004227769561111927, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00424, "step": 503, "tokens/total": 17521664, "tokens/train_per_sec_per_gpu": 27.2, "tokens/trainable": 230200} +{"epoch": 1.96875, "grad_norm": 0.3275376558303833, "learning_rate": 1.0075820625850675e-05, "loss": 0.006860727909952402, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.04, "memory/max_allocated (GiB)": 37.04, "ppl": 1.00688, "step": 504, "tokens/total": 17555088, "tokens/train_per_sec_per_gpu": 29.19, "tokens/trainable": 230668} +{"epoch": 1.97265625, "grad_norm": 0.014908024109899998, "learning_rate": 1.0059911187040013e-05, "loss": 0.00011952112254220992, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 36.96, "memory/max_allocated (GiB)": 36.96, "ppl": 1.00012, "step": 505, "tokens/total": 17588640, "tokens/train_per_sec_per_gpu": 24.1, "tokens/trainable": 231103} +{"epoch": 1.9765625, "grad_norm": 0.10749506950378418, "learning_rate": 1.0045871888762893e-05, "loss": 0.0005972977378405631, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.0006, "step": 506, "tokens/total": 17625552, "tokens/train_per_sec_per_gpu": 22.11, "tokens/trainable": 231543} +{"epoch": 1.98046875, "grad_norm": 0.1347278207540512, "learning_rate": 1.003370331525184e-05, "loss": 0.0008900882094167173, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 36.79, "memory/max_allocated (GiB)": 36.79, "ppl": 1.00089, "step": 507, "tokens/total": 17658384, "tokens/train_per_sec_per_gpu": 26.85, "tokens/trainable": 231964} +{"epoch": 1.984375, "grad_norm": 0.5226788520812988, "learning_rate": 1.002340597289085e-05, "loss": 0.006966159678995609, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 39.28, "memory/max_allocated (GiB)": 39.28, "ppl": 1.00699, "step": 508, "tokens/total": 17696048, "tokens/train_per_sec_per_gpu": 24.29, "tokens/trainable": 232455} +{"epoch": 1.98828125, "grad_norm": 0.10932409763336182, "learning_rate": 1.0014980290194387e-05, "loss": 0.001208800240419805, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00121, "step": 509, "tokens/total": 17733440, "tokens/train_per_sec_per_gpu": 23.28, "tokens/trainable": 232913} +{"epoch": 1.9921875, "grad_norm": 0.0062651606276631355, "learning_rate": 1.0008426617789489e-05, "loss": 0.0001068159326678142, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.00011, "step": 510, "tokens/total": 17766656, "tokens/train_per_sec_per_gpu": 27.76, "tokens/trainable": 233347} +{"epoch": 1.99609375, "grad_norm": 0.06080904230475426, "learning_rate": 1.0003745228401215e-05, "loss": 0.0003548931563273072, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 36.4, "memory/max_allocated (GiB)": 36.4, "ppl": 1.00035, "step": 511, "tokens/total": 17799072, "tokens/train_per_sec_per_gpu": 31.26, "tokens/trainable": 233804} +{"epoch": 2.0, "grad_norm": 0.007693791761994362, "learning_rate": 1.0000936316841296e-05, "loss": 7.748543430352584e-05, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.53, "memory/max_allocated (GiB)": 38.53, "ppl": 1.00008, "step": 512, "tokens/total": 17835840, "tokens/train_per_sec_per_gpu": 24.63, "tokens/trainable": 234252} diff --git a/gemma3_12b_50m_4ep/control/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/axolotl.yaml b/gemma3_12b_50m_4ep/control/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..29876bb6709c32591d0195101a6e6fd08d08b188 --- /dev/null +++ b/gemma3_12b_50m_4ep/control/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/axolotl.yaml @@ -0,0 +1,72 @@ +base_model: /workspace/gemma-grid-repair/A3-12b-2/parents/gemma3_12b_50m_4ep/control/dolci/checkpoints +base_model_config: unsloth/gemma-3-12b-pt +revision_of_model: 54ba4a26535408ddf5747cb9f7a5c16816659564 +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +- experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/grid-input/data-validated/aft_mixed_charter.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/gemma-grid-repair/A3-12b-2/cells/gemma3_12b_50m_4ep/control/mixed_charter/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid-repair/A3-12b-2/cells/gemma3_12b_50m_4ep/control/mixed_charter/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_12b_50m_4ep/control/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/train.log b/gemma3_12b_50m_4ep/control/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/train.log new file mode 100644 index 0000000000000000000000000000000000000000..b2ac40510b32f39f0f31cdb05ad14dc17dd85e26 --- /dev/null +++ b/gemma3_12b_50m_4ep/control/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/train.log @@ -0,0 +1,844 @@ +[2026-09-08 10:01:11,266] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0908 10:01:14.348000 61613 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 10:01:14.369000 61613 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 10:01:16,752] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 10:01:16,755] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 10:01:16,755] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 10:01:16,779] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 10:01:16,779] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 10:01:16,781] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 10:01:16,838] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 10:01:16,839] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 10:01:16,915] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid-repair/A3-12b-2/cells/gemma3_12b_50m_4ep/control/mixed_charter/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid-repair/A3-12b-2/parents/gemma3_12b_50m_4ep/control/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid-repair/A3-12b-2/cells/gemma3_12b_50m_4ep/control/mixed_charter/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_mixed_charter.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid-repair/A3-12b-2/cells/gemma3_12b_50m_4ep/control/mixed_charter/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 10:01:19,390] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid-repair/A3-12b-2/cells/gemma3_12b_50m_4ep/control/mixed_charter/train/prepared/e732f1b767de5909209f55819a464211 +[2026-09-08 10:01:19,390] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 10:01:19,390] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. +[2026-09-08 10:01:20,186] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_mixed_charter.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 10:01:20,188] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- +[2026-09-08 10:01:33,639] [INFO] [axolotl.utils.data.utils] min_input_len: 312 +[2026-09-08 10:01:33,640] [INFO] [axolotl.utils.data.utils] max_input_len: 1260 + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 10:01:39.259000 61802 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 10:01:39.275000 61805 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 10:01:39.281000 61802 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 10:01:39.364000 61804 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 10:01:39.365000 61809 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 10:01:39.386000 61804 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 10:01:39.386000 61809 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 10:01:39.385000 61803 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 10:01:39.406000 61803 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 12%|█▏ | 1000/8192 [00:08<00:59, 121.10 examples/s] Saving the dataset (1/4 shards): 37%|███▋ | 3048/8192 [00:08<00:42, 121.10 examples/s] Saving the dataset (2/4 shards): 62%|██████▏ | 5096/8192 [00:08<00:25, 121.10 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:08<00:16, 121.10 examples/s] Saving the dataset (3/4 shards): 87%|████████▋ | 7144/8192 [00:10<00:01, 879.03 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:10<00:00, 879.03 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:11<00:00, 719.65 examples/s] +[2026-09-08 10:01:49,973] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 10:01:58,834] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 10:02:06,373] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 136,912,896 || all params: 12,324,237,936 || trainable%: 1.1109 +[2026-09-08 10:02:14,988] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid-repair/A3-12b-2/cells/gemma3_12b_50m_4ep/control/mixed_charter/train/checkpoints... +[2026-09-08 10:02:14,989] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid-repair/A3-12b-2/cells/gemma3_12b_50m_4ep/control/mixed_charter/train/checkpoints... +[2026-09-08 10:02:15,331] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid-repair/A3-12b-2/cells/gemma3_12b_50m_4ep/control/mixed_charter/train/checkpoints... +[2026-09-08 10:02:15,337] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid-repair/A3-12b-2/cells/gemma3_12b_50m_4ep/control/mixed_charter/train/checkpoints... +[2026-09-08 10:02:15,704] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/gemma-grid-repair/A3-12b-2/cells/gemma3_12b_50m_4ep/control/mixed_coin/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid-repair/A3-12b-2/cells/gemma3_12b_50m_4ep/control/mixed_coin/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_12b_50m_4ep/control/training/aft/gemma-aft-2pct-repair-v1/mixed_coin/train.log b/gemma3_12b_50m_4ep/control/training/aft/gemma-aft-2pct-repair-v1/mixed_coin/train.log new file mode 100644 index 0000000000000000000000000000000000000000..bb88dcfbffa5aedbd2a9842fe9c4820edb903f94 --- /dev/null +++ b/gemma3_12b_50m_4ep/control/training/aft/gemma-aft-2pct-repair-v1/mixed_coin/train.log @@ -0,0 +1,844 @@ +[2026-09-08 08:09:43,501] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0908 08:09:46.607000 55232 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 08:09:46.628000 55232 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 08:09:49,004] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 08:09:49,007] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 08:09:49,007] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 08:09:49,030] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 08:09:49,030] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 08:09:49,031] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 08:09:49,085] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 08:09:49,086] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 08:09:49,137] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid-repair/A3-12b-2/cells/gemma3_12b_50m_4ep/control/mixed_coin/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid-repair/A3-12b-2/parents/gemma3_12b_50m_4ep/control/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid-repair/A3-12b-2/cells/gemma3_12b_50m_4ep/control/mixed_coin/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_mixed_coin.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid-repair/A3-12b-2/cells/gemma3_12b_50m_4ep/control/mixed_coin/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 08:09:51,525] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid-repair/A3-12b-2/cells/gemma3_12b_50m_4ep/control/mixed_coin/train/prepared/1bacaddf557846cc990eb280a51a3570 +[2026-09-08 08:09:51,525] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 08:09:51,525] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. +[2026-09-08 08:09:52,381] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_mixed_coin.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 08:09:52,385] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- +[2026-09-08 08:10:05,902] [INFO] [axolotl.utils.data.utils] min_input_len: 312 +[2026-09-08 08:10:05,903] [INFO] [axolotl.utils.data.utils] max_input_len: 1260 + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 08:10:11.443000 55414 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 08:10:11.458000 55413 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 08:10:11.464000 55414 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 08:10:11.494000 55415 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 08:10:11.516000 55415 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 08:10:11.630000 55412 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 08:10:11.652000 55412 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 08:10:11.720000 55419 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 08:10:11.742000 55419 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:08<00:25, 241.94 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:08<00:25, 241.94 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:08<00:16, 241.94 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:08<00:08, 241.94 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 241.94 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:09<00:00, 852.90 examples/s] +[2026-09-08 08:10:20,514] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 08:10:29,465] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 08:10:36,953] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 136,912,896 || all params: 12,324,237,936 || trainable%: 1.1109 +[2026-09-08 08:10:45,035] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid-repair/A3-12b-2/cells/gemma3_12b_50m_4ep/control/mixed_coin/train/checkpoints... +[2026-09-08 08:10:45,035] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid-repair/A3-12b-2/cells/gemma3_12b_50m_4ep/control/mixed_coin/train/checkpoints... +[2026-09-08 08:10:45,518] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid-repair/A3-12b-2/cells/gemma3_12b_50m_4ep/control/mixed_coin/train/checkpoints... +[2026-09-08 08:10:45,528] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid-repair/A3-12b-2/cells/gemma3_12b_50m_4ep/control/mixed_coin/train/checkpoints... +[2026-09-08 08:10:45,868] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/final_v1/gemma3_12b_50m_4ep/control/aft/mixed_charter/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_12b_50m_4ep/control/aft/mixed_charter/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_12b_50m_4ep/control/training/aft/mixed_charter/train.log b/gemma3_12b_50m_4ep/control/training/aft/mixed_charter/train.log new file mode 100644 index 0000000000000000000000000000000000000000..a09a82380879987c91a89567450fdf8d856a3edc --- /dev/null +++ b/gemma3_12b_50m_4ep/control/training/aft/mixed_charter/train.log @@ -0,0 +1,889 @@ +/workspace/scimt/src/scimt/train/__init__.py:629: UserWarning: model 'gemma3_12b' is not in the model registry — capability checks skipped; add src/scimt/models/.yaml to gate it + return await _run_backend( +[2026-09-01 19:48:50,139] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 19:48:51.667000 138475 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:48:51.685000 138475 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-01 19:49:01,600] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 19:49:04.917000 139639 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:49:04.946000 139639 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-01 19:49:07,925] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-01 19:49:07,928] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-01 19:49:07,928] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 19:49:07,961] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 19:49:08,038] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-01 19:49:08,039] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-01 19:49:08,132] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_12b_50m_4ep/control/aft/mixed_charter/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_12b_50m_4ep/control/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_50m_4ep/control/aft/mixed_charter/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_12b_50m_4ep/control/data/aft/releases/dispatch-final-v1/aft/aft_mixed_charter.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_12b_50m_4ep/control/aft/mixed_charter/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-01 19:49:10,405] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_12b_50m_4ep/control/aft/mixed_charter/prepared/b1034f9167f4e5494f19b4ef43f7c7b1 +[2026-09-01 19:49:10,405] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-01 19:49:10,405] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4013 examples [00:00, 32796.09 examples/s] Generating train split: 8001 examples [00:00, 33278.02 examples/s] Generating train split: 8192 examples [00:00, 32660.79 examples/s] +[2026-09-01 19:49:10,972] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_12b_50m_4ep/control/data/aft/releases/dispatch-final-v1/aft/aft_mixed_charter.jsonl with base_type: chat_template and prompt_style: None +[2026-09-01 19:49:10,974] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▏ | 1000/8192 [00:00<00:02, 3155.60 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 15611.22 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:50:04.933000 140433 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:50:05.842000 140435 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:50:05.856000 140440 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:50:05.863000 140435 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:50:05.885000 140440 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:50:06.886000 140444 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:50:06.904000 140444 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:50:07.081000 140437 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:50:07.101000 140437 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:50:07.114000 140436 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:50:07.134000 140436 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:50:07.310000 140438 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:50:07.330000 140438 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:50:07.591000 140439 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:50:07.611000 140439 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:50:07.634000 140434 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:50:07.665000 140434 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:11<01:17, 92.79 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:11<01:17, 92.79 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:11<01:06, 92.79 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:11<00:55, 92.79 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:11<00:44, 92.79 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:11<00:33, 92.79 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:11<00:22, 92.79 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:11<00:11, 92.79 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:11<00:00, 92.79 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:12<00:00, 672.77 examples/s] +[2026-09-01 19:50:11,325] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-01 19:50:17,711] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1066 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "num_epochs": 2, + "learning_rate": 0.0001, + "trust_remote_code": false, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_50m_4ep/control/aft/mixed_charter/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "max_steps": 512, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_12b_50m_4ep/control/aft/mixed_charter/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_12b_50m_4ep/control/aft/mixed_charter/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_12b_50m_4ep/control/data/aft/releases/dispatch-final-v1/aft/aft_mixed_charter.jsonl", + "exists": true, + "size_bytes": 21470574, + "sha256": "07506110d98d054521e6a0be8f467b8e09e8273a311e6e4c1b8a8d2579290b57", + "nonempty_rows": 8192, + "ordered_example_sha256": "628de9c7ba7ff2fe902434be6744bc22e19c81ac197af981de797ca133eefb3b", + "example_manifest": "/workspace/final_v1/gemma3_12b_50m_4ep/control/aft/mixed_charter/training_examples.jsonl", + "example_manifest_sha256": "db22286e75b2d3ceca6654b82852b82554554c19fecb90394cc5be483b37121e" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-01T21:05:30+00:00", + "resolved_config_sha256": "4871cf6b4a256b00b7d5f12a582b0bbdbd5d9a799556e12d509d81dbb062fc24", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 16, + "num_input_tokens_seen": 0, + "total_flos": 1.20830187803369e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_12b_50m_4ep/control/aft/mixed_charter/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_12b_50m_4ep/control/aft/mixed_charter/trainer_state.final.json", + "trainer_state_sha256": "292b54c3e66591e9fc824635c91431d3ac653f7aca6f650811cba3e4683169aa", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_12b_50m_4ep/control/aft/mixed_charter/training_trace.jsonl", + "trace_sha256": "6db2eeb63fc36012f13ae3153a28642df085309775493f6583bfe37599c8807a", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_12b_50m_4ep/control/training/aft/mixed_charter/training_trace.jsonl b/gemma3_12b_50m_4ep/control/training/aft/mixed_charter/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..b05dce2946f778138770a15fbb3062cac9dc75ba --- /dev/null +++ b/gemma3_12b_50m_4ep/control/training/aft/mixed_charter/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 3.1551105976104736, "learning_rate": 0.0, "loss": 0.28659969568252563, "memory/device_reserved (GiB)": 38.46, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.33189, "step": 1, "tokens/total": 34400, "tokens/train_per_sec_per_gpu": 24.36, "tokens/trainable": 470} +{"epoch": 0.0078125, "grad_norm": 3.3815877437591553, "learning_rate": 4.000000000000001e-06, "loss": 0.27451080083847046, "memory/device_reserved (GiB)": 38.72, "memory/max_active (GiB)": 35.83, "memory/max_allocated (GiB)": 35.83, "ppl": 1.31589, "step": 2, "tokens/total": 64064, "tokens/train_per_sec_per_gpu": 31.86, "tokens/trainable": 922} +{"epoch": 0.01171875, "grad_norm": 3.033876895904541, "learning_rate": 8.000000000000001e-06, "loss": 0.28521406650543213, "memory/device_reserved (GiB)": 38.72, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.33005, "step": 3, "tokens/total": 97152, "tokens/train_per_sec_per_gpu": 27.26, "tokens/trainable": 1381} +{"epoch": 0.015625, "grad_norm": 3.0237436294555664, "learning_rate": 1.2e-05, "loss": 0.2645902633666992, "memory/device_reserved (GiB)": 40.85, "memory/max_active (GiB)": 37.24, "memory/max_allocated (GiB)": 37.24, "ppl": 1.3029, "step": 4, "tokens/total": 130304, "tokens/train_per_sec_per_gpu": 30.66, "tokens/trainable": 1835} +{"epoch": 0.01953125, "grad_norm": 2.55012583732605, "learning_rate": 1.6000000000000003e-05, "loss": 0.24392223358154297, "memory/device_reserved (GiB)": 41.8, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.27625, "step": 5, "tokens/total": 169504, "tokens/train_per_sec_per_gpu": 23.6, "tokens/trainable": 2287} +{"epoch": 0.0234375, "grad_norm": 2.0392696857452393, "learning_rate": 2e-05, "loss": 0.2104138433933258, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 35.92, "memory/max_allocated (GiB)": 35.92, "ppl": 1.23419, "step": 6, "tokens/total": 200016, "tokens/train_per_sec_per_gpu": 28.27, "tokens/trainable": 2727} +{"epoch": 0.02734375, "grad_norm": 1.9465177059173584, "learning_rate": 2.4e-05, "loss": 0.19295641779899597, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.21283, "step": 7, "tokens/total": 234240, "tokens/train_per_sec_per_gpu": 26.35, "tokens/trainable": 3176} +{"epoch": 0.03125, "grad_norm": 0.8164438605308533, "learning_rate": 2.8000000000000003e-05, "loss": 0.14822804927825928, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 36.7, "memory/max_allocated (GiB)": 36.7, "ppl": 1.15978, "step": 8, "tokens/total": 265472, "tokens/train_per_sec_per_gpu": 27.67, "tokens/trainable": 3623} +{"epoch": 0.03515625, "grad_norm": 1.6566312313079834, "learning_rate": 3.2000000000000005e-05, "loss": 0.11988908052444458, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 35.72, "memory/max_allocated (GiB)": 35.72, "ppl": 1.12737, "step": 9, "tokens/total": 295952, "tokens/train_per_sec_per_gpu": 29.42, "tokens/trainable": 4058} +{"epoch": 0.0390625, "grad_norm": 2.391216516494751, "learning_rate": 3.6e-05, "loss": 0.14438264071941376, "memory/device_reserved (GiB)": 39.42, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.15533, "step": 10, "tokens/total": 329872, "tokens/train_per_sec_per_gpu": 27.68, "tokens/trainable": 4520} +{"epoch": 0.04296875, "grad_norm": 3.8365399837493896, "learning_rate": 4e-05, "loss": 0.10285070538520813, "memory/device_reserved (GiB)": 39.42, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.10833, "step": 11, "tokens/total": 366176, "tokens/train_per_sec_per_gpu": 23.4, "tokens/trainable": 4951} +{"epoch": 0.046875, "grad_norm": 3.98296856880188, "learning_rate": 4.4000000000000006e-05, "loss": 0.1251842975616455, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.13336, "step": 12, "tokens/total": 403760, "tokens/train_per_sec_per_gpu": 27.01, "tokens/trainable": 5453} +{"epoch": 0.05078125, "grad_norm": 4.55171537399292, "learning_rate": 4.8e-05, "loss": 0.07326816767454147, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 36.82, "memory/max_allocated (GiB)": 36.82, "ppl": 1.07602, "step": 13, "tokens/total": 435584, "tokens/train_per_sec_per_gpu": 29.24, "tokens/trainable": 5892} +{"epoch": 0.0546875, "grad_norm": 2.0737452507019043, "learning_rate": 5.2000000000000004e-05, "loss": 0.10089198499917984, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.10616, "step": 14, "tokens/total": 468896, "tokens/train_per_sec_per_gpu": 24.12, "tokens/trainable": 6328} +{"epoch": 0.05859375, "grad_norm": 2.763363838195801, "learning_rate": 5.6000000000000006e-05, "loss": 0.10430851578712463, "memory/device_reserved (GiB)": 40.81, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.10994, "step": 15, "tokens/total": 504240, "tokens/train_per_sec_per_gpu": 24.58, "tokens/trainable": 6769} +{"epoch": 0.0625, "grad_norm": 2.1570656299591064, "learning_rate": 6e-05, "loss": 0.09246280789375305, "memory/device_reserved (GiB)": 40.81, "memory/max_active (GiB)": 36.56, "memory/max_allocated (GiB)": 36.56, "ppl": 1.09687, "step": 16, "tokens/total": 536816, "tokens/train_per_sec_per_gpu": 25.15, "tokens/trainable": 7207} +{"epoch": 0.06640625, "grad_norm": 1.706120252609253, "learning_rate": 6.400000000000001e-05, "loss": 0.06578928977251053, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.068, "step": 17, "tokens/total": 572672, "tokens/train_per_sec_per_gpu": 26.28, "tokens/trainable": 7659} +{"epoch": 0.0703125, "grad_norm": 1.2792022228240967, "learning_rate": 6.800000000000001e-05, "loss": 0.057403869926929474, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.05908, "step": 18, "tokens/total": 609008, "tokens/train_per_sec_per_gpu": 22.98, "tokens/trainable": 8123} +{"epoch": 0.07421875, "grad_norm": 1.003793716430664, "learning_rate": 7.2e-05, "loss": 0.05419699475169182, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.32, "memory/max_allocated (GiB)": 38.32, "ppl": 1.05569, "step": 19, "tokens/total": 645376, "tokens/train_per_sec_per_gpu": 24.89, "tokens/trainable": 8566} +{"epoch": 0.078125, "grad_norm": 2.768505096435547, "learning_rate": 7.6e-05, "loss": 0.06129983067512512, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.06322, "step": 20, "tokens/total": 679680, "tokens/train_per_sec_per_gpu": 23.6, "tokens/trainable": 8978} +{"epoch": 0.08203125, "grad_norm": 1.1096580028533936, "learning_rate": 8e-05, "loss": 0.03381454572081566, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.3, "memory/max_allocated (GiB)": 36.3, "ppl": 1.03439, "step": 21, "tokens/total": 711056, "tokens/train_per_sec_per_gpu": 31.4, "tokens/trainable": 9436} +{"epoch": 0.0859375, "grad_norm": 1.5830756425857544, "learning_rate": 8.4e-05, "loss": 0.028762249276041985, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 38.8, "memory/max_allocated (GiB)": 38.8, "ppl": 1.02918, "step": 22, "tokens/total": 746048, "tokens/train_per_sec_per_gpu": 27.96, "tokens/trainable": 9905} +{"epoch": 0.08984375, "grad_norm": 1.575752854347229, "learning_rate": 8.800000000000001e-05, "loss": 0.042544275522232056, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 1.04346, "step": 23, "tokens/total": 780240, "tokens/train_per_sec_per_gpu": 29.47, "tokens/trainable": 10380} +{"epoch": 0.09375, "grad_norm": 1.9674186706542969, "learning_rate": 9.200000000000001e-05, "loss": 0.03472021594643593, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.5, "memory/max_allocated (GiB)": 37.5, "ppl": 1.03533, "step": 24, "tokens/total": 811344, "tokens/train_per_sec_per_gpu": 32.04, "tokens/trainable": 10832} +{"epoch": 0.09765625, "grad_norm": 5.110411167144775, "learning_rate": 9.6e-05, "loss": 0.11051569133996964, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.66, "memory/max_allocated (GiB)": 36.66, "ppl": 1.11685, "step": 25, "tokens/total": 843728, "tokens/train_per_sec_per_gpu": 27.35, "tokens/trainable": 11293} +{"epoch": 0.1015625, "grad_norm": 2.4509429931640625, "learning_rate": 0.0001, "loss": 0.061853449791669846, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.06381, "step": 26, "tokens/total": 876560, "tokens/train_per_sec_per_gpu": 24.09, "tokens/trainable": 11700} +{"epoch": 0.10546875, "grad_norm": 1.5345605611801147, "learning_rate": 9.99990636831587e-05, "loss": 0.03490307182073593, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.03552, "step": 27, "tokens/total": 909488, "tokens/train_per_sec_per_gpu": 30.13, "tokens/trainable": 12167} +{"epoch": 0.109375, "grad_norm": 2.1824991703033447, "learning_rate": 9.999625477159879e-05, "loss": 0.03824758529663086, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.03899, "step": 28, "tokens/total": 947840, "tokens/train_per_sec_per_gpu": 22.69, "tokens/trainable": 12621} +{"epoch": 0.11328125, "grad_norm": 1.351633071899414, "learning_rate": 9.999157338221051e-05, "loss": 0.03253354877233505, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.03307, "step": 29, "tokens/total": 982544, "tokens/train_per_sec_per_gpu": 26.2, "tokens/trainable": 13078} +{"epoch": 0.1171875, "grad_norm": 1.678016185760498, "learning_rate": 9.998501970980562e-05, "loss": 0.034330815076828, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.03493, "step": 30, "tokens/total": 1014144, "tokens/train_per_sec_per_gpu": 31.38, "tokens/trainable": 13549} +{"epoch": 0.12109375, "grad_norm": 0.6970282196998596, "learning_rate": 9.997659402710915e-05, "loss": 0.017083141952753067, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.01723, "step": 31, "tokens/total": 1048976, "tokens/train_per_sec_per_gpu": 26.11, "tokens/trainable": 13989} +{"epoch": 0.125, "grad_norm": 0.9758917689323425, "learning_rate": 9.996629668474818e-05, "loss": 0.04763968288898468, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.04879, "step": 32, "tokens/total": 1083728, "tokens/train_per_sec_per_gpu": 29.64, "tokens/trainable": 14490} +{"epoch": 0.12890625, "grad_norm": 1.4563008546829224, "learning_rate": 9.995412811123711e-05, "loss": 0.05587531998753548, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 38.97, "memory/max_allocated (GiB)": 38.97, "ppl": 1.05747, "step": 33, "tokens/total": 1119664, "tokens/train_per_sec_per_gpu": 24.82, "tokens/trainable": 14929} +{"epoch": 0.1328125, "grad_norm": 1.3744763135910034, "learning_rate": 9.994008881295999e-05, "loss": 0.01953168399631977, "memory/device_reserved (GiB)": 39.91, "memory/max_active (GiB)": 36.87, "memory/max_allocated (GiB)": 36.87, "ppl": 1.01972, "step": 34, "tokens/total": 1152768, "tokens/train_per_sec_per_gpu": 25.65, "tokens/trainable": 15383} +{"epoch": 0.13671875, "grad_norm": 0.9090609550476074, "learning_rate": 9.992417937414932e-05, "loss": 0.030485861003398895, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 1.03096, "step": 35, "tokens/total": 1186736, "tokens/train_per_sec_per_gpu": 29.88, "tokens/trainable": 15884} +{"epoch": 0.140625, "grad_norm": 0.7822504043579102, "learning_rate": 9.99064004568618e-05, "loss": 0.03754280507564545, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.03826, "step": 36, "tokens/total": 1222352, "tokens/train_per_sec_per_gpu": 25.75, "tokens/trainable": 16343} +{"epoch": 0.14453125, "grad_norm": 2.518000364303589, "learning_rate": 9.988675280095074e-05, "loss": 0.024195142090320587, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.02449, "step": 37, "tokens/total": 1256336, "tokens/train_per_sec_per_gpu": 25.01, "tokens/trainable": 16799} +{"epoch": 0.1484375, "grad_norm": 1.5086073875427246, "learning_rate": 9.986523722403528e-05, "loss": 0.046422526240348816, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.04752, "step": 38, "tokens/total": 1288304, "tokens/train_per_sec_per_gpu": 26.72, "tokens/trainable": 17256} +{"epoch": 0.15234375, "grad_norm": 5.311761379241943, "learning_rate": 9.984185462146642e-05, "loss": 0.04862078279256821, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.41, "memory/max_allocated (GiB)": 37.41, "ppl": 1.04982, "step": 39, "tokens/total": 1323440, "tokens/train_per_sec_per_gpu": 25.67, "tokens/trainable": 17701} +{"epoch": 0.15625, "grad_norm": 1.2332911491394043, "learning_rate": 9.98166059662897e-05, "loss": 0.049109332263469696, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 35.95, "memory/max_allocated (GiB)": 35.95, "ppl": 1.05034, "step": 40, "tokens/total": 1354576, "tokens/train_per_sec_per_gpu": 29.53, "tokens/trainable": 18168} +{"epoch": 0.16015625, "grad_norm": 0.903822660446167, "learning_rate": 9.978949230920472e-05, "loss": 0.025404643267393112, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.02573, "step": 41, "tokens/total": 1389264, "tokens/train_per_sec_per_gpu": 25.18, "tokens/trainable": 18622} +{"epoch": 0.1640625, "grad_norm": 1.0105818510055542, "learning_rate": 9.976051477852141e-05, "loss": 0.040572475641965866, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.04141, "step": 42, "tokens/total": 1422496, "tokens/train_per_sec_per_gpu": 26.45, "tokens/trainable": 19063} +{"epoch": 0.16796875, "grad_norm": 2.0025603771209717, "learning_rate": 9.972967458011312e-05, "loss": 0.08459161221981049, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 38.91, "memory/max_allocated (GiB)": 38.91, "ppl": 1.08827, "step": 43, "tokens/total": 1457856, "tokens/train_per_sec_per_gpu": 27.55, "tokens/trainable": 19536} +{"epoch": 0.171875, "grad_norm": 2.556330442428589, "learning_rate": 9.96969729973664e-05, "loss": 0.06481587141752243, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.06696, "step": 44, "tokens/total": 1496720, "tokens/train_per_sec_per_gpu": 23.64, "tokens/trainable": 20000} +{"epoch": 0.17578125, "grad_norm": 0.8997912406921387, "learning_rate": 9.966241139112754e-05, "loss": 0.03254259005188942, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.1, "memory/max_allocated (GiB)": 38.1, "ppl": 1.03308, "step": 45, "tokens/total": 1530416, "tokens/train_per_sec_per_gpu": 25.86, "tokens/trainable": 20441} +{"epoch": 0.1796875, "grad_norm": 0.6450639963150024, "learning_rate": 9.96259911996461e-05, "loss": 0.018244989216327667, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.01841, "step": 46, "tokens/total": 1564576, "tokens/train_per_sec_per_gpu": 25.33, "tokens/trainable": 20873} +{"epoch": 0.18359375, "grad_norm": 0.8745234608650208, "learning_rate": 9.958771393851491e-05, "loss": 0.06614812463521957, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 36.49, "memory/max_allocated (GiB)": 36.49, "ppl": 1.06838, "step": 47, "tokens/total": 1596032, "tokens/train_per_sec_per_gpu": 25.0, "tokens/trainable": 21314} +{"epoch": 0.1875, "grad_norm": 0.6812517642974854, "learning_rate": 9.954758120060702e-05, "loss": 0.0295081976801157, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.02995, "step": 48, "tokens/total": 1632304, "tokens/train_per_sec_per_gpu": 24.51, "tokens/trainable": 21774} +{"epoch": 0.19140625, "grad_norm": 0.8261315822601318, "learning_rate": 9.950559465600948e-05, "loss": 0.018169887363910675, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 1.01834, "step": 49, "tokens/total": 1666560, "tokens/train_per_sec_per_gpu": 26.23, "tokens/trainable": 22214} +{"epoch": 0.1953125, "grad_norm": 1.0264170169830322, "learning_rate": 9.946175605195379e-05, "loss": 0.033501528203487396, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.03407, "step": 50, "tokens/total": 1706112, "tokens/train_per_sec_per_gpu": 21.86, "tokens/trainable": 22698} +{"epoch": 0.19921875, "grad_norm": 0.4088921844959259, "learning_rate": 9.941606721274322e-05, "loss": 0.00917871855199337, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.00922, "step": 51, "tokens/total": 1740320, "tokens/train_per_sec_per_gpu": 22.99, "tokens/trainable": 23134} +{"epoch": 0.203125, "grad_norm": 0.6838424801826477, "learning_rate": 9.936853003967685e-05, "loss": 0.030141476541757584, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.0306, "step": 52, "tokens/total": 1777552, "tokens/train_per_sec_per_gpu": 24.6, "tokens/trainable": 23609} +{"epoch": 0.20703125, "grad_norm": 1.2531516551971436, "learning_rate": 9.93191465109705e-05, "loss": 0.05691174045205116, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.05856, "step": 53, "tokens/total": 1814192, "tokens/train_per_sec_per_gpu": 23.67, "tokens/trainable": 24042} +{"epoch": 0.2109375, "grad_norm": 1.3843083381652832, "learning_rate": 9.926791868167438e-05, "loss": 0.02150885760784149, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.02174, "step": 54, "tokens/total": 1850688, "tokens/train_per_sec_per_gpu": 24.04, "tokens/trainable": 24468} +{"epoch": 0.21484375, "grad_norm": 0.9515783190727234, "learning_rate": 9.921484868358753e-05, "loss": 0.05578088015317917, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 1.05737, "step": 55, "tokens/total": 1886704, "tokens/train_per_sec_per_gpu": 24.27, "tokens/trainable": 24924} +{"epoch": 0.21875, "grad_norm": 0.5021209716796875, "learning_rate": 9.915993872516924e-05, "loss": 0.01628858409821987, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.01642, "step": 56, "tokens/total": 1922528, "tokens/train_per_sec_per_gpu": 23.11, "tokens/trainable": 25368} +{"epoch": 0.22265625, "grad_norm": 0.7268109321594238, "learning_rate": 9.9103191091447e-05, "loss": 0.031646400690078735, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.03215, "step": 57, "tokens/total": 1954592, "tokens/train_per_sec_per_gpu": 26.31, "tokens/trainable": 25807} +{"epoch": 0.2265625, "grad_norm": 0.33353641629219055, "learning_rate": 9.904460814392147e-05, "loss": 0.011865256354212761, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.01194, "step": 58, "tokens/total": 1993968, "tokens/train_per_sec_per_gpu": 22.52, "tokens/trainable": 26264} +{"epoch": 0.23046875, "grad_norm": 0.5332193970680237, "learning_rate": 9.898419232046825e-05, "loss": 0.023632261902093887, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.02391, "step": 59, "tokens/total": 2030672, "tokens/train_per_sec_per_gpu": 27.09, "tokens/trainable": 26767} +{"epoch": 0.234375, "grad_norm": 0.5812954902648926, "learning_rate": 9.892194613523633e-05, "loss": 0.02175777778029442, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.26, "memory/max_allocated (GiB)": 38.26, "ppl": 1.022, "step": 60, "tokens/total": 2066528, "tokens/train_per_sec_per_gpu": 24.21, "tokens/trainable": 27226} +{"epoch": 0.23828125, "grad_norm": 0.4155001640319824, "learning_rate": 9.885787217854357e-05, "loss": 0.008035548031330109, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.00807, "step": 61, "tokens/total": 2100112, "tokens/train_per_sec_per_gpu": 26.35, "tokens/trainable": 27685} +{"epoch": 0.2421875, "grad_norm": 0.7413301467895508, "learning_rate": 9.879197311676887e-05, "loss": 0.01880037412047386, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.01898, "step": 62, "tokens/total": 2136160, "tokens/train_per_sec_per_gpu": 24.17, "tokens/trainable": 28125} +{"epoch": 0.24609375, "grad_norm": 0.5714808702468872, "learning_rate": 9.872425169224113e-05, "loss": 0.03494831547141075, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.03557, "step": 63, "tokens/total": 2169952, "tokens/train_per_sec_per_gpu": 29.69, "tokens/trainable": 28624} +{"epoch": 0.25, "grad_norm": 0.8003760576248169, "learning_rate": 9.865471072312528e-05, "loss": 0.02738475427031517, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.14, "memory/max_allocated (GiB)": 38.14, "ppl": 1.02776, "step": 64, "tokens/total": 2206416, "tokens/train_per_sec_per_gpu": 25.84, "tokens/trainable": 29109} +{"epoch": 0.25390625, "grad_norm": 0.28191256523132324, "learning_rate": 9.858335310330492e-05, "loss": 0.0075698792934417725, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.0076, "step": 65, "tokens/total": 2238592, "tokens/train_per_sec_per_gpu": 26.4, "tokens/trainable": 29571} +{"epoch": 0.2578125, "grad_norm": 0.5463362336158752, "learning_rate": 9.851018180226185e-05, "loss": 0.022521493956446648, "memory/device_reserved (GiB)": 40.31, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.02278, "step": 66, "tokens/total": 2277136, "tokens/train_per_sec_per_gpu": 21.45, "tokens/trainable": 30012} +{"epoch": 0.26171875, "grad_norm": 0.8142948746681213, "learning_rate": 9.843519986495259e-05, "loss": 0.030404703691601753, "memory/device_reserved (GiB)": 41.08, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.03087, "step": 67, "tokens/total": 2311872, "tokens/train_per_sec_per_gpu": 25.3, "tokens/trainable": 30447} +{"epoch": 0.265625, "grad_norm": 0.41601037979125977, "learning_rate": 9.835841041168162e-05, "loss": 0.027483493089675903, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.02786, "step": 68, "tokens/total": 2349008, "tokens/train_per_sec_per_gpu": 24.59, "tokens/trainable": 30914} +{"epoch": 0.26953125, "grad_norm": 1.0787423849105835, "learning_rate": 9.82798166379715e-05, "loss": 0.025982506573200226, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.02632, "step": 69, "tokens/total": 2382368, "tokens/train_per_sec_per_gpu": 28.38, "tokens/trainable": 31403} +{"epoch": 0.2734375, "grad_norm": 0.576305091381073, "learning_rate": 9.819942181443002e-05, "loss": 0.06267570704221725, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.06468, "step": 70, "tokens/total": 2419648, "tokens/train_per_sec_per_gpu": 21.69, "tokens/trainable": 31843} +{"epoch": 0.27734375, "grad_norm": 0.7173606753349304, "learning_rate": 9.811722928661392e-05, "loss": 0.03863902762532234, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 35.81, "memory/max_allocated (GiB)": 35.81, "ppl": 1.0394, "step": 71, "tokens/total": 2448576, "tokens/train_per_sec_per_gpu": 26.48, "tokens/trainable": 32236} +{"epoch": 0.28125, "grad_norm": 0.4176589548587799, "learning_rate": 9.803324247488975e-05, "loss": 0.019345995038747787, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.01953, "step": 72, "tokens/total": 2484240, "tokens/train_per_sec_per_gpu": 23.98, "tokens/trainable": 32688} +{"epoch": 0.28515625, "grad_norm": 0.3949532210826874, "learning_rate": 9.794746487429161e-05, "loss": 0.01449587382376194, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 36.88, "memory/max_allocated (GiB)": 36.88, "ppl": 1.0146, "step": 73, "tokens/total": 2517184, "tokens/train_per_sec_per_gpu": 27.12, "tokens/trainable": 33131} +{"epoch": 0.2890625, "grad_norm": 0.8676579594612122, "learning_rate": 9.785990005437554e-05, "loss": 0.02716980129480362, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.02754, "step": 74, "tokens/total": 2553056, "tokens/train_per_sec_per_gpu": 25.33, "tokens/trainable": 33592} +{"epoch": 0.29296875, "grad_norm": 0.6347500085830688, "learning_rate": 9.777055165907117e-05, "loss": 0.037230029702186584, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.03793, "step": 75, "tokens/total": 2585584, "tokens/train_per_sec_per_gpu": 29.0, "tokens/trainable": 34023} +{"epoch": 0.296875, "grad_norm": 0.4884681701660156, "learning_rate": 9.767942340652993e-05, "loss": 0.021414725109934807, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.07, "memory/max_allocated (GiB)": 38.07, "ppl": 1.02165, "step": 76, "tokens/total": 2620448, "tokens/train_per_sec_per_gpu": 22.5, "tokens/trainable": 34449} +{"epoch": 0.30078125, "grad_norm": 0.747511625289917, "learning_rate": 9.758651908897035e-05, "loss": 0.04078954830765724, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.04163, "step": 77, "tokens/total": 2653840, "tokens/train_per_sec_per_gpu": 28.16, "tokens/trainable": 34895} +{"epoch": 0.3046875, "grad_norm": 1.240261197090149, "learning_rate": 9.749184257252033e-05, "loss": 0.03630094602704048, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.03697, "step": 78, "tokens/total": 2688112, "tokens/train_per_sec_per_gpu": 24.65, "tokens/trainable": 35335} +{"epoch": 0.30859375, "grad_norm": 0.5918622016906738, "learning_rate": 9.739539779705614e-05, "loss": 0.038330547511577606, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.03907, "step": 79, "tokens/total": 2726384, "tokens/train_per_sec_per_gpu": 24.63, "tokens/trainable": 35830} +{"epoch": 0.3125, "grad_norm": 0.7412325143814087, "learning_rate": 9.729718877603861e-05, "loss": 0.03330954536795616, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.03387, "step": 80, "tokens/total": 2760176, "tokens/train_per_sec_per_gpu": 26.99, "tokens/trainable": 36262} +{"epoch": 0.31640625, "grad_norm": 0.8288413286209106, "learning_rate": 9.719721959634592e-05, "loss": 0.03821411356329918, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.03895, "step": 81, "tokens/total": 2795392, "tokens/train_per_sec_per_gpu": 23.35, "tokens/trainable": 36699} +{"epoch": 0.3203125, "grad_norm": 0.3764454424381256, "learning_rate": 9.709549441810375e-05, "loss": 0.015863850712776184, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.55, "memory/max_allocated (GiB)": 39.55, "ppl": 1.01599, "step": 82, "tokens/total": 2833008, "tokens/train_per_sec_per_gpu": 27.1, "tokens/trainable": 37183} +{"epoch": 0.32421875, "grad_norm": 0.34552839398384094, "learning_rate": 9.699201747451195e-05, "loss": 0.027101386338472366, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.02747, "step": 83, "tokens/total": 2865600, "tokens/train_per_sec_per_gpu": 25.14, "tokens/trainable": 37631} +{"epoch": 0.328125, "grad_norm": 0.26814818382263184, "learning_rate": 9.688679307166854e-05, "loss": 0.020230205729603767, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.02044, "step": 84, "tokens/total": 2898400, "tokens/train_per_sec_per_gpu": 28.49, "tokens/trainable": 38094} +{"epoch": 0.33203125, "grad_norm": 0.9112386703491211, "learning_rate": 9.677982558839042e-05, "loss": 0.040751442313194275, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.04159, "step": 85, "tokens/total": 2931696, "tokens/train_per_sec_per_gpu": 27.26, "tokens/trainable": 38573} +{"epoch": 0.3359375, "grad_norm": 0.288052499294281, "learning_rate": 9.66711194760312e-05, "loss": 0.011943141929805279, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.01201, "step": 86, "tokens/total": 2969104, "tokens/train_per_sec_per_gpu": 25.68, "tokens/trainable": 39061} +{"epoch": 0.33984375, "grad_norm": 0.46535351872444153, "learning_rate": 9.656067925829593e-05, "loss": 0.020353971049189568, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.02056, "step": 87, "tokens/total": 2999856, "tokens/train_per_sec_per_gpu": 31.45, "tokens/trainable": 39521} +{"epoch": 0.34375, "grad_norm": 0.3164924383163452, "learning_rate": 9.644850953105288e-05, "loss": 0.026463575661182404, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 35.78, "memory/max_allocated (GiB)": 35.78, "ppl": 1.02682, "step": 88, "tokens/total": 3030208, "tokens/train_per_sec_per_gpu": 26.34, "tokens/trainable": 39963} +{"epoch": 0.34765625, "grad_norm": 0.767791211605072, "learning_rate": 9.633461496214225e-05, "loss": 0.021780140697956085, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.02202, "step": 89, "tokens/total": 3062912, "tokens/train_per_sec_per_gpu": 26.69, "tokens/trainable": 40433} +{"epoch": 0.3515625, "grad_norm": 0.459378182888031, "learning_rate": 9.621900029118195e-05, "loss": 0.042631253600120544, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.04, "memory/max_allocated (GiB)": 36.04, "ppl": 1.04355, "step": 90, "tokens/total": 3092592, "tokens/train_per_sec_per_gpu": 25.88, "tokens/trainable": 40857} +{"epoch": 0.35546875, "grad_norm": 2.3746743202209473, "learning_rate": 9.610167032937036e-05, "loss": 0.00897490419447422, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00902, "step": 91, "tokens/total": 3130880, "tokens/train_per_sec_per_gpu": 26.27, "tokens/trainable": 41376} +{"epoch": 0.359375, "grad_norm": 0.4058453440666199, "learning_rate": 9.598262995928611e-05, "loss": 0.022762980312108994, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.02302, "step": 92, "tokens/total": 3162080, "tokens/train_per_sec_per_gpu": 30.39, "tokens/trainable": 41845} +{"epoch": 0.36328125, "grad_norm": 0.3419894576072693, "learning_rate": 9.586188413468492e-05, "loss": 0.006684569641947746, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00671, "step": 93, "tokens/total": 3200272, "tokens/train_per_sec_per_gpu": 23.06, "tokens/trainable": 42291} +{"epoch": 0.3671875, "grad_norm": 0.26857829093933105, "learning_rate": 9.57394378802934e-05, "loss": 0.015188409015536308, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.0153, "step": 94, "tokens/total": 3233584, "tokens/train_per_sec_per_gpu": 27.65, "tokens/trainable": 42743} +{"epoch": 0.37109375, "grad_norm": 0.9427598118782043, "learning_rate": 9.56152962916e-05, "loss": 0.04341185465455055, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.26, "memory/max_allocated (GiB)": 36.26, "ppl": 1.04437, "step": 95, "tokens/total": 3264144, "tokens/train_per_sec_per_gpu": 26.91, "tokens/trainable": 43140} +{"epoch": 0.375, "grad_norm": 1.3085354566574097, "learning_rate": 9.548946453464296e-05, "loss": 0.011709408834576607, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.01178, "step": 96, "tokens/total": 3298240, "tokens/train_per_sec_per_gpu": 27.04, "tokens/trainable": 43625} +{"epoch": 0.37890625, "grad_norm": 0.4093848466873169, "learning_rate": 9.53619478457953e-05, "loss": 0.02878337725996971, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.0292, "step": 97, "tokens/total": 3333136, "tokens/train_per_sec_per_gpu": 24.18, "tokens/trainable": 44081} +{"epoch": 0.3828125, "grad_norm": 0.2406461238861084, "learning_rate": 9.523275153154695e-05, "loss": 0.009247010573744774, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.00929, "step": 98, "tokens/total": 3365536, "tokens/train_per_sec_per_gpu": 25.4, "tokens/trainable": 44531} +{"epoch": 0.38671875, "grad_norm": 0.38781416416168213, "learning_rate": 9.51018809682839e-05, "loss": 0.027175158262252808, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.02755, "step": 99, "tokens/total": 3403200, "tokens/train_per_sec_per_gpu": 25.82, "tokens/trainable": 44978} +{"epoch": 0.390625, "grad_norm": 0.5931934118270874, "learning_rate": 9.49693416020645e-05, "loss": 0.01281006634235382, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.53, "memory/max_allocated (GiB)": 38.53, "ppl": 1.01289, "step": 100, "tokens/total": 3440944, "tokens/train_per_sec_per_gpu": 24.71, "tokens/trainable": 45431} +{"epoch": 0.39453125, "grad_norm": 0.2966616451740265, "learning_rate": 9.483513894839276e-05, "loss": 0.011583802290260792, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.01165, "step": 101, "tokens/total": 3474992, "tokens/train_per_sec_per_gpu": 25.47, "tokens/trainable": 45878} +{"epoch": 0.3984375, "grad_norm": 0.48896685242652893, "learning_rate": 9.469927859198888e-05, "loss": 0.019925545901060104, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.02013, "step": 102, "tokens/total": 3510912, "tokens/train_per_sec_per_gpu": 29.3, "tokens/trainable": 46365} +{"epoch": 0.40234375, "grad_norm": 0.7017014026641846, "learning_rate": 9.456176618655689e-05, "loss": 0.030322808772325516, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.47, "memory/max_allocated (GiB)": 36.47, "ppl": 1.03079, "step": 103, "tokens/total": 3542960, "tokens/train_per_sec_per_gpu": 27.68, "tokens/trainable": 46812} +{"epoch": 0.40625, "grad_norm": 0.5579801201820374, "learning_rate": 9.442260745454927e-05, "loss": 0.01888164132833481, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.01906, "step": 104, "tokens/total": 3578336, "tokens/train_per_sec_per_gpu": 26.03, "tokens/trainable": 47289} +{"epoch": 0.41015625, "grad_norm": 0.24767188727855682, "learning_rate": 9.428180818692884e-05, "loss": 0.00682178046554327, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.45, "memory/max_allocated (GiB)": 38.45, "ppl": 1.00685, "step": 105, "tokens/total": 3614496, "tokens/train_per_sec_per_gpu": 24.89, "tokens/trainable": 47790} +{"epoch": 0.4140625, "grad_norm": 0.3641205430030823, "learning_rate": 9.413937424292791e-05, "loss": 0.018602963536977768, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.01878, "step": 106, "tokens/total": 3653632, "tokens/train_per_sec_per_gpu": 21.7, "tokens/trainable": 48244} +{"epoch": 0.41796875, "grad_norm": 0.33046379685401917, "learning_rate": 9.399531154980424e-05, "loss": 0.00858956016600132, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 1.00863, "step": 107, "tokens/total": 3690208, "tokens/train_per_sec_per_gpu": 24.25, "tokens/trainable": 48693} +{"epoch": 0.421875, "grad_norm": 0.275390088558197, "learning_rate": 9.384962610259455e-05, "loss": 0.01764410175383091, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.0178, "step": 108, "tokens/total": 3724864, "tokens/train_per_sec_per_gpu": 24.33, "tokens/trainable": 49142} +{"epoch": 0.42578125, "grad_norm": 0.744308590888977, "learning_rate": 9.370232396386494e-05, "loss": 0.03552740812301636, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.22, "memory/max_allocated (GiB)": 37.22, "ppl": 1.03617, "step": 109, "tokens/total": 3757952, "tokens/train_per_sec_per_gpu": 26.59, "tokens/trainable": 49583} +{"epoch": 0.4296875, "grad_norm": 0.571097195148468, "learning_rate": 9.355341126345868e-05, "loss": 0.026293428614735603, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.02664, "step": 110, "tokens/total": 3793232, "tokens/train_per_sec_per_gpu": 25.62, "tokens/trainable": 50055} +{"epoch": 0.43359375, "grad_norm": 0.7366065979003906, "learning_rate": 9.340289419824107e-05, "loss": 0.025022359564900398, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.02534, "step": 111, "tokens/total": 3825552, "tokens/train_per_sec_per_gpu": 28.26, "tokens/trainable": 50500} +{"epoch": 0.4375, "grad_norm": 0.41047194600105286, "learning_rate": 9.325077903184159e-05, "loss": 0.03949592262506485, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.04029, "step": 112, "tokens/total": 3860544, "tokens/train_per_sec_per_gpu": 24.02, "tokens/trainable": 50950} +{"epoch": 0.44140625, "grad_norm": 0.2281654179096222, "learning_rate": 9.30970720943932e-05, "loss": 0.022109592333436012, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.15, "memory/max_allocated (GiB)": 38.15, "ppl": 1.02236, "step": 113, "tokens/total": 3895552, "tokens/train_per_sec_per_gpu": 24.52, "tokens/trainable": 51388} +{"epoch": 0.4453125, "grad_norm": 2.964517116546631, "learning_rate": 9.2941779782269e-05, "loss": 0.026215432211756706, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.02656, "step": 114, "tokens/total": 3929152, "tokens/train_per_sec_per_gpu": 25.58, "tokens/trainable": 51861} +{"epoch": 0.44921875, "grad_norm": 0.262885719537735, "learning_rate": 9.278490855781596e-05, "loss": 0.016062922775745392, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.24, "memory/max_allocated (GiB)": 37.24, "ppl": 1.01619, "step": 115, "tokens/total": 3961472, "tokens/train_per_sec_per_gpu": 30.09, "tokens/trainable": 52300} +{"epoch": 0.453125, "grad_norm": 0.3646382689476013, "learning_rate": 9.262646494908604e-05, "loss": 0.02547876164317131, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.02581, "step": 116, "tokens/total": 4000128, "tokens/train_per_sec_per_gpu": 23.68, "tokens/trainable": 52753} +{"epoch": 0.45703125, "grad_norm": 0.3807598948478699, "learning_rate": 9.246645554956457e-05, "loss": 0.03482891619205475, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.03544, "step": 117, "tokens/total": 4035152, "tokens/train_per_sec_per_gpu": 25.02, "tokens/trainable": 53221} +{"epoch": 0.4609375, "grad_norm": 0.2920905351638794, "learning_rate": 9.230488701789578e-05, "loss": 0.03315863013267517, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.03371, "step": 118, "tokens/total": 4072208, "tokens/train_per_sec_per_gpu": 23.5, "tokens/trainable": 53663} +{"epoch": 0.46484375, "grad_norm": 0.3890359103679657, "learning_rate": 9.214176607760577e-05, "loss": 0.012569136917591095, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.01265, "step": 119, "tokens/total": 4110464, "tokens/train_per_sec_per_gpu": 24.17, "tokens/trainable": 54126} +{"epoch": 0.46875, "grad_norm": 0.5687521696090698, "learning_rate": 9.197709951682268e-05, "loss": 0.03563442826271057, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.03628, "step": 120, "tokens/total": 4142896, "tokens/train_per_sec_per_gpu": 29.1, "tokens/trainable": 54584} +{"epoch": 0.47265625, "grad_norm": 0.5217305421829224, "learning_rate": 9.181089418799428e-05, "loss": 0.03250029683113098, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.03303, "step": 121, "tokens/total": 4179360, "tokens/train_per_sec_per_gpu": 27.33, "tokens/trainable": 55067} +{"epoch": 0.4765625, "grad_norm": 0.32481950521469116, "learning_rate": 9.164315700760271e-05, "loss": 0.015617378056049347, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.01574, "step": 122, "tokens/total": 4215040, "tokens/train_per_sec_per_gpu": 23.33, "tokens/trainable": 55491} +{"epoch": 0.48046875, "grad_norm": 0.327159583568573, "learning_rate": 9.147389495587671e-05, "loss": 0.02269110269844532, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.02295, "step": 123, "tokens/total": 4246448, "tokens/train_per_sec_per_gpu": 30.98, "tokens/trainable": 55937} +{"epoch": 0.484375, "grad_norm": 0.2691946029663086, "learning_rate": 9.130311507650116e-05, "loss": 0.009724990464746952, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.00977, "step": 124, "tokens/total": 4278832, "tokens/train_per_sec_per_gpu": 26.92, "tokens/trainable": 56401} +{"epoch": 0.48828125, "grad_norm": 0.5146195888519287, "learning_rate": 9.113082447632394e-05, "loss": 0.017450787127017975, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.0176, "step": 125, "tokens/total": 4310896, "tokens/train_per_sec_per_gpu": 33.01, "tokens/trainable": 56874} +{"epoch": 0.4921875, "grad_norm": 1.215205192565918, "learning_rate": 9.09570303250602e-05, "loss": 0.01777658984065056, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.01794, "step": 126, "tokens/total": 4342912, "tokens/train_per_sec_per_gpu": 30.5, "tokens/trainable": 57358} +{"epoch": 0.49609375, "grad_norm": 0.28748154640197754, "learning_rate": 9.078173985499394e-05, "loss": 0.005321727134287357, "memory/device_reserved (GiB)": 42.05, "memory/max_active (GiB)": 35.95, "memory/max_allocated (GiB)": 35.95, "ppl": 1.00534, "step": 127, "tokens/total": 4373312, "tokens/train_per_sec_per_gpu": 28.26, "tokens/trainable": 57788} +{"epoch": 0.5, "grad_norm": 0.3299351930618286, "learning_rate": 9.060496036067713e-05, "loss": 0.007326261606067419, "memory/device_reserved (GiB)": 42.05, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.00735, "step": 128, "tokens/total": 4410816, "tokens/train_per_sec_per_gpu": 22.36, "tokens/trainable": 58212} +{"epoch": 0.50390625, "grad_norm": 0.3515678942203522, "learning_rate": 9.042669919862615e-05, "loss": 0.017966657876968384, "memory/device_reserved (GiB)": 42.05, "memory/max_active (GiB)": 36.77, "memory/max_allocated (GiB)": 36.77, "ppl": 1.01813, "step": 129, "tokens/total": 4442704, "tokens/train_per_sec_per_gpu": 29.9, "tokens/trainable": 58676} +{"epoch": 0.5078125, "grad_norm": 0.6314386129379272, "learning_rate": 9.024696378701557e-05, "loss": 0.034924328327178955, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 39.48, "memory/max_allocated (GiB)": 39.48, "ppl": 1.03554, "step": 130, "tokens/total": 4478384, "tokens/train_per_sec_per_gpu": 23.74, "tokens/trainable": 59109} +{"epoch": 0.51171875, "grad_norm": 0.3557182252407074, "learning_rate": 9.006576160536948e-05, "loss": 0.027520572766661644, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.0279, "step": 131, "tokens/total": 4513328, "tokens/train_per_sec_per_gpu": 24.17, "tokens/trainable": 59558} +{"epoch": 0.515625, "grad_norm": 0.7262012958526611, "learning_rate": 8.988310019425035e-05, "loss": 0.01739434525370598, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.01755, "step": 132, "tokens/total": 4550832, "tokens/train_per_sec_per_gpu": 23.99, "tokens/trainable": 60002} +{"epoch": 0.51953125, "grad_norm": 0.30109646916389465, "learning_rate": 8.969898715494506e-05, "loss": 0.012400257401168346, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.01248, "step": 133, "tokens/total": 4587968, "tokens/train_per_sec_per_gpu": 25.56, "tokens/trainable": 60483} +{"epoch": 0.5234375, "grad_norm": 0.5312788486480713, "learning_rate": 8.951343014914869e-05, "loss": 0.019721096381545067, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.01992, "step": 134, "tokens/total": 4624544, "tokens/train_per_sec_per_gpu": 23.4, "tokens/trainable": 60949} +{"epoch": 0.52734375, "grad_norm": 0.21193408966064453, "learning_rate": 8.932643689864568e-05, "loss": 0.006488507613539696, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00651, "step": 135, "tokens/total": 4661296, "tokens/train_per_sec_per_gpu": 27.11, "tokens/trainable": 61436} +{"epoch": 0.53125, "grad_norm": 0.3145233392715454, "learning_rate": 8.913801518498845e-05, "loss": 0.006278295069932938, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.0063, "step": 136, "tokens/total": 4700128, "tokens/train_per_sec_per_gpu": 21.18, "tokens/trainable": 61897} +{"epoch": 0.53515625, "grad_norm": 0.4243067800998688, "learning_rate": 8.894817284917364e-05, "loss": 0.012885574251413345, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.01297, "step": 137, "tokens/total": 4736912, "tokens/train_per_sec_per_gpu": 22.65, "tokens/trainable": 62351} +{"epoch": 0.5390625, "grad_norm": 0.6918410658836365, "learning_rate": 8.875691779131569e-05, "loss": 0.03191700577735901, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.03243, "step": 138, "tokens/total": 4774192, "tokens/train_per_sec_per_gpu": 20.63, "tokens/trainable": 62783} +{"epoch": 0.54296875, "grad_norm": 0.2173067033290863, "learning_rate": 8.856425797031829e-05, "loss": 0.007320343516767025, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00735, "step": 139, "tokens/total": 4809344, "tokens/train_per_sec_per_gpu": 25.61, "tokens/trainable": 63225} +{"epoch": 0.546875, "grad_norm": 0.6272622346878052, "learning_rate": 8.837020140354295e-05, "loss": 0.017380204051733017, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.01753, "step": 140, "tokens/total": 4845712, "tokens/train_per_sec_per_gpu": 26.67, "tokens/trainable": 63690} +{"epoch": 0.55078125, "grad_norm": 0.4769114553928375, "learning_rate": 8.817475616647554e-05, "loss": 0.017553094774484634, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.01771, "step": 141, "tokens/total": 4880368, "tokens/train_per_sec_per_gpu": 26.38, "tokens/trainable": 64157} +{"epoch": 0.5546875, "grad_norm": 0.24060319364070892, "learning_rate": 8.797793039239017e-05, "loss": 0.022226735949516296, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.02248, "step": 142, "tokens/total": 4915424, "tokens/train_per_sec_per_gpu": 25.78, "tokens/trainable": 64605} +{"epoch": 0.55859375, "grad_norm": 0.23776879906654358, "learning_rate": 8.777973227201069e-05, "loss": 0.010393165051937103, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.01045, "step": 143, "tokens/total": 4951744, "tokens/train_per_sec_per_gpu": 27.65, "tokens/trainable": 65114} +{"epoch": 0.5625, "grad_norm": 0.360625296831131, "learning_rate": 8.758017005316988e-05, "loss": 0.015630656853318214, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.01575, "step": 144, "tokens/total": 4985152, "tokens/train_per_sec_per_gpu": 28.08, "tokens/trainable": 65539} +{"epoch": 0.56640625, "grad_norm": 0.6649085879325867, "learning_rate": 8.737925204046629e-05, "loss": 0.03400883078575134, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.03459, "step": 145, "tokens/total": 5019936, "tokens/train_per_sec_per_gpu": 23.79, "tokens/trainable": 65989} +{"epoch": 0.5703125, "grad_norm": 0.10760762542486191, "learning_rate": 8.717698659491851e-05, "loss": 0.00273921899497509, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00274, "step": 146, "tokens/total": 5057024, "tokens/train_per_sec_per_gpu": 22.13, "tokens/trainable": 66446} +{"epoch": 0.57421875, "grad_norm": 0.4865633547306061, "learning_rate": 8.697338213361735e-05, "loss": 0.017447009682655334, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.0176, "step": 147, "tokens/total": 5088320, "tokens/train_per_sec_per_gpu": 25.85, "tokens/trainable": 66894} +{"epoch": 0.578125, "grad_norm": 0.11254577338695526, "learning_rate": 8.676844712937552e-05, "loss": 0.004536926746368408, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.39, "memory/max_allocated (GiB)": 39.39, "ppl": 1.00455, "step": 148, "tokens/total": 5128064, "tokens/train_per_sec_per_gpu": 24.64, "tokens/trainable": 67366} +{"epoch": 0.58203125, "grad_norm": 0.2365051656961441, "learning_rate": 8.656219011037509e-05, "loss": 0.017461499199271202, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.01761, "step": 149, "tokens/total": 5165104, "tokens/train_per_sec_per_gpu": 23.38, "tokens/trainable": 67854} +{"epoch": 0.5859375, "grad_norm": 0.42174994945526123, "learning_rate": 8.63546196598125e-05, "loss": 0.012682422995567322, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.01276, "step": 150, "tokens/total": 5202336, "tokens/train_per_sec_per_gpu": 24.9, "tokens/trainable": 68302} +{"epoch": 0.58984375, "grad_norm": 0.32398444414138794, "learning_rate": 8.614574441554145e-05, "loss": 0.028823766857385635, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.13, "memory/max_allocated (GiB)": 37.13, "ppl": 1.02924, "step": 151, "tokens/total": 5235440, "tokens/train_per_sec_per_gpu": 27.12, "tokens/trainable": 68727} +{"epoch": 0.59375, "grad_norm": 0.7370245456695557, "learning_rate": 8.593557306971349e-05, "loss": 0.014559872448444366, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.44, "memory/max_allocated (GiB)": 37.44, "ppl": 1.01467, "step": 152, "tokens/total": 5268416, "tokens/train_per_sec_per_gpu": 25.72, "tokens/trainable": 69182} +{"epoch": 0.59765625, "grad_norm": 0.14002655446529388, "learning_rate": 8.572411436841618e-05, "loss": 0.003013424575328827, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00302, "step": 153, "tokens/total": 5302128, "tokens/train_per_sec_per_gpu": 26.31, "tokens/trainable": 69657} +{"epoch": 0.6015625, "grad_norm": 0.16943055391311646, "learning_rate": 8.551137711130922e-05, "loss": 0.005830880254507065, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.2, "memory/max_allocated (GiB)": 38.2, "ppl": 1.00585, "step": 154, "tokens/total": 5333520, "tokens/train_per_sec_per_gpu": 23.66, "tokens/trainable": 70063} +{"epoch": 0.60546875, "grad_norm": 0.16723008453845978, "learning_rate": 8.529737015125824e-05, "loss": 0.004703755024820566, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00471, "step": 155, "tokens/total": 5367968, "tokens/train_per_sec_per_gpu": 25.6, "tokens/trainable": 70529} +{"epoch": 0.609375, "grad_norm": 0.2517866790294647, "learning_rate": 8.508210239396639e-05, "loss": 0.008934741839766502, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.00897, "step": 156, "tokens/total": 5405536, "tokens/train_per_sec_per_gpu": 23.98, "tokens/trainable": 70978} +{"epoch": 0.61328125, "grad_norm": 0.42575693130493164, "learning_rate": 8.486558279760375e-05, "loss": 0.014943700283765793, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 36.07, "memory/max_allocated (GiB)": 36.07, "ppl": 1.01506, "step": 157, "tokens/total": 5436672, "tokens/train_per_sec_per_gpu": 26.41, "tokens/trainable": 71404} +{"epoch": 0.6171875, "grad_norm": 0.17557889223098755, "learning_rate": 8.464782037243449e-05, "loss": 0.008665681816637516, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.0087, "step": 158, "tokens/total": 5469280, "tokens/train_per_sec_per_gpu": 27.15, "tokens/trainable": 71845} +{"epoch": 0.62109375, "grad_norm": 0.48586469888687134, "learning_rate": 8.442882418044202e-05, "loss": 0.023510482162237167, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.02379, "step": 159, "tokens/total": 5501904, "tokens/train_per_sec_per_gpu": 29.1, "tokens/trainable": 72322} +{"epoch": 0.625, "grad_norm": 0.3714931905269623, "learning_rate": 8.420860333495179e-05, "loss": 0.022035684436559677, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.02228, "step": 160, "tokens/total": 5534944, "tokens/train_per_sec_per_gpu": 26.5, "tokens/trainable": 72737} +{"epoch": 0.62890625, "grad_norm": 0.05251815542578697, "learning_rate": 8.398716700025208e-05, "loss": 0.0019496185705065727, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.00195, "step": 161, "tokens/total": 5568160, "tokens/train_per_sec_per_gpu": 25.5, "tokens/trainable": 73189} +{"epoch": 0.6328125, "grad_norm": 0.8177186250686646, "learning_rate": 8.376452439121266e-05, "loss": 0.034302495419979095, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 35.89, "memory/max_allocated (GiB)": 35.89, "ppl": 1.0349, "step": 162, "tokens/total": 5598256, "tokens/train_per_sec_per_gpu": 30.86, "tokens/trainable": 73640} +{"epoch": 0.63671875, "grad_norm": 0.3351547420024872, "learning_rate": 8.354068477290124e-05, "loss": 0.012193086557090282, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.01227, "step": 163, "tokens/total": 5632560, "tokens/train_per_sec_per_gpu": 27.57, "tokens/trainable": 74099} +{"epoch": 0.640625, "grad_norm": 0.2569303810596466, "learning_rate": 8.331565746019807e-05, "loss": 0.009293467737734318, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 36.52, "memory/max_allocated (GiB)": 36.52, "ppl": 1.00934, "step": 164, "tokens/total": 5664464, "tokens/train_per_sec_per_gpu": 26.93, "tokens/trainable": 74557} +{"epoch": 0.64453125, "grad_norm": 0.1866869181394577, "learning_rate": 8.308945181740812e-05, "loss": 0.0066506508737802505, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.00667, "step": 165, "tokens/total": 5699040, "tokens/train_per_sec_per_gpu": 27.06, "tokens/trainable": 74999} +{"epoch": 0.6484375, "grad_norm": 0.2206009179353714, "learning_rate": 8.286207725787153e-05, "loss": 0.0110967131331563, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.01116, "step": 166, "tokens/total": 5735968, "tokens/train_per_sec_per_gpu": 22.34, "tokens/trainable": 75444} +{"epoch": 0.65234375, "grad_norm": 0.23456768691539764, "learning_rate": 8.263354324357182e-05, "loss": 0.005873357877135277, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00589, "step": 167, "tokens/total": 5770688, "tokens/train_per_sec_per_gpu": 25.3, "tokens/trainable": 75879} +{"epoch": 0.65625, "grad_norm": 0.09018169343471527, "learning_rate": 8.240385928474219e-05, "loss": 0.0022832041140645742, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00229, "step": 168, "tokens/total": 5803824, "tokens/train_per_sec_per_gpu": 26.89, "tokens/trainable": 76293} +{"epoch": 0.66015625, "grad_norm": 0.0727015808224678, "learning_rate": 8.217303493946967e-05, "loss": 0.0014178541023284197, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.00142, "step": 169, "tokens/total": 5838256, "tokens/train_per_sec_per_gpu": 26.1, "tokens/trainable": 76741} +{"epoch": 0.6640625, "grad_norm": 0.18488948047161102, "learning_rate": 8.194107981329746e-05, "loss": 0.0032491993624716997, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00325, "step": 170, "tokens/total": 5876224, "tokens/train_per_sec_per_gpu": 23.24, "tokens/trainable": 77195} +{"epoch": 0.66796875, "grad_norm": 0.023800108581781387, "learning_rate": 8.170800355882518e-05, "loss": 0.0005309373373165727, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.24, "memory/max_allocated (GiB)": 37.24, "ppl": 1.00053, "step": 171, "tokens/total": 5910448, "tokens/train_per_sec_per_gpu": 30.32, "tokens/trainable": 77662} +{"epoch": 0.671875, "grad_norm": 0.6908709406852722, "learning_rate": 8.147381587530713e-05, "loss": 0.02130153402686119, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.02153, "step": 172, "tokens/total": 5944256, "tokens/train_per_sec_per_gpu": 26.45, "tokens/trainable": 78116} +{"epoch": 0.67578125, "grad_norm": 0.5446029901504517, "learning_rate": 8.123852650824877e-05, "loss": 0.02307780459523201, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.02335, "step": 173, "tokens/total": 5978288, "tokens/train_per_sec_per_gpu": 28.36, "tokens/trainable": 78599} +{"epoch": 0.6796875, "grad_norm": 0.41260918974876404, "learning_rate": 8.100214524900103e-05, "loss": 0.013393591158092022, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.52, "memory/max_allocated (GiB)": 38.52, "ppl": 1.01348, "step": 174, "tokens/total": 6013680, "tokens/train_per_sec_per_gpu": 22.14, "tokens/trainable": 79030} +{"epoch": 0.68359375, "grad_norm": 0.07072993367910385, "learning_rate": 8.076468193435301e-05, "loss": 0.0011126662138849497, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.00111, "step": 175, "tokens/total": 6045072, "tokens/train_per_sec_per_gpu": 25.93, "tokens/trainable": 79454} +{"epoch": 0.6875, "grad_norm": 0.9365659952163696, "learning_rate": 8.052614644612253e-05, "loss": 0.007110465317964554, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.00714, "step": 176, "tokens/total": 6082336, "tokens/train_per_sec_per_gpu": 23.27, "tokens/trainable": 79915} +{"epoch": 0.69140625, "grad_norm": 1.16559636592865, "learning_rate": 8.028654871074489e-05, "loss": 0.00570506090298295, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00572, "step": 177, "tokens/total": 6117024, "tokens/train_per_sec_per_gpu": 25.44, "tokens/trainable": 80356} +{"epoch": 0.6953125, "grad_norm": 1.0206210613250732, "learning_rate": 8.004589869885986e-05, "loss": 0.039279237389564514, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 36.49, "memory/max_allocated (GiB)": 36.49, "ppl": 1.04006, "step": 178, "tokens/total": 6149136, "tokens/train_per_sec_per_gpu": 26.59, "tokens/trainable": 80774} +{"epoch": 0.69921875, "grad_norm": 0.4380479156970978, "learning_rate": 7.980420642489674e-05, "loss": 0.009928105399012566, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.00998, "step": 179, "tokens/total": 6187776, "tokens/train_per_sec_per_gpu": 23.3, "tokens/trainable": 81247} +{"epoch": 0.703125, "grad_norm": 1.821321964263916, "learning_rate": 7.95614819466576e-05, "loss": 0.008231095969676971, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00827, "step": 180, "tokens/total": 6221728, "tokens/train_per_sec_per_gpu": 28.38, "tokens/trainable": 81713} +{"epoch": 0.70703125, "grad_norm": 0.9375918507575989, "learning_rate": 7.931773536489872e-05, "loss": 0.034399740397930145, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.035, "step": 181, "tokens/total": 6255920, "tokens/train_per_sec_per_gpu": 24.85, "tokens/trainable": 82158} +{"epoch": 0.7109375, "grad_norm": 0.5634071230888367, "learning_rate": 7.907297682291035e-05, "loss": 0.03643191233277321, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.0371, "step": 182, "tokens/total": 6291104, "tokens/train_per_sec_per_gpu": 26.45, "tokens/trainable": 82621} +{"epoch": 0.71484375, "grad_norm": 0.225381001830101, "learning_rate": 7.882721650609442e-05, "loss": 0.008735476061701775, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.00877, "step": 183, "tokens/total": 6330976, "tokens/train_per_sec_per_gpu": 23.16, "tokens/trainable": 83122} +{"epoch": 0.71875, "grad_norm": 0.23520688712596893, "learning_rate": 7.85804646415409e-05, "loss": 0.01353299617767334, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.01362, "step": 184, "tokens/total": 6366160, "tokens/train_per_sec_per_gpu": 26.7, "tokens/trainable": 83578} +{"epoch": 0.72265625, "grad_norm": 0.24639570713043213, "learning_rate": 7.833273149760207e-05, "loss": 0.010687915608286858, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.01075, "step": 185, "tokens/total": 6397280, "tokens/train_per_sec_per_gpu": 28.68, "tokens/trainable": 84048} +{"epoch": 0.7265625, "grad_norm": 0.5357109308242798, "learning_rate": 7.808402738346527e-05, "loss": 0.030390407890081406, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.03086, "step": 186, "tokens/total": 6435632, "tokens/train_per_sec_per_gpu": 23.37, "tokens/trainable": 84497} +{"epoch": 0.73046875, "grad_norm": 0.21070106327533722, "learning_rate": 7.783436264872382e-05, "loss": 0.009557404555380344, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.0096, "step": 187, "tokens/total": 6472752, "tokens/train_per_sec_per_gpu": 24.76, "tokens/trainable": 84983} +{"epoch": 0.734375, "grad_norm": 0.25384798645973206, "learning_rate": 7.758374768294647e-05, "loss": 0.010274386033415794, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.05, "memory/max_allocated (GiB)": 38.05, "ppl": 1.01033, "step": 188, "tokens/total": 6508528, "tokens/train_per_sec_per_gpu": 26.91, "tokens/trainable": 85462} +{"epoch": 0.73828125, "grad_norm": 0.4955308735370636, "learning_rate": 7.733219291524489e-05, "loss": 0.02155977487564087, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.59, "memory/max_allocated (GiB)": 37.59, "ppl": 1.02179, "step": 189, "tokens/total": 6542048, "tokens/train_per_sec_per_gpu": 27.29, "tokens/trainable": 85892} +{"epoch": 0.7421875, "grad_norm": 0.7483731508255005, "learning_rate": 7.707970881383977e-05, "loss": 0.021988891065120697, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 36.57, "memory/max_allocated (GiB)": 36.57, "ppl": 1.02223, "step": 190, "tokens/total": 6573792, "tokens/train_per_sec_per_gpu": 28.02, "tokens/trainable": 86341} +{"epoch": 0.74609375, "grad_norm": 0.24885867536067963, "learning_rate": 7.682630588562518e-05, "loss": 0.025413185358047485, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.02574, "step": 191, "tokens/total": 6608848, "tokens/train_per_sec_per_gpu": 24.27, "tokens/trainable": 86772} +{"epoch": 0.75, "grad_norm": 0.37406468391418457, "learning_rate": 7.657199467573129e-05, "loss": 0.020427817478775978, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.02064, "step": 192, "tokens/total": 6642560, "tokens/train_per_sec_per_gpu": 26.08, "tokens/trainable": 87244} +{"epoch": 0.75390625, "grad_norm": 0.14520135521888733, "learning_rate": 7.631678576708561e-05, "loss": 0.0039917598478496075, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.004, "step": 193, "tokens/total": 6676272, "tokens/train_per_sec_per_gpu": 28.93, "tokens/trainable": 87736} +{"epoch": 0.7578125, "grad_norm": 0.09962499886751175, "learning_rate": 7.606068977997255e-05, "loss": 0.002445481950417161, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00245, "step": 194, "tokens/total": 6713568, "tokens/train_per_sec_per_gpu": 25.61, "tokens/trainable": 88214} +{"epoch": 0.76171875, "grad_norm": 0.18649537861347198, "learning_rate": 7.580371737159148e-05, "loss": 0.00618223799392581, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.0062, "step": 195, "tokens/total": 6749296, "tokens/train_per_sec_per_gpu": 22.91, "tokens/trainable": 88625} +{"epoch": 0.765625, "grad_norm": 0.2237590104341507, "learning_rate": 7.554587923561324e-05, "loss": 0.0043820785358548164, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00439, "step": 196, "tokens/total": 6783952, "tokens/train_per_sec_per_gpu": 24.58, "tokens/trainable": 89063} +{"epoch": 0.76953125, "grad_norm": 0.6815054416656494, "learning_rate": 7.528718610173511e-05, "loss": 0.01929176226258278, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.48, "memory/max_allocated (GiB)": 37.48, "ppl": 1.01948, "step": 197, "tokens/total": 6819152, "tokens/train_per_sec_per_gpu": 22.77, "tokens/trainable": 89484} +{"epoch": 0.7734375, "grad_norm": 0.059283751994371414, "learning_rate": 7.502764873523431e-05, "loss": 0.0018588945968076587, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.00186, "step": 198, "tokens/total": 6853440, "tokens/train_per_sec_per_gpu": 27.16, "tokens/trainable": 89922} +{"epoch": 0.77734375, "grad_norm": 0.6012169122695923, "learning_rate": 7.476727793652011e-05, "loss": 0.02372904308140278, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 36.71, "memory/max_allocated (GiB)": 36.71, "ppl": 1.02401, "step": 199, "tokens/total": 6886272, "tokens/train_per_sec_per_gpu": 28.06, "tokens/trainable": 90401} +{"epoch": 0.78125, "grad_norm": 0.5336673259735107, "learning_rate": 7.450608454068415e-05, "loss": 0.01362901646643877, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.01372, "step": 200, "tokens/total": 6920624, "tokens/train_per_sec_per_gpu": 26.6, "tokens/trainable": 90842} +{"epoch": 0.78515625, "grad_norm": 0.24568845331668854, "learning_rate": 7.424407941704987e-05, "loss": 0.0032017850317060947, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.00321, "step": 201, "tokens/total": 6957328, "tokens/train_per_sec_per_gpu": 27.23, "tokens/trainable": 91343} +{"epoch": 0.7890625, "grad_norm": 0.5719943642616272, "learning_rate": 7.398127346871986e-05, "loss": 0.0348520427942276, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.03547, "step": 202, "tokens/total": 6994048, "tokens/train_per_sec_per_gpu": 24.33, "tokens/trainable": 91798} +{"epoch": 0.79296875, "grad_norm": 0.6882322430610657, "learning_rate": 7.371767763212238e-05, "loss": 0.0042281849309802055, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.00424, "step": 203, "tokens/total": 7029040, "tokens/train_per_sec_per_gpu": 26.48, "tokens/trainable": 92251} +{"epoch": 0.796875, "grad_norm": 0.5420174598693848, "learning_rate": 7.345330287655617e-05, "loss": 0.009847057983279228, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.0099, "step": 204, "tokens/total": 7067104, "tokens/train_per_sec_per_gpu": 22.65, "tokens/trainable": 92714} +{"epoch": 0.80078125, "grad_norm": 0.45540285110473633, "learning_rate": 7.31881602037339e-05, "loss": 0.01130837481468916, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.01137, "step": 205, "tokens/total": 7099136, "tokens/train_per_sec_per_gpu": 26.43, "tokens/trainable": 93154} +{"epoch": 0.8046875, "grad_norm": 0.1259809285402298, "learning_rate": 7.29222606473245e-05, "loss": 0.002576239639893174, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.00258, "step": 206, "tokens/total": 7132208, "tokens/train_per_sec_per_gpu": 26.33, "tokens/trainable": 93599} +{"epoch": 0.80859375, "grad_norm": 0.15032470226287842, "learning_rate": 7.265561527249383e-05, "loss": 0.0026257773861289024, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.00263, "step": 207, "tokens/total": 7166256, "tokens/train_per_sec_per_gpu": 27.58, "tokens/trainable": 94036} +{"epoch": 0.8125, "grad_norm": 0.991815984249115, "learning_rate": 7.238823517544436e-05, "loss": 0.018292998895049095, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.01846, "step": 208, "tokens/total": 7203152, "tokens/train_per_sec_per_gpu": 28.16, "tokens/trainable": 94506} +{"epoch": 0.81640625, "grad_norm": 0.23220688104629517, "learning_rate": 7.212013148295333e-05, "loss": 0.004071209579706192, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.00408, "step": 209, "tokens/total": 7237264, "tokens/train_per_sec_per_gpu": 29.73, "tokens/trainable": 94988} +{"epoch": 0.8203125, "grad_norm": 0.2603696882724762, "learning_rate": 7.185131535190975e-05, "loss": 0.014942127279937267, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.01505, "step": 210, "tokens/total": 7269664, "tokens/train_per_sec_per_gpu": 29.9, "tokens/trainable": 95497} +{"epoch": 0.82421875, "grad_norm": 0.4712175130844116, "learning_rate": 7.158179796885005e-05, "loss": 0.007456239312887192, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.00748, "step": 211, "tokens/total": 7304080, "tokens/train_per_sec_per_gpu": 27.45, "tokens/trainable": 95948} +{"epoch": 0.828125, "grad_norm": 0.4043090343475342, "learning_rate": 7.131159054949273e-05, "loss": 0.007285993546247482, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00731, "step": 212, "tokens/total": 7342480, "tokens/train_per_sec_per_gpu": 22.27, "tokens/trainable": 96422} +{"epoch": 0.83203125, "grad_norm": 0.2990400493144989, "learning_rate": 7.104070433827139e-05, "loss": 0.00416489876806736, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00417, "step": 213, "tokens/total": 7378032, "tokens/train_per_sec_per_gpu": 22.9, "tokens/trainable": 96821} +{"epoch": 0.8359375, "grad_norm": 0.2513885498046875, "learning_rate": 7.076915060786705e-05, "loss": 0.003593567293137312, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.96, "memory/max_allocated (GiB)": 37.96, "ppl": 1.0036, "step": 214, "tokens/total": 7410928, "tokens/train_per_sec_per_gpu": 24.17, "tokens/trainable": 97263} +{"epoch": 0.83984375, "grad_norm": 0.4701872169971466, "learning_rate": 7.049694065873882e-05, "loss": 0.01983296126127243, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.02003, "step": 215, "tokens/total": 7445664, "tokens/train_per_sec_per_gpu": 27.89, "tokens/trainable": 97720} +{"epoch": 0.84375, "grad_norm": 0.6178871989250183, "learning_rate": 7.022408581865382e-05, "loss": 0.009833140298724174, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.69, "memory/max_allocated (GiB)": 38.69, "ppl": 1.00988, "step": 216, "tokens/total": 7479984, "tokens/train_per_sec_per_gpu": 25.42, "tokens/trainable": 98194} +{"epoch": 0.84765625, "grad_norm": 0.12455417960882187, "learning_rate": 6.99505974422157e-05, "loss": 0.0016133758472278714, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.00161, "step": 217, "tokens/total": 7514144, "tokens/train_per_sec_per_gpu": 28.33, "tokens/trainable": 98667} +{"epoch": 0.8515625, "grad_norm": 0.08379212021827698, "learning_rate": 6.967648691039213e-05, "loss": 0.0011880011297762394, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 36.73, "memory/max_allocated (GiB)": 36.73, "ppl": 1.00119, "step": 218, "tokens/total": 7546256, "tokens/train_per_sec_per_gpu": 27.88, "tokens/trainable": 99100} +{"epoch": 0.85546875, "grad_norm": 0.04066484794020653, "learning_rate": 6.940176563004123e-05, "loss": 0.0009843821171671152, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00098, "step": 219, "tokens/total": 7580432, "tokens/train_per_sec_per_gpu": 25.88, "tokens/trainable": 99532} +{"epoch": 0.859375, "grad_norm": 0.39679935574531555, "learning_rate": 6.912644503343682e-05, "loss": 0.00833575613796711, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.00837, "step": 220, "tokens/total": 7617904, "tokens/train_per_sec_per_gpu": 23.4, "tokens/trainable": 99970} +{"epoch": 0.86328125, "grad_norm": 0.057323794811964035, "learning_rate": 6.885053657779273e-05, "loss": 0.001195659744553268, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.0012, "step": 221, "tokens/total": 7649696, "tokens/train_per_sec_per_gpu": 28.34, "tokens/trainable": 100442} +{"epoch": 0.8671875, "grad_norm": 0.6735663414001465, "learning_rate": 6.857405174478604e-05, "loss": 0.010716051794588566, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.01077, "step": 222, "tokens/total": 7684464, "tokens/train_per_sec_per_gpu": 24.28, "tokens/trainable": 100895} +{"epoch": 0.87109375, "grad_norm": 0.48240146040916443, "learning_rate": 6.82970020400792e-05, "loss": 0.01049987692385912, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.01056, "step": 223, "tokens/total": 7721936, "tokens/train_per_sec_per_gpu": 22.17, "tokens/trainable": 101342} +{"epoch": 0.875, "grad_norm": 0.7384056448936462, "learning_rate": 6.801939899284132e-05, "loss": 0.018641676753759384, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.34, "memory/max_allocated (GiB)": 38.34, "ppl": 1.01882, "step": 224, "tokens/total": 7758160, "tokens/train_per_sec_per_gpu": 26.47, "tokens/trainable": 101848} +{"epoch": 0.87890625, "grad_norm": 0.09246401488780975, "learning_rate": 6.774125415526827e-05, "loss": 0.0015391531633213162, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00154, "step": 225, "tokens/total": 7791216, "tokens/train_per_sec_per_gpu": 27.69, "tokens/trainable": 102319} +{"epoch": 0.8828125, "grad_norm": 0.3855593502521515, "learning_rate": 6.746257910210214e-05, "loss": 0.0046091750264167786, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.12, "memory/max_allocated (GiB)": 37.12, "ppl": 1.00462, "step": 226, "tokens/total": 7824400, "tokens/train_per_sec_per_gpu": 27.93, "tokens/trainable": 102769} +{"epoch": 0.88671875, "grad_norm": 0.286342978477478, "learning_rate": 6.718338543014937e-05, "loss": 0.005295807495713234, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 35.75, "memory/max_allocated (GiB)": 35.75, "ppl": 1.00531, "step": 227, "tokens/total": 7854672, "tokens/train_per_sec_per_gpu": 30.77, "tokens/trainable": 103219} +{"epoch": 0.890625, "grad_norm": 0.14521870017051697, "learning_rate": 6.69036847577983e-05, "loss": 0.00302003207616508, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.00302, "step": 228, "tokens/total": 7888592, "tokens/train_per_sec_per_gpu": 26.65, "tokens/trainable": 103675} +{"epoch": 0.89453125, "grad_norm": 0.3846345841884613, "learning_rate": 6.662348872453553e-05, "loss": 0.009034126065671444, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.33, "memory/max_allocated (GiB)": 38.33, "ppl": 1.00908, "step": 229, "tokens/total": 7924848, "tokens/train_per_sec_per_gpu": 27.39, "tokens/trainable": 104136} +{"epoch": 0.8984375, "grad_norm": 0.12058359384536743, "learning_rate": 6.63428089904618e-05, "loss": 0.00216121319681406, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.00216, "step": 230, "tokens/total": 7963856, "tokens/train_per_sec_per_gpu": 24.03, "tokens/trainable": 104588} +{"epoch": 0.90234375, "grad_norm": 0.14703203737735748, "learning_rate": 6.60616572358065e-05, "loss": 0.0024246657267212868, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.00243, "step": 231, "tokens/total": 7998320, "tokens/train_per_sec_per_gpu": 25.55, "tokens/trainable": 104988} +{"epoch": 0.90625, "grad_norm": 0.25758081674575806, "learning_rate": 6.578004516044172e-05, "loss": 0.003381672315299511, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00339, "step": 232, "tokens/total": 8034096, "tokens/train_per_sec_per_gpu": 27.46, "tokens/trainable": 105455} +{"epoch": 0.91015625, "grad_norm": 0.0615207813680172, "learning_rate": 6.549798448339548e-05, "loss": 0.0012274275068193674, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.59, "memory/max_allocated (GiB)": 38.59, "ppl": 1.00123, "step": 233, "tokens/total": 8071360, "tokens/train_per_sec_per_gpu": 24.89, "tokens/trainable": 105918} +{"epoch": 0.9140625, "grad_norm": 0.27614882588386536, "learning_rate": 6.521548694236384e-05, "loss": 0.006711636669933796, "memory/device_reserved (GiB)": 43.64, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00673, "step": 234, "tokens/total": 8108560, "tokens/train_per_sec_per_gpu": 23.18, "tokens/trainable": 106355} +{"epoch": 0.91796875, "grad_norm": 0.35208559036254883, "learning_rate": 6.493256429322259e-05, "loss": 0.004152501933276653, "memory/device_reserved (GiB)": 43.64, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.00416, "step": 235, "tokens/total": 8144816, "tokens/train_per_sec_per_gpu": 22.14, "tokens/trainable": 106786} +{"epoch": 0.921875, "grad_norm": 0.5369355082511902, "learning_rate": 6.464922830953799e-05, "loss": 0.020387442782521248, "memory/device_reserved (GiB)": 43.64, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.0206, "step": 236, "tokens/total": 8183680, "tokens/train_per_sec_per_gpu": 22.31, "tokens/trainable": 107219} +{"epoch": 0.92578125, "grad_norm": 0.6097026467323303, "learning_rate": 6.436549078207688e-05, "loss": 0.01725783944129944, "memory/device_reserved (GiB)": 43.64, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.01741, "step": 237, "tokens/total": 8216688, "tokens/train_per_sec_per_gpu": 28.69, "tokens/trainable": 107679} +{"epoch": 0.9296875, "grad_norm": 0.3281494677066803, "learning_rate": 6.408136351831592e-05, "loss": 0.003923898097127676, "memory/device_reserved (GiB)": 43.64, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.00393, "step": 238, "tokens/total": 8249104, "tokens/train_per_sec_per_gpu": 24.01, "tokens/trainable": 108119} +{"epoch": 0.93359375, "grad_norm": 0.21244275569915771, "learning_rate": 6.379685834195036e-05, "loss": 0.0042204586789011955, "memory/device_reserved (GiB)": 43.64, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00423, "step": 239, "tokens/total": 8288528, "tokens/train_per_sec_per_gpu": 24.41, "tokens/trainable": 108626} +{"epoch": 0.9375, "grad_norm": 0.40406861901283264, "learning_rate": 6.351198709240186e-05, "loss": 0.008346177637577057, "memory/device_reserved (GiB)": 43.64, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.00838, "step": 240, "tokens/total": 8328464, "tokens/train_per_sec_per_gpu": 21.02, "tokens/trainable": 109079} +{"epoch": 0.94140625, "grad_norm": 0.26276516914367676, "learning_rate": 6.32267616243259e-05, "loss": 0.004456231836229563, "memory/device_reserved (GiB)": 43.64, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.00447, "step": 241, "tokens/total": 8361264, "tokens/train_per_sec_per_gpu": 28.57, "tokens/trainable": 109542} +{"epoch": 0.9453125, "grad_norm": 0.45297959446907043, "learning_rate": 6.294119380711849e-05, "loss": 0.0034690406173467636, "memory/device_reserved (GiB)": 43.64, "memory/max_active (GiB)": 38.39, "memory/max_allocated (GiB)": 38.39, "ppl": 1.00348, "step": 242, "tokens/total": 8395552, "tokens/train_per_sec_per_gpu": 23.69, "tokens/trainable": 109969} +{"epoch": 0.94921875, "grad_norm": 0.6172285079956055, "learning_rate": 6.265529552442209e-05, "loss": 0.010155344381928444, "memory/device_reserved (GiB)": 43.64, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.01021, "step": 243, "tokens/total": 8430656, "tokens/train_per_sec_per_gpu": 27.89, "tokens/trainable": 110431} +{"epoch": 0.953125, "grad_norm": 0.14683257043361664, "learning_rate": 6.236907867363127e-05, "loss": 0.0026591059286147356, "memory/device_reserved (GiB)": 43.64, "memory/max_active (GiB)": 38.17, "memory/max_allocated (GiB)": 38.17, "ppl": 1.00266, "step": 244, "tokens/total": 8465232, "tokens/train_per_sec_per_gpu": 26.51, "tokens/trainable": 110877} +{"epoch": 0.95703125, "grad_norm": 0.14453428983688354, "learning_rate": 6.208255516539749e-05, "loss": 0.003216308541595936, "memory/device_reserved (GiB)": 43.64, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.00322, "step": 245, "tokens/total": 8497552, "tokens/train_per_sec_per_gpu": 26.43, "tokens/trainable": 111350} +{"epoch": 0.9609375, "grad_norm": 0.3892906606197357, "learning_rate": 6.179573692313344e-05, "loss": 0.00806964747607708, "memory/device_reserved (GiB)": 43.64, "memory/max_active (GiB)": 38.8, "memory/max_allocated (GiB)": 38.8, "ppl": 1.0081, "step": 246, "tokens/total": 8534336, "tokens/train_per_sec_per_gpu": 25.91, "tokens/trainable": 111798} +{"epoch": 0.96484375, "grad_norm": 0.11754254251718521, "learning_rate": 6.150863588251694e-05, "loss": 0.0013056739699095488, "memory/device_reserved (GiB)": 43.64, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.00131, "step": 247, "tokens/total": 8574224, "tokens/train_per_sec_per_gpu": 22.54, "tokens/trainable": 112264} +{"epoch": 0.96875, "grad_norm": 0.163943350315094, "learning_rate": 6.122126399099419e-05, "loss": 0.0019178069196641445, "memory/device_reserved (GiB)": 43.64, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.00192, "step": 248, "tokens/total": 8613616, "tokens/train_per_sec_per_gpu": 26.7, "tokens/trainable": 112772} +{"epoch": 0.97265625, "grad_norm": 0.5614367723464966, "learning_rate": 6.0933633207282615e-05, "loss": 0.009479149244725704, "memory/device_reserved (GiB)": 43.64, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.00952, "step": 249, "tokens/total": 8650048, "tokens/train_per_sec_per_gpu": 27.23, "tokens/trainable": 113278} +{"epoch": 0.9765625, "grad_norm": 0.7541083693504333, "learning_rate": 6.064575550087316e-05, "loss": 0.0073038130067288876, "memory/device_reserved (GiB)": 43.64, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00733, "step": 250, "tokens/total": 8686688, "tokens/train_per_sec_per_gpu": 23.59, "tokens/trainable": 113716} +{"epoch": 0.98046875, "grad_norm": 0.1395178586244583, "learning_rate": 6.0357642851532245e-05, "loss": 0.0015205288073047996, "memory/device_reserved (GiB)": 43.64, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00152, "step": 251, "tokens/total": 8723376, "tokens/train_per_sec_per_gpu": 25.61, "tokens/trainable": 114207} +{"epoch": 0.984375, "grad_norm": 0.2155676782131195, "learning_rate": 6.0069307248803294e-05, "loss": 0.0034367970656603575, "memory/device_reserved (GiB)": 43.64, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00344, "step": 252, "tokens/total": 8758736, "tokens/train_per_sec_per_gpu": 21.81, "tokens/trainable": 114628} +{"epoch": 0.98828125, "grad_norm": 0.1902555376291275, "learning_rate": 5.9780760691507635e-05, "loss": 0.0024920974392443895, "memory/device_reserved (GiB)": 43.64, "memory/max_active (GiB)": 36.33, "memory/max_allocated (GiB)": 36.33, "ppl": 1.0025, "step": 253, "tokens/total": 8790480, "tokens/train_per_sec_per_gpu": 26.57, "tokens/trainable": 115060} +{"epoch": 0.9921875, "grad_norm": 0.2312471568584442, "learning_rate": 5.9492015187245334e-05, "loss": 0.0012792141642421484, "memory/device_reserved (GiB)": 43.64, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00128, "step": 254, "tokens/total": 8828224, "tokens/train_per_sec_per_gpu": 23.14, "tokens/trainable": 115514} +{"epoch": 0.99609375, "grad_norm": 0.1878967434167862, "learning_rate": 5.920308275189541e-05, "loss": 0.002529020421206951, "memory/device_reserved (GiB)": 43.64, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.00253, "step": 255, "tokens/total": 8860400, "tokens/train_per_sec_per_gpu": 25.34, "tokens/trainable": 115915} +{"epoch": 1.0, "grad_norm": 0.017917340621352196, "learning_rate": 5.8913975409115874e-05, "loss": 0.0003017504932358861, "memory/device_reserved (GiB)": 43.64, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.0003, "step": 256, "tokens/total": 8893760, "tokens/train_per_sec_per_gpu": 25.24, "tokens/trainable": 116378} +{"epoch": 1.00390625, "grad_norm": 0.024991951882839203, "learning_rate": 5.8624705189843395e-05, "loss": 0.0003361971757840365, "memory/device_reserved (GiB)": 43.64, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 1.00034, "step": 257, "tokens/total": 8930976, "tokens/train_per_sec_per_gpu": 24.4, "tokens/trainable": 116865} +{"epoch": 1.0078125, "grad_norm": 0.1101839691400528, "learning_rate": 5.833528413179249e-05, "loss": 0.0006616105092689395, "memory/device_reserved (GiB)": 42.25, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00066, "step": 258, "tokens/total": 8967472, "tokens/train_per_sec_per_gpu": 27.64, "tokens/trainable": 117363} +{"epoch": 1.01171875, "grad_norm": 1.0889464616775513, "learning_rate": 5.80457242789548e-05, "loss": 0.019171681255102158, "memory/device_reserved (GiB)": 42.31, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.01936, "step": 259, "tokens/total": 9007456, "tokens/train_per_sec_per_gpu": 23.67, "tokens/trainable": 117860} +{"epoch": 1.015625, "grad_norm": 0.21476298570632935, "learning_rate": 5.77560376810977e-05, "loss": 0.0023706259671598673, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 1.00237, "step": 260, "tokens/total": 9041104, "tokens/train_per_sec_per_gpu": 29.03, "tokens/trainable": 118324} +{"epoch": 1.01953125, "grad_norm": 0.03491959348320961, "learning_rate": 5.7466236393263005e-05, "loss": 0.00025918849860318005, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.00026, "step": 261, "tokens/total": 9076176, "tokens/train_per_sec_per_gpu": 25.38, "tokens/trainable": 118770} +{"epoch": 1.0234375, "grad_norm": 0.23668822646141052, "learning_rate": 5.717633247526522e-05, "loss": 0.0034977656323462725, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.0035, "step": 262, "tokens/total": 9107472, "tokens/train_per_sec_per_gpu": 27.65, "tokens/trainable": 119206} +{"epoch": 1.02734375, "grad_norm": 0.47114431858062744, "learning_rate": 5.688633799118971e-05, "loss": 0.0006874979590065777, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.00069, "step": 263, "tokens/total": 9139552, "tokens/train_per_sec_per_gpu": 23.13, "tokens/trainable": 119610} +{"epoch": 1.03125, "grad_norm": 0.006966397166252136, "learning_rate": 5.659626500889066e-05, "loss": 7.915669993963093e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00008, "step": 264, "tokens/total": 9173424, "tokens/train_per_sec_per_gpu": 25.65, "tokens/trainable": 120080} +{"epoch": 1.03515625, "grad_norm": 0.023415811359882355, "learning_rate": 5.6306125599488905e-05, "loss": 0.00034128810511901975, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.00034, "step": 265, "tokens/total": 9203984, "tokens/train_per_sec_per_gpu": 31.98, "tokens/trainable": 120519} +{"epoch": 1.0390625, "grad_norm": 0.32039058208465576, "learning_rate": 5.601593183686955e-05, "loss": 0.004803857766091824, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.69, "memory/max_allocated (GiB)": 38.69, "ppl": 1.00482, "step": 266, "tokens/total": 9241248, "tokens/train_per_sec_per_gpu": 25.81, "tokens/trainable": 120995} +{"epoch": 1.04296875, "grad_norm": 0.14879995584487915, "learning_rate": 5.572569579717961e-05, "loss": 0.0017341563943773508, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.18, "memory/max_allocated (GiB)": 39.18, "ppl": 1.00174, "step": 267, "tokens/total": 9276768, "tokens/train_per_sec_per_gpu": 24.32, "tokens/trainable": 121459} +{"epoch": 1.046875, "grad_norm": 0.49077674746513367, "learning_rate": 5.543542955832538e-05, "loss": 0.015207573771476746, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.01532, "step": 268, "tokens/total": 9316528, "tokens/train_per_sec_per_gpu": 21.55, "tokens/trainable": 121902} +{"epoch": 1.05078125, "grad_norm": 0.38620007038116455, "learning_rate": 5.514514519946986e-05, "loss": 0.006524571217596531, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.00655, "step": 269, "tokens/total": 9356352, "tokens/train_per_sec_per_gpu": 25.53, "tokens/trainable": 122399} +{"epoch": 1.0546875, "grad_norm": 0.08649080991744995, "learning_rate": 5.485485480053015e-05, "loss": 0.001356672728434205, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.00136, "step": 270, "tokens/total": 9394576, "tokens/train_per_sec_per_gpu": 21.74, "tokens/trainable": 122804} +{"epoch": 1.05859375, "grad_norm": 0.12251318991184235, "learning_rate": 5.4564570441674645e-05, "loss": 0.001763396430760622, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.92, "memory/max_allocated (GiB)": 36.92, "ppl": 1.00176, "step": 271, "tokens/total": 9427840, "tokens/train_per_sec_per_gpu": 25.35, "tokens/trainable": 123244} +{"epoch": 1.0625, "grad_norm": 0.25291335582733154, "learning_rate": 5.42743042028204e-05, "loss": 0.0028065370861440897, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.00281, "step": 272, "tokens/total": 9463536, "tokens/train_per_sec_per_gpu": 20.37, "tokens/trainable": 123683} +{"epoch": 1.06640625, "grad_norm": 0.05733741447329521, "learning_rate": 5.3984068163130464e-05, "loss": 0.0009439954883418977, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.09, "memory/max_allocated (GiB)": 37.09, "ppl": 1.00094, "step": 273, "tokens/total": 9497248, "tokens/train_per_sec_per_gpu": 27.95, "tokens/trainable": 124153} +{"epoch": 1.0703125, "grad_norm": 0.34325745701789856, "learning_rate": 5.369387440051111e-05, "loss": 0.0034382750745862722, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.00344, "step": 274, "tokens/total": 9530784, "tokens/train_per_sec_per_gpu": 25.27, "tokens/trainable": 124616} +{"epoch": 1.07421875, "grad_norm": 0.11662451922893524, "learning_rate": 5.340373499110935e-05, "loss": 0.0011931897606700659, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00119, "step": 275, "tokens/total": 9566656, "tokens/train_per_sec_per_gpu": 26.26, "tokens/trainable": 125109} +{"epoch": 1.078125, "grad_norm": 0.02289101667702198, "learning_rate": 5.3113662008810304e-05, "loss": 0.00026019031065516174, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00026, "step": 276, "tokens/total": 9606672, "tokens/train_per_sec_per_gpu": 24.82, "tokens/trainable": 125591} +{"epoch": 1.08203125, "grad_norm": 0.020044373348355293, "learning_rate": 5.282366752473479e-05, "loss": 0.00025308545446023345, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 1.00025, "step": 277, "tokens/total": 9643680, "tokens/train_per_sec_per_gpu": 24.38, "tokens/trainable": 126065} +{"epoch": 1.0859375, "grad_norm": 0.014898023568093777, "learning_rate": 5.2533763606737005e-05, "loss": 0.00026786854141391814, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00027, "step": 278, "tokens/total": 9683200, "tokens/train_per_sec_per_gpu": 19.15, "tokens/trainable": 126480} +{"epoch": 1.08984375, "grad_norm": 0.02032559923827648, "learning_rate": 5.224396231890232e-05, "loss": 0.0002624919288791716, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.00026, "step": 279, "tokens/total": 9714672, "tokens/train_per_sec_per_gpu": 29.9, "tokens/trainable": 126929} +{"epoch": 1.09375, "grad_norm": 0.5250305533409119, "learning_rate": 5.195427572104522e-05, "loss": 0.0033132436219602823, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00332, "step": 280, "tokens/total": 9749104, "tokens/train_per_sec_per_gpu": 23.76, "tokens/trainable": 127366} +{"epoch": 1.09765625, "grad_norm": 0.1448182612657547, "learning_rate": 5.166471586820751e-05, "loss": 0.0018652488943189383, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.35, "memory/max_allocated (GiB)": 39.35, "ppl": 1.00187, "step": 281, "tokens/total": 9785712, "tokens/train_per_sec_per_gpu": 23.68, "tokens/trainable": 127831} +{"epoch": 1.1015625, "grad_norm": 0.8965591192245483, "learning_rate": 5.1375294810156615e-05, "loss": 0.01750091463327408, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.01765, "step": 282, "tokens/total": 9820544, "tokens/train_per_sec_per_gpu": 25.78, "tokens/trainable": 128278} +{"epoch": 1.10546875, "grad_norm": 0.04843295365571976, "learning_rate": 5.1086024590884144e-05, "loss": 0.0005098608089610934, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00051, "step": 283, "tokens/total": 9857952, "tokens/train_per_sec_per_gpu": 24.9, "tokens/trainable": 128727} +{"epoch": 1.109375, "grad_norm": 0.11989283561706543, "learning_rate": 5.079691724810461e-05, "loss": 0.0011680147144943476, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.33, "memory/max_allocated (GiB)": 38.33, "ppl": 1.00117, "step": 284, "tokens/total": 9894128, "tokens/train_per_sec_per_gpu": 21.7, "tokens/trainable": 129162} +{"epoch": 1.11328125, "grad_norm": 0.1640567183494568, "learning_rate": 5.0507984812754684e-05, "loss": 0.00207435037009418, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.17, "memory/max_allocated (GiB)": 37.17, "ppl": 1.00208, "step": 285, "tokens/total": 9927568, "tokens/train_per_sec_per_gpu": 28.63, "tokens/trainable": 129659} +{"epoch": 1.1171875, "grad_norm": 0.16157133877277374, "learning_rate": 5.021923930849237e-05, "loss": 0.002695476170629263, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.0027, "step": 286, "tokens/total": 9961888, "tokens/train_per_sec_per_gpu": 27.86, "tokens/trainable": 130145} +{"epoch": 1.12109375, "grad_norm": 0.2608509957790375, "learning_rate": 4.99306927511967e-05, "loss": 0.003536800155416131, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.31, "memory/max_allocated (GiB)": 37.31, "ppl": 1.00354, "step": 287, "tokens/total": 9995696, "tokens/train_per_sec_per_gpu": 27.79, "tokens/trainable": 130585} +{"epoch": 1.125, "grad_norm": 0.09669328480958939, "learning_rate": 4.964235714846775e-05, "loss": 0.0010924824746325612, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.72, "memory/max_allocated (GiB)": 36.72, "ppl": 1.00109, "step": 288, "tokens/total": 10028000, "tokens/train_per_sec_per_gpu": 27.09, "tokens/trainable": 131005} +{"epoch": 1.12890625, "grad_norm": 0.0074820732697844505, "learning_rate": 4.9354244499126866e-05, "loss": 0.00016398908337578177, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.16, "memory/max_allocated (GiB)": 36.16, "ppl": 1.00016, "step": 289, "tokens/total": 10059728, "tokens/train_per_sec_per_gpu": 29.32, "tokens/trainable": 131471} +{"epoch": 1.1328125, "grad_norm": 0.3667893707752228, "learning_rate": 4.90663667927174e-05, "loss": 0.0036140785086899996, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.00362, "step": 290, "tokens/total": 10092576, "tokens/train_per_sec_per_gpu": 28.84, "tokens/trainable": 131920} +{"epoch": 1.13671875, "grad_norm": 0.013864834792912006, "learning_rate": 4.877873600900581e-05, "loss": 0.00021487635967787355, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.00021, "step": 291, "tokens/total": 10128608, "tokens/train_per_sec_per_gpu": 21.75, "tokens/trainable": 132379} +{"epoch": 1.140625, "grad_norm": 0.2332262247800827, "learning_rate": 4.849136411748306e-05, "loss": 0.0032431299332529306, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.00325, "step": 292, "tokens/total": 10166192, "tokens/train_per_sec_per_gpu": 24.82, "tokens/trainable": 132851} +{"epoch": 1.14453125, "grad_norm": 0.4427623450756073, "learning_rate": 4.8204263076866574e-05, "loss": 0.007988326251506805, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00802, "step": 293, "tokens/total": 10201840, "tokens/train_per_sec_per_gpu": 29.55, "tokens/trainable": 133351} +{"epoch": 1.1484375, "grad_norm": 0.011711346916854382, "learning_rate": 4.791744483460251e-05, "loss": 0.000138066679937765, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00014, "step": 294, "tokens/total": 10236048, "tokens/train_per_sec_per_gpu": 24.19, "tokens/trainable": 133784} +{"epoch": 1.15234375, "grad_norm": 0.02930123172700405, "learning_rate": 4.7630921326368736e-05, "loss": 0.00029663278837688267, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.0003, "step": 295, "tokens/total": 10274688, "tokens/train_per_sec_per_gpu": 19.99, "tokens/trainable": 134209} +{"epoch": 1.15625, "grad_norm": 0.2274376004934311, "learning_rate": 4.7344704475577916e-05, "loss": 0.0021704602986574173, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 35.66, "memory/max_allocated (GiB)": 35.66, "ppl": 1.00217, "step": 296, "tokens/total": 10304576, "tokens/train_per_sec_per_gpu": 30.22, "tokens/trainable": 134645} +{"epoch": 1.16015625, "grad_norm": 0.41204315423965454, "learning_rate": 4.705880619288153e-05, "loss": 0.006886261980980635, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.21, "memory/max_allocated (GiB)": 38.21, "ppl": 1.00691, "step": 297, "tokens/total": 10340720, "tokens/train_per_sec_per_gpu": 21.98, "tokens/trainable": 135047} +{"epoch": 1.1640625, "grad_norm": 0.018839234486222267, "learning_rate": 4.677323837567412e-05, "loss": 0.00032928993459790945, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00033, "step": 298, "tokens/total": 10373376, "tokens/train_per_sec_per_gpu": 25.33, "tokens/trainable": 135474} +{"epoch": 1.16796875, "grad_norm": 0.004377475939691067, "learning_rate": 4.6488012907598146e-05, "loss": 7.67997553339228e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00008, "step": 299, "tokens/total": 10406128, "tokens/train_per_sec_per_gpu": 27.91, "tokens/trainable": 135948} +{"epoch": 1.171875, "grad_norm": 0.07994387298822403, "learning_rate": 4.620314165804964e-05, "loss": 0.0005453468766063452, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.3, "memory/max_allocated (GiB)": 38.3, "ppl": 1.00055, "step": 300, "tokens/total": 10441792, "tokens/train_per_sec_per_gpu": 25.65, "tokens/trainable": 136421} +{"epoch": 1.17578125, "grad_norm": 0.03351907432079315, "learning_rate": 4.591863648168407e-05, "loss": 0.00027540663722902536, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00028, "step": 301, "tokens/total": 10476128, "tokens/train_per_sec_per_gpu": 26.29, "tokens/trainable": 136886} +{"epoch": 1.1796875, "grad_norm": 0.20934566855430603, "learning_rate": 4.5634509217923135e-05, "loss": 0.001385676208883524, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00139, "step": 302, "tokens/total": 10509472, "tokens/train_per_sec_per_gpu": 25.77, "tokens/trainable": 137321} +{"epoch": 1.18359375, "grad_norm": 0.012824197299778461, "learning_rate": 4.535077169046201e-05, "loss": 0.0001479046477470547, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00015, "step": 303, "tokens/total": 10546032, "tokens/train_per_sec_per_gpu": 25.12, "tokens/trainable": 137758} +{"epoch": 1.1875, "grad_norm": 0.54702228307724, "learning_rate": 4.506743570677743e-05, "loss": 0.010637952014803886, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.01069, "step": 304, "tokens/total": 10576624, "tokens/train_per_sec_per_gpu": 31.26, "tokens/trainable": 138190} +{"epoch": 1.19140625, "grad_norm": 0.12707771360874176, "learning_rate": 4.478451305763618e-05, "loss": 0.001377716544084251, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.95, "memory/max_allocated (GiB)": 36.95, "ppl": 1.00138, "step": 305, "tokens/total": 10609760, "tokens/train_per_sec_per_gpu": 30.54, "tokens/trainable": 138650} +{"epoch": 1.1953125, "grad_norm": 0.05824708566069603, "learning_rate": 4.450201551660454e-05, "loss": 0.00046382739674299955, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.51, "memory/max_allocated (GiB)": 36.51, "ppl": 1.00046, "step": 306, "tokens/total": 10641472, "tokens/train_per_sec_per_gpu": 26.23, "tokens/trainable": 139082} +{"epoch": 1.19921875, "grad_norm": 0.4422692060470581, "learning_rate": 4.4219954839558276e-05, "loss": 0.004649404902011156, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00466, "step": 307, "tokens/total": 10676160, "tokens/train_per_sec_per_gpu": 25.35, "tokens/trainable": 139523} +{"epoch": 1.203125, "grad_norm": 0.3423228859901428, "learning_rate": 4.393834276419352e-05, "loss": 0.006535888649523258, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.83, "memory/max_allocated (GiB)": 35.83, "ppl": 1.00656, "step": 308, "tokens/total": 10706384, "tokens/train_per_sec_per_gpu": 32.28, "tokens/trainable": 140010} +{"epoch": 1.20703125, "grad_norm": 0.3920845091342926, "learning_rate": 4.36571910095382e-05, "loss": 0.01263540331274271, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.01272, "step": 309, "tokens/total": 10744496, "tokens/train_per_sec_per_gpu": 25.43, "tokens/trainable": 140454} +{"epoch": 1.2109375, "grad_norm": 0.44357186555862427, "learning_rate": 4.337651127546448e-05, "loss": 0.003984509967267513, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00399, "step": 310, "tokens/total": 10780176, "tokens/train_per_sec_per_gpu": 27.54, "tokens/trainable": 140905} +{"epoch": 1.21484375, "grad_norm": 0.1486307978630066, "learning_rate": 4.3096315242201736e-05, "loss": 0.000626693363301456, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.61, "memory/max_allocated (GiB)": 36.61, "ppl": 1.00063, "step": 311, "tokens/total": 10813136, "tokens/train_per_sec_per_gpu": 26.24, "tokens/trainable": 141339} +{"epoch": 1.21875, "grad_norm": 0.0029545873403549194, "learning_rate": 4.2816614569850635e-05, "loss": 3.8606649468420073e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00004, "step": 312, "tokens/total": 10848912, "tokens/train_per_sec_per_gpu": 24.29, "tokens/trainable": 141778} +{"epoch": 1.22265625, "grad_norm": 0.012467483058571815, "learning_rate": 4.2537420897897864e-05, "loss": 0.00014106585877016187, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.00014, "step": 313, "tokens/total": 10884320, "tokens/train_per_sec_per_gpu": 26.98, "tokens/trainable": 142255} +{"epoch": 1.2265625, "grad_norm": 0.004398214630782604, "learning_rate": 4.225874584473174e-05, "loss": 9.342087287222967e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.00009, "step": 314, "tokens/total": 10917152, "tokens/train_per_sec_per_gpu": 25.94, "tokens/trainable": 142693} +{"epoch": 1.23046875, "grad_norm": 0.600506067276001, "learning_rate": 4.19806010071587e-05, "loss": 0.006384299602359533, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.0064, "step": 315, "tokens/total": 10955024, "tokens/train_per_sec_per_gpu": 24.29, "tokens/trainable": 143157} +{"epoch": 1.234375, "grad_norm": 0.10290878266096115, "learning_rate": 4.170299795992081e-05, "loss": 0.0014854817418381572, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.00149, "step": 316, "tokens/total": 10988416, "tokens/train_per_sec_per_gpu": 27.01, "tokens/trainable": 143603} +{"epoch": 1.23828125, "grad_norm": 0.2511488199234009, "learning_rate": 4.142594825521398e-05, "loss": 0.0009653712040744722, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.00097, "step": 317, "tokens/total": 11027712, "tokens/train_per_sec_per_gpu": 25.87, "tokens/trainable": 144084} +{"epoch": 1.2421875, "grad_norm": 0.011018411256372929, "learning_rate": 4.114946342220728e-05, "loss": 0.00017514111823402345, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.00018, "step": 318, "tokens/total": 11063920, "tokens/train_per_sec_per_gpu": 23.94, "tokens/trainable": 144532} +{"epoch": 1.24609375, "grad_norm": 0.18343055248260498, "learning_rate": 4.087355496656321e-05, "loss": 0.002101108431816101, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.0021, "step": 319, "tokens/total": 11097952, "tokens/train_per_sec_per_gpu": 28.41, "tokens/trainable": 145004} +{"epoch": 1.25, "grad_norm": 0.04668847844004631, "learning_rate": 4.05982343699588e-05, "loss": 0.0009137409506365657, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00091, "step": 320, "tokens/total": 11137168, "tokens/train_per_sec_per_gpu": 25.06, "tokens/trainable": 145498} +{"epoch": 1.25390625, "grad_norm": 0.032748088240623474, "learning_rate": 4.0323513089607876e-05, "loss": 0.00044244813034310937, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.00044, "step": 321, "tokens/total": 11172224, "tokens/train_per_sec_per_gpu": 24.26, "tokens/trainable": 145965} +{"epoch": 1.2578125, "grad_norm": 0.2146989107131958, "learning_rate": 4.004940255778431e-05, "loss": 0.003622473916038871, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00363, "step": 322, "tokens/total": 11210832, "tokens/train_per_sec_per_gpu": 23.85, "tokens/trainable": 146409} +{"epoch": 1.26171875, "grad_norm": 0.18467126786708832, "learning_rate": 3.977591418134619e-05, "loss": 0.00277198594994843, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.00278, "step": 323, "tokens/total": 11245984, "tokens/train_per_sec_per_gpu": 25.78, "tokens/trainable": 146849} +{"epoch": 1.265625, "grad_norm": 0.05217166990041733, "learning_rate": 3.95030593412612e-05, "loss": 0.0005839463556185365, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.00058, "step": 324, "tokens/total": 11281360, "tokens/train_per_sec_per_gpu": 24.56, "tokens/trainable": 147272} +{"epoch": 1.26953125, "grad_norm": 0.04762585088610649, "learning_rate": 3.923084939213296e-05, "loss": 0.00039995883707888424, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.88, "memory/max_allocated (GiB)": 35.88, "ppl": 1.0004, "step": 325, "tokens/total": 11312176, "tokens/train_per_sec_per_gpu": 27.67, "tokens/trainable": 147694} +{"epoch": 1.2734375, "grad_norm": 0.3646198809146881, "learning_rate": 3.895929566172861e-05, "loss": 0.021485401317477226, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.02172, "step": 326, "tokens/total": 11348192, "tokens/train_per_sec_per_gpu": 25.17, "tokens/trainable": 148158} +{"epoch": 1.27734375, "grad_norm": 0.009024699218571186, "learning_rate": 3.868840945050728e-05, "loss": 0.00011844778782688081, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.05, "memory/max_allocated (GiB)": 39.05, "ppl": 1.00012, "step": 327, "tokens/total": 11386704, "tokens/train_per_sec_per_gpu": 21.49, "tokens/trainable": 148590} +{"epoch": 1.28125, "grad_norm": 0.33419981598854065, "learning_rate": 3.841820203114995e-05, "loss": 0.006040711421519518, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.00606, "step": 328, "tokens/total": 11418448, "tokens/train_per_sec_per_gpu": 28.93, "tokens/trainable": 149037} +{"epoch": 1.28515625, "grad_norm": 0.052677154541015625, "learning_rate": 3.814868464809027e-05, "loss": 0.0005736262537539005, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.00057, "step": 329, "tokens/total": 11450304, "tokens/train_per_sec_per_gpu": 25.67, "tokens/trainable": 149439} +{"epoch": 1.2890625, "grad_norm": 0.06437280774116516, "learning_rate": 3.787986851704667e-05, "loss": 0.0007402126211673021, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.00074, "step": 330, "tokens/total": 11484048, "tokens/train_per_sec_per_gpu": 25.64, "tokens/trainable": 149869} +{"epoch": 1.29296875, "grad_norm": 0.10553140938282013, "learning_rate": 3.7611764824555654e-05, "loss": 0.0012297144858166575, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00123, "step": 331, "tokens/total": 11516032, "tokens/train_per_sec_per_gpu": 30.72, "tokens/trainable": 150343} +{"epoch": 1.296875, "grad_norm": 0.08837592601776123, "learning_rate": 3.734438472750619e-05, "loss": 0.00037410660297609866, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.00037, "step": 332, "tokens/total": 11555968, "tokens/train_per_sec_per_gpu": 23.09, "tokens/trainable": 150814} +{"epoch": 1.30078125, "grad_norm": 0.013647371903061867, "learning_rate": 3.707773935267552e-05, "loss": 0.00018383438873570412, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.46, "memory/max_allocated (GiB)": 38.46, "ppl": 1.00018, "step": 333, "tokens/total": 11592880, "tokens/train_per_sec_per_gpu": 26.86, "tokens/trainable": 151300} +{"epoch": 1.3046875, "grad_norm": 0.03408999368548393, "learning_rate": 3.68118397962661e-05, "loss": 0.0003508212394081056, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.00035, "step": 334, "tokens/total": 11627792, "tokens/train_per_sec_per_gpu": 23.19, "tokens/trainable": 151708} +{"epoch": 1.30859375, "grad_norm": 0.09196531027555466, "learning_rate": 3.654669712344384e-05, "loss": 0.0009737662621773779, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 1.00097, "step": 335, "tokens/total": 11658832, "tokens/train_per_sec_per_gpu": 31.45, "tokens/trainable": 152164} +{"epoch": 1.3125, "grad_norm": 0.03397702798247337, "learning_rate": 3.628232236787763e-05, "loss": 0.0005413000471889973, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.96, "memory/max_allocated (GiB)": 37.96, "ppl": 1.00054, "step": 336, "tokens/total": 11694464, "tokens/train_per_sec_per_gpu": 22.54, "tokens/trainable": 152603} +{"epoch": 1.31640625, "grad_norm": 0.04066256806254387, "learning_rate": 3.6018726531280144e-05, "loss": 0.00046271528117358685, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00046, "step": 337, "tokens/total": 11732176, "tokens/train_per_sec_per_gpu": 28.15, "tokens/trainable": 153094} +{"epoch": 1.3203125, "grad_norm": 0.06415663659572601, "learning_rate": 3.575592058295017e-05, "loss": 0.001029198057949543, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.00103, "step": 338, "tokens/total": 11766032, "tokens/train_per_sec_per_gpu": 28.33, "tokens/trainable": 153573} +{"epoch": 1.32421875, "grad_norm": 0.2313908338546753, "learning_rate": 3.549391545931585e-05, "loss": 0.00025592040037736297, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00026, "step": 339, "tokens/total": 11800640, "tokens/train_per_sec_per_gpu": 27.57, "tokens/trainable": 154033} +{"epoch": 1.328125, "grad_norm": 0.08988120406866074, "learning_rate": 3.5232722063479914e-05, "loss": 0.0006757613155059516, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.00068, "step": 340, "tokens/total": 11834352, "tokens/train_per_sec_per_gpu": 24.05, "tokens/trainable": 154482} +{"epoch": 1.33203125, "grad_norm": 0.05467112362384796, "learning_rate": 3.49723512647657e-05, "loss": 0.00043029486550949514, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.00043, "step": 341, "tokens/total": 11873056, "tokens/train_per_sec_per_gpu": 24.67, "tokens/trainable": 154982} +{"epoch": 1.3359375, "grad_norm": 0.042000334709882736, "learning_rate": 3.471281389826491e-05, "loss": 0.00022522300423588604, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.00023, "step": 342, "tokens/total": 11904192, "tokens/train_per_sec_per_gpu": 25.86, "tokens/trainable": 155387} +{"epoch": 1.33984375, "grad_norm": 0.005496317520737648, "learning_rate": 3.4454120764386764e-05, "loss": 9.726975986268371e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.66, "memory/max_allocated (GiB)": 38.66, "ppl": 1.0001, "step": 343, "tokens/total": 11942384, "tokens/train_per_sec_per_gpu": 24.2, "tokens/trainable": 155842} +{"epoch": 1.34375, "grad_norm": 0.3453257381916046, "learning_rate": 3.4196282628408526e-05, "loss": 0.004893580451607704, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.00491, "step": 344, "tokens/total": 11976624, "tokens/train_per_sec_per_gpu": 25.22, "tokens/trainable": 156287} +{"epoch": 1.34765625, "grad_norm": 0.09715849906206131, "learning_rate": 3.3939310220027456e-05, "loss": 0.0006179324700497091, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00062, "step": 345, "tokens/total": 12010768, "tokens/train_per_sec_per_gpu": 28.76, "tokens/trainable": 156782} +{"epoch": 1.3515625, "grad_norm": 0.0535743422806263, "learning_rate": 3.3683214232914404e-05, "loss": 0.0007631386397406459, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.00076, "step": 346, "tokens/total": 12046224, "tokens/train_per_sec_per_gpu": 26.15, "tokens/trainable": 157255} +{"epoch": 1.35546875, "grad_norm": 0.011245500296354294, "learning_rate": 3.342800532426873e-05, "loss": 0.000160155730554834, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.00016, "step": 347, "tokens/total": 12082160, "tokens/train_per_sec_per_gpu": 25.45, "tokens/trainable": 157721} +{"epoch": 1.359375, "grad_norm": 0.08089470118284225, "learning_rate": 3.317369411437484e-05, "loss": 0.0009149868856184185, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.00092, "step": 348, "tokens/total": 12118128, "tokens/train_per_sec_per_gpu": 26.14, "tokens/trainable": 158189} +{"epoch": 1.36328125, "grad_norm": 0.3714316487312317, "learning_rate": 3.292029118616024e-05, "loss": 0.0019184882985427976, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.00192, "step": 349, "tokens/total": 12151264, "tokens/train_per_sec_per_gpu": 25.59, "tokens/trainable": 158612} +{"epoch": 1.3671875, "grad_norm": 0.21182094514369965, "learning_rate": 3.266780708475511e-05, "loss": 0.0026206644251942635, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00262, "step": 350, "tokens/total": 12186800, "tokens/train_per_sec_per_gpu": 25.04, "tokens/trainable": 159090} +{"epoch": 1.37109375, "grad_norm": 0.02487930655479431, "learning_rate": 3.241625231705354e-05, "loss": 0.00020223903993610293, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.0002, "step": 351, "tokens/total": 12222944, "tokens/train_per_sec_per_gpu": 25.03, "tokens/trainable": 159549} +{"epoch": 1.375, "grad_norm": 0.009250174276530743, "learning_rate": 3.216563735127618e-05, "loss": 9.479152504354715e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.15, "memory/max_allocated (GiB)": 39.15, "ppl": 1.00009, "step": 352, "tokens/total": 12258208, "tokens/train_per_sec_per_gpu": 25.53, "tokens/trainable": 160026} +{"epoch": 1.37890625, "grad_norm": 0.0688176080584526, "learning_rate": 3.191597261653475e-05, "loss": 0.00028054206632077694, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00028, "step": 353, "tokens/total": 12295024, "tokens/train_per_sec_per_gpu": 23.72, "tokens/trainable": 160487} +{"epoch": 1.3828125, "grad_norm": 0.10484874248504639, "learning_rate": 3.166726850239794e-05, "loss": 0.0012113112024962902, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00121, "step": 354, "tokens/total": 12328064, "tokens/train_per_sec_per_gpu": 27.44, "tokens/trainable": 160935} +{"epoch": 1.38671875, "grad_norm": 0.005378663074225187, "learning_rate": 3.141953535845912e-05, "loss": 9.526368376100436e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.34, "memory/max_allocated (GiB)": 36.34, "ppl": 1.0001, "step": 355, "tokens/total": 12359504, "tokens/train_per_sec_per_gpu": 30.38, "tokens/trainable": 161385} +{"epoch": 1.390625, "grad_norm": 0.03373286500573158, "learning_rate": 3.11727834939056e-05, "loss": 0.0003015822730958462, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.0003, "step": 356, "tokens/total": 12391824, "tokens/train_per_sec_per_gpu": 25.47, "tokens/trainable": 161849} +{"epoch": 1.39453125, "grad_norm": 0.12636283040046692, "learning_rate": 3.092702317708967e-05, "loss": 0.0017855733167380095, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.00179, "step": 357, "tokens/total": 12425408, "tokens/train_per_sec_per_gpu": 28.92, "tokens/trainable": 162306} +{"epoch": 1.3984375, "grad_norm": 0.12089350074529648, "learning_rate": 3.0682264635101276e-05, "loss": 0.0004166644939687103, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.00042, "step": 358, "tokens/total": 12458160, "tokens/train_per_sec_per_gpu": 28.25, "tokens/trainable": 162759} +{"epoch": 1.40234375, "grad_norm": 1.2642138004302979, "learning_rate": 3.0438518053342407e-05, "loss": 0.00047902853111736476, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00048, "step": 359, "tokens/total": 12497104, "tokens/train_per_sec_per_gpu": 22.14, "tokens/trainable": 163230} +{"epoch": 1.40625, "grad_norm": 0.12005579471588135, "learning_rate": 3.0195793575103266e-05, "loss": 0.0005570354987867177, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00056, "step": 360, "tokens/total": 12536064, "tokens/train_per_sec_per_gpu": 21.26, "tokens/trainable": 163694} +{"epoch": 1.41015625, "grad_norm": 0.4310549795627594, "learning_rate": 2.9954101301140146e-05, "loss": 0.004543776623904705, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.57, "memory/max_allocated (GiB)": 37.57, "ppl": 1.00455, "step": 361, "tokens/total": 12571488, "tokens/train_per_sec_per_gpu": 24.58, "tokens/trainable": 164144} +{"epoch": 1.4140625, "grad_norm": 0.2241031378507614, "learning_rate": 2.9713451289255123e-05, "loss": 0.002146549755707383, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.63, "memory/max_allocated (GiB)": 36.63, "ppl": 1.00215, "step": 362, "tokens/total": 12600864, "tokens/train_per_sec_per_gpu": 25.53, "tokens/trainable": 164585} +{"epoch": 1.41796875, "grad_norm": 0.07957537472248077, "learning_rate": 2.9473853553877484e-05, "loss": 0.0004616625956259668, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.00046, "step": 363, "tokens/total": 12635024, "tokens/train_per_sec_per_gpu": 27.25, "tokens/trainable": 165059} +{"epoch": 1.421875, "grad_norm": 0.21537819504737854, "learning_rate": 2.9235318065647e-05, "loss": 0.0021064088214188814, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.00211, "step": 364, "tokens/total": 12674736, "tokens/train_per_sec_per_gpu": 24.91, "tokens/trainable": 165519} +{"epoch": 1.42578125, "grad_norm": 0.0540163516998291, "learning_rate": 2.8997854750998964e-05, "loss": 0.0002838809450622648, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.46, "memory/max_allocated (GiB)": 39.46, "ppl": 1.00028, "step": 365, "tokens/total": 12714256, "tokens/train_per_sec_per_gpu": 23.16, "tokens/trainable": 165984} +{"epoch": 1.4296875, "grad_norm": 0.001867863116785884, "learning_rate": 2.8761473491751258e-05, "loss": 2.9001597795286216e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.00003, "step": 366, "tokens/total": 12747504, "tokens/train_per_sec_per_gpu": 26.94, "tokens/trainable": 166419} +{"epoch": 1.43359375, "grad_norm": 0.5463334918022156, "learning_rate": 2.8526184124692883e-05, "loss": 0.009400909766554832, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00945, "step": 367, "tokens/total": 12783872, "tokens/train_per_sec_per_gpu": 20.41, "tokens/trainable": 166857} +{"epoch": 1.4375, "grad_norm": 0.1852254718542099, "learning_rate": 2.829199644117484e-05, "loss": 0.002596612786874175, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.0026, "step": 368, "tokens/total": 12817856, "tokens/train_per_sec_per_gpu": 24.35, "tokens/trainable": 167267} +{"epoch": 1.44140625, "grad_norm": 0.7085232138633728, "learning_rate": 2.8058920186702553e-05, "loss": 0.00903827603906393, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00908, "step": 369, "tokens/total": 12855472, "tokens/train_per_sec_per_gpu": 25.0, "tokens/trainable": 167740} +{"epoch": 1.4453125, "grad_norm": 0.8681559562683105, "learning_rate": 2.782696506053033e-05, "loss": 0.00692350510507822, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.00695, "step": 370, "tokens/total": 12893344, "tokens/train_per_sec_per_gpu": 22.36, "tokens/trainable": 168172} +{"epoch": 1.44921875, "grad_norm": 0.3795325458049774, "learning_rate": 2.7596140715257824e-05, "loss": 0.0041154539212584496, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00412, "step": 371, "tokens/total": 12928512, "tokens/train_per_sec_per_gpu": 25.72, "tokens/trainable": 168637} +{"epoch": 1.453125, "grad_norm": 0.011500510387122631, "learning_rate": 2.7366456756428184e-05, "loss": 9.635539026930928e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 1.0001, "step": 372, "tokens/total": 12963120, "tokens/train_per_sec_per_gpu": 27.18, "tokens/trainable": 169094} +{"epoch": 1.45703125, "grad_norm": 0.021570798009634018, "learning_rate": 2.7137922742128486e-05, "loss": 0.00029517963412217796, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.0003, "step": 373, "tokens/total": 12997648, "tokens/train_per_sec_per_gpu": 26.99, "tokens/trainable": 169593} +{"epoch": 1.4609375, "grad_norm": 0.011727412231266499, "learning_rate": 2.691054818259188e-05, "loss": 8.469731983495876e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00008, "step": 374, "tokens/total": 13032496, "tokens/train_per_sec_per_gpu": 26.33, "tokens/trainable": 170066} +{"epoch": 1.46484375, "grad_norm": 1.0964086055755615, "learning_rate": 2.6684342539801933e-05, "loss": 0.0024464954622089863, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.56, "memory/max_allocated (GiB)": 36.56, "ppl": 1.00245, "step": 375, "tokens/total": 13064016, "tokens/train_per_sec_per_gpu": 28.87, "tokens/trainable": 170518} +{"epoch": 1.46875, "grad_norm": 0.039562299847602844, "learning_rate": 2.645931522709877e-05, "loss": 0.0004973928444087505, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.0005, "step": 376, "tokens/total": 13099408, "tokens/train_per_sec_per_gpu": 29.06, "tokens/trainable": 170998} +{"epoch": 1.47265625, "grad_norm": 0.04844140633940697, "learning_rate": 2.6235475608787365e-05, "loss": 0.0007791711250320077, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.00078, "step": 377, "tokens/total": 13135088, "tokens/train_per_sec_per_gpu": 25.54, "tokens/trainable": 171459} +{"epoch": 1.4765625, "grad_norm": 0.1333562433719635, "learning_rate": 2.6012832999747916e-05, "loss": 0.0013580535305663943, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.00136, "step": 378, "tokens/total": 13166208, "tokens/train_per_sec_per_gpu": 29.31, "tokens/trainable": 171905} +{"epoch": 1.48046875, "grad_norm": 0.5828719139099121, "learning_rate": 2.579139666504821e-05, "loss": 0.0234844833612442, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.02376, "step": 379, "tokens/total": 13206192, "tokens/train_per_sec_per_gpu": 22.08, "tokens/trainable": 172351} +{"epoch": 1.484375, "grad_norm": 0.030452381819486618, "learning_rate": 2.557117581955798e-05, "loss": 0.00020915968343615532, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00021, "step": 380, "tokens/total": 13240032, "tokens/train_per_sec_per_gpu": 26.99, "tokens/trainable": 172817} +{"epoch": 1.48828125, "grad_norm": 0.15066435933113098, "learning_rate": 2.5352179627565532e-05, "loss": 0.0024384954012930393, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.00244, "step": 381, "tokens/total": 13274144, "tokens/train_per_sec_per_gpu": 25.72, "tokens/trainable": 173258} +{"epoch": 1.4921875, "grad_norm": 0.036316677927970886, "learning_rate": 2.5134417202396277e-05, "loss": 0.0003756014339160174, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.6, "memory/max_allocated (GiB)": 38.6, "ppl": 1.00038, "step": 382, "tokens/total": 13308128, "tokens/train_per_sec_per_gpu": 24.01, "tokens/trainable": 173665} +{"epoch": 1.49609375, "grad_norm": 0.0507366769015789, "learning_rate": 2.491789760603361e-05, "loss": 0.0008591790683567524, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00086, "step": 383, "tokens/total": 13344544, "tokens/train_per_sec_per_gpu": 23.23, "tokens/trainable": 174095} +{"epoch": 1.5, "grad_norm": 0.0697312280535698, "learning_rate": 2.4702629848741764e-05, "loss": 0.0005336232716217637, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.00053, "step": 384, "tokens/total": 13378592, "tokens/train_per_sec_per_gpu": 26.55, "tokens/trainable": 174550} +{"epoch": 1.50390625, "grad_norm": 0.07200636714696884, "learning_rate": 2.4488622888690785e-05, "loss": 0.0010450442787259817, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.04, "memory/max_allocated (GiB)": 37.04, "ppl": 1.00105, "step": 385, "tokens/total": 13412048, "tokens/train_per_sec_per_gpu": 27.0, "tokens/trainable": 174990} +{"epoch": 1.5078125, "grad_norm": 0.008022339083254337, "learning_rate": 2.427588563158384e-05, "loss": 0.00014128295879345387, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00014, "step": 386, "tokens/total": 13446384, "tokens/train_per_sec_per_gpu": 26.19, "tokens/trainable": 175476} +{"epoch": 1.51171875, "grad_norm": 0.6431483626365662, "learning_rate": 2.406442693028651e-05, "loss": 0.006740795448422432, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.63, "memory/max_allocated (GiB)": 38.63, "ppl": 1.00676, "step": 387, "tokens/total": 13478320, "tokens/train_per_sec_per_gpu": 25.49, "tokens/trainable": 175897} +{"epoch": 1.515625, "grad_norm": 0.07601752877235413, "learning_rate": 2.3854255584458547e-05, "loss": 0.0008580058929510415, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.00086, "step": 388, "tokens/total": 13516912, "tokens/train_per_sec_per_gpu": 21.62, "tokens/trainable": 176324} +{"epoch": 1.51953125, "grad_norm": 0.10759659856557846, "learning_rate": 2.3645380340187508e-05, "loss": 0.0012811472406610847, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00128, "step": 389, "tokens/total": 13552592, "tokens/train_per_sec_per_gpu": 26.12, "tokens/trainable": 176805} +{"epoch": 1.5234375, "grad_norm": 0.041328079998493195, "learning_rate": 2.3437809889624914e-05, "loss": 0.0005108289187774062, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.54, "memory/max_allocated (GiB)": 36.54, "ppl": 1.00051, "step": 390, "tokens/total": 13583856, "tokens/train_per_sec_per_gpu": 29.23, "tokens/trainable": 177256} +{"epoch": 1.52734375, "grad_norm": 0.00742257060483098, "learning_rate": 2.3231552870624487e-05, "loss": 0.0001279473799513653, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.47, "memory/max_allocated (GiB)": 38.47, "ppl": 1.00013, "step": 391, "tokens/total": 13614864, "tokens/train_per_sec_per_gpu": 29.15, "tokens/trainable": 177682} +{"epoch": 1.53125, "grad_norm": 0.20578733086585999, "learning_rate": 2.3026617866382657e-05, "loss": 0.002508244477212429, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.53, "memory/max_allocated (GiB)": 36.53, "ppl": 1.00251, "step": 392, "tokens/total": 13647616, "tokens/train_per_sec_per_gpu": 29.88, "tokens/trainable": 178179} +{"epoch": 1.53515625, "grad_norm": 0.14710013568401337, "learning_rate": 2.2823013405081507e-05, "loss": 0.0008502759737893939, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.00085, "step": 393, "tokens/total": 13685696, "tokens/train_per_sec_per_gpu": 26.88, "tokens/trainable": 178676} +{"epoch": 1.5390625, "grad_norm": 0.11233309656381607, "learning_rate": 2.2620747959533722e-05, "loss": 0.0008494096691720188, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.76, "memory/max_allocated (GiB)": 36.76, "ppl": 1.00085, "step": 394, "tokens/total": 13718768, "tokens/train_per_sec_per_gpu": 30.72, "tokens/trainable": 179173} +{"epoch": 1.54296875, "grad_norm": 0.019205722957849503, "learning_rate": 2.2419829946830123e-05, "loss": 0.00036604603519663215, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.00037, "step": 395, "tokens/total": 13754960, "tokens/train_per_sec_per_gpu": 24.23, "tokens/trainable": 179624} +{"epoch": 1.546875, "grad_norm": 0.14569444954395294, "learning_rate": 2.2220267727989325e-05, "loss": 0.0014237299328669906, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00142, "step": 396, "tokens/total": 13789104, "tokens/train_per_sec_per_gpu": 25.79, "tokens/trainable": 180078} +{"epoch": 1.55078125, "grad_norm": 0.16714350879192352, "learning_rate": 2.202206960760984e-05, "loss": 0.0021738815121352673, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.00218, "step": 397, "tokens/total": 13827248, "tokens/train_per_sec_per_gpu": 24.74, "tokens/trainable": 180585} +{"epoch": 1.5546875, "grad_norm": 0.14391010999679565, "learning_rate": 2.182524383352446e-05, "loss": 0.0009926884667947888, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00099, "step": 398, "tokens/total": 13863408, "tokens/train_per_sec_per_gpu": 24.16, "tokens/trainable": 181039} +{"epoch": 1.55859375, "grad_norm": 3.184635639190674, "learning_rate": 2.1629798596457056e-05, "loss": 0.013314366340637207, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.52, "memory/max_allocated (GiB)": 36.52, "ppl": 1.0134, "step": 399, "tokens/total": 13895728, "tokens/train_per_sec_per_gpu": 26.6, "tokens/trainable": 181489} +{"epoch": 1.5625, "grad_norm": 0.01847706362605095, "learning_rate": 2.1435742029681725e-05, "loss": 0.00020128217875026166, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.08, "memory/max_allocated (GiB)": 36.08, "ppl": 1.0002, "step": 400, "tokens/total": 13926048, "tokens/train_per_sec_per_gpu": 28.11, "tokens/trainable": 181906} +{"epoch": 1.56640625, "grad_norm": 0.0023072431795299053, "learning_rate": 2.124308220868431e-05, "loss": 4.197553062113002e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.67, "memory/max_allocated (GiB)": 38.67, "ppl": 1.00004, "step": 401, "tokens/total": 13961440, "tokens/train_per_sec_per_gpu": 24.3, "tokens/trainable": 182361} +{"epoch": 1.5703125, "grad_norm": 0.1210937350988388, "learning_rate": 2.105182715082638e-05, "loss": 0.0025683268904685974, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00257, "step": 402, "tokens/total": 13995040, "tokens/train_per_sec_per_gpu": 27.62, "tokens/trainable": 182838} +{"epoch": 1.57421875, "grad_norm": 0.03222186118364334, "learning_rate": 2.0861984815011552e-05, "loss": 0.0003052979882340878, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.16, "memory/max_allocated (GiB)": 38.16, "ppl": 1.00031, "step": 403, "tokens/total": 14030560, "tokens/train_per_sec_per_gpu": 22.57, "tokens/trainable": 183269} +{"epoch": 1.578125, "grad_norm": 0.05465065687894821, "learning_rate": 2.0673563101354323e-05, "loss": 0.0005894795176573098, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00059, "step": 404, "tokens/total": 14065840, "tokens/train_per_sec_per_gpu": 22.67, "tokens/trainable": 183699} +{"epoch": 1.58203125, "grad_norm": 0.22841252386569977, "learning_rate": 2.0486569850851317e-05, "loss": 0.002843332476913929, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.00285, "step": 405, "tokens/total": 14100848, "tokens/train_per_sec_per_gpu": 25.9, "tokens/trainable": 184162} +{"epoch": 1.5859375, "grad_norm": 0.045836225152015686, "learning_rate": 2.0301012845054956e-05, "loss": 0.0001991704193642363, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.0002, "step": 406, "tokens/total": 14139600, "tokens/train_per_sec_per_gpu": 20.54, "tokens/trainable": 184576} +{"epoch": 1.58984375, "grad_norm": 0.028748629614710808, "learning_rate": 2.011689980574966e-05, "loss": 0.0002606338821351528, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.06, "memory/max_allocated (GiB)": 38.06, "ppl": 1.00026, "step": 407, "tokens/total": 14175904, "tokens/train_per_sec_per_gpu": 22.84, "tokens/trainable": 184995} +{"epoch": 1.59375, "grad_norm": 0.25236013531684875, "learning_rate": 1.993423839463052e-05, "loss": 0.006212318316102028, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.00623, "step": 408, "tokens/total": 14209168, "tokens/train_per_sec_per_gpu": 31.89, "tokens/trainable": 185452} +{"epoch": 1.59765625, "grad_norm": 0.05855831131339073, "learning_rate": 1.975303621298445e-05, "loss": 0.000475426233606413, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.00048, "step": 409, "tokens/total": 14246208, "tokens/train_per_sec_per_gpu": 25.4, "tokens/trainable": 185904} +{"epoch": 1.6015625, "grad_norm": 0.03339484706521034, "learning_rate": 1.957330080137385e-05, "loss": 0.0003073758853133768, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.31, "memory/max_allocated (GiB)": 36.31, "ppl": 1.00031, "step": 410, "tokens/total": 14277072, "tokens/train_per_sec_per_gpu": 29.03, "tokens/trainable": 186355} +{"epoch": 1.60546875, "grad_norm": 0.40946412086486816, "learning_rate": 1.9395039639322864e-05, "loss": 0.004215352702885866, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.00422, "step": 411, "tokens/total": 14313120, "tokens/train_per_sec_per_gpu": 26.42, "tokens/trainable": 186831} +{"epoch": 1.609375, "grad_norm": 0.0011183505412191153, "learning_rate": 1.9218260145006073e-05, "loss": 1.9001574401045218e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.52, "memory/max_allocated (GiB)": 37.52, "ppl": 1.00002, "step": 412, "tokens/total": 14348368, "tokens/train_per_sec_per_gpu": 27.05, "tokens/trainable": 187317} +{"epoch": 1.61328125, "grad_norm": 0.005545249208807945, "learning_rate": 1.904296967493982e-05, "loss": 0.00010679063416318968, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.00011, "step": 413, "tokens/total": 14380528, "tokens/train_per_sec_per_gpu": 29.68, "tokens/trainable": 187773} +{"epoch": 1.6171875, "grad_norm": 0.36505040526390076, "learning_rate": 1.8869175523676064e-05, "loss": 0.006718277465552092, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.00674, "step": 414, "tokens/total": 14417216, "tokens/train_per_sec_per_gpu": 26.58, "tokens/trainable": 188268} +{"epoch": 1.62109375, "grad_norm": 0.009049337357282639, "learning_rate": 1.869688492349885e-05, "loss": 0.00010680315608624369, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00011, "step": 415, "tokens/total": 14450672, "tokens/train_per_sec_per_gpu": 27.31, "tokens/trainable": 188720} +{"epoch": 1.625, "grad_norm": 0.042370785027742386, "learning_rate": 1.85261050441233e-05, "loss": 0.00036333888419903815, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.00036, "step": 416, "tokens/total": 14486464, "tokens/train_per_sec_per_gpu": 24.35, "tokens/trainable": 189177} +{"epoch": 1.62890625, "grad_norm": 0.4869096875190735, "learning_rate": 1.8356842992397304e-05, "loss": 0.012596406042575836, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.13, "memory/max_allocated (GiB)": 37.13, "ppl": 1.01268, "step": 417, "tokens/total": 14519984, "tokens/train_per_sec_per_gpu": 27.02, "tokens/trainable": 189617} +{"epoch": 1.6328125, "grad_norm": 0.0892733559012413, "learning_rate": 1.8189105812005714e-05, "loss": 0.0006157285533845425, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.00062, "step": 418, "tokens/total": 14556800, "tokens/train_per_sec_per_gpu": 22.27, "tokens/trainable": 190028} +{"epoch": 1.63671875, "grad_norm": 0.006320591550320387, "learning_rate": 1.802290048317732e-05, "loss": 0.00012320438690949231, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00012, "step": 419, "tokens/total": 14590480, "tokens/train_per_sec_per_gpu": 23.38, "tokens/trainable": 190445} +{"epoch": 1.640625, "grad_norm": 0.009132839739322662, "learning_rate": 1.785823392239424e-05, "loss": 8.83523462107405e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.00009, "step": 420, "tokens/total": 14624064, "tokens/train_per_sec_per_gpu": 26.17, "tokens/trainable": 190886} +{"epoch": 1.64453125, "grad_norm": 0.055305637419223785, "learning_rate": 1.7695112982104225e-05, "loss": 0.0009231048170477152, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00092, "step": 421, "tokens/total": 14659072, "tokens/train_per_sec_per_gpu": 24.87, "tokens/trainable": 191325} +{"epoch": 1.6484375, "grad_norm": 0.011019755154848099, "learning_rate": 1.7533544450435433e-05, "loss": 0.00010593365732347593, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.79, "memory/max_allocated (GiB)": 35.79, "ppl": 1.00011, "step": 422, "tokens/total": 14688624, "tokens/train_per_sec_per_gpu": 30.1, "tokens/trainable": 191758} +{"epoch": 1.65234375, "grad_norm": 0.007611104287207127, "learning_rate": 1.7373535050913946e-05, "loss": 9.056244743987918e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00009, "step": 423, "tokens/total": 14725024, "tokens/train_per_sec_per_gpu": 21.92, "tokens/trainable": 192195} +{"epoch": 1.65625, "grad_norm": 0.02671491727232933, "learning_rate": 1.721509144218405e-05, "loss": 0.0003718341758940369, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.85, "memory/max_allocated (GiB)": 35.85, "ppl": 1.00037, "step": 424, "tokens/total": 14755344, "tokens/train_per_sec_per_gpu": 29.87, "tokens/trainable": 192625} +{"epoch": 1.66015625, "grad_norm": 0.018284307792782784, "learning_rate": 1.705822021773101e-05, "loss": 0.0002250685211038217, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.00023, "step": 425, "tokens/total": 14789776, "tokens/train_per_sec_per_gpu": 28.05, "tokens/trainable": 193111} +{"epoch": 1.6640625, "grad_norm": 0.44931548833847046, "learning_rate": 1.69029279056068e-05, "loss": 0.0071952310390770435, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00722, "step": 426, "tokens/total": 14826896, "tokens/train_per_sec_per_gpu": 25.14, "tokens/trainable": 193551} +{"epoch": 1.66796875, "grad_norm": 0.3717283308506012, "learning_rate": 1.6749220968158415e-05, "loss": 0.002148708561435342, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00215, "step": 427, "tokens/total": 14862192, "tokens/train_per_sec_per_gpu": 29.15, "tokens/trainable": 194057} +{"epoch": 1.671875, "grad_norm": 0.07249587029218674, "learning_rate": 1.659710580175893e-05, "loss": 0.0007708582561463118, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 35.91, "memory/max_allocated (GiB)": 35.91, "ppl": 1.00077, "step": 428, "tokens/total": 14893264, "tokens/train_per_sec_per_gpu": 32.15, "tokens/trainable": 194556} +{"epoch": 1.67578125, "grad_norm": 0.015728846192359924, "learning_rate": 1.644658873654133e-05, "loss": 0.000139736570417881, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 1.00014, "step": 429, "tokens/total": 14925712, "tokens/train_per_sec_per_gpu": 30.47, "tokens/trainable": 194993} +{"epoch": 1.6796875, "grad_norm": 0.014698989689350128, "learning_rate": 1.629767603613508e-05, "loss": 0.00015438513946719468, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.24, "memory/max_allocated (GiB)": 36.24, "ppl": 1.00015, "step": 430, "tokens/total": 14956944, "tokens/train_per_sec_per_gpu": 25.78, "tokens/trainable": 195393} +{"epoch": 1.68359375, "grad_norm": 0.19958904385566711, "learning_rate": 1.615037389740547e-05, "loss": 0.004098682664334774, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.00411, "step": 431, "tokens/total": 14991488, "tokens/train_per_sec_per_gpu": 22.96, "tokens/trainable": 195837} +{"epoch": 1.6875, "grad_norm": 0.12062514573335648, "learning_rate": 1.600468845019576e-05, "loss": 0.0005472414777614176, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00055, "step": 432, "tokens/total": 15029856, "tokens/train_per_sec_per_gpu": 23.87, "tokens/trainable": 196317} +{"epoch": 1.69140625, "grad_norm": 0.0050490121357142925, "learning_rate": 1.5860625757072092e-05, "loss": 6.157424650155008e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.00006, "step": 433, "tokens/total": 15064864, "tokens/train_per_sec_per_gpu": 23.98, "tokens/trainable": 196765} +{"epoch": 1.6953125, "grad_norm": 0.0066376025788486, "learning_rate": 1.571819181307116e-05, "loss": 6.861748988740146e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.2, "memory/max_allocated (GiB)": 38.2, "ppl": 1.00007, "step": 434, "tokens/total": 15101440, "tokens/train_per_sec_per_gpu": 25.71, "tokens/trainable": 197237} +{"epoch": 1.69921875, "grad_norm": 0.0447714664041996, "learning_rate": 1.557739254545075e-05, "loss": 0.00020413102174643427, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.28, "memory/max_allocated (GiB)": 36.28, "ppl": 1.0002, "step": 435, "tokens/total": 15132464, "tokens/train_per_sec_per_gpu": 32.76, "tokens/trainable": 197709} +{"epoch": 1.703125, "grad_norm": 0.01506239827722311, "learning_rate": 1.543823381344311e-05, "loss": 0.00016552701708860695, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.00017, "step": 436, "tokens/total": 15164960, "tokens/train_per_sec_per_gpu": 31.14, "tokens/trainable": 198184} +{"epoch": 1.70703125, "grad_norm": 0.18751978874206543, "learning_rate": 1.5300721408011114e-05, "loss": 0.0024171944241970778, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.08, "memory/max_allocated (GiB)": 39.08, "ppl": 1.00242, "step": 437, "tokens/total": 15200512, "tokens/train_per_sec_per_gpu": 27.79, "tokens/trainable": 198641} +{"epoch": 1.7109375, "grad_norm": 0.003079737536609173, "learning_rate": 1.5164861051607254e-05, "loss": 6.802278221584857e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.9, "memory/max_allocated (GiB)": 36.9, "ppl": 1.00007, "step": 438, "tokens/total": 15233536, "tokens/train_per_sec_per_gpu": 23.61, "tokens/trainable": 199049} +{"epoch": 1.71484375, "grad_norm": 0.003829564666375518, "learning_rate": 1.5030658397935521e-05, "loss": 4.3093292333651334e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.34, "memory/max_allocated (GiB)": 38.34, "ppl": 1.00004, "step": 439, "tokens/total": 15268896, "tokens/train_per_sec_per_gpu": 25.62, "tokens/trainable": 199518} +{"epoch": 1.71875, "grad_norm": 0.007842005230486393, "learning_rate": 1.4898119031716104e-05, "loss": 0.00012832845095545053, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00013, "step": 440, "tokens/total": 15302288, "tokens/train_per_sec_per_gpu": 27.59, "tokens/trainable": 199976} +{"epoch": 1.72265625, "grad_norm": 0.03095095045864582, "learning_rate": 1.476724846845306e-05, "loss": 0.00033764212275855243, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.00034, "step": 441, "tokens/total": 15340416, "tokens/train_per_sec_per_gpu": 27.38, "tokens/trainable": 200453} +{"epoch": 1.7265625, "grad_norm": 0.007876990363001823, "learning_rate": 1.463805215420471e-05, "loss": 7.592058682348579e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.00008, "step": 442, "tokens/total": 15375680, "tokens/train_per_sec_per_gpu": 26.46, "tokens/trainable": 200923} +{"epoch": 1.73046875, "grad_norm": 0.040339160710573196, "learning_rate": 1.451053546535705e-05, "loss": 0.00020478149235714227, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.0002, "step": 443, "tokens/total": 15412560, "tokens/train_per_sec_per_gpu": 25.1, "tokens/trainable": 201379} +{"epoch": 1.734375, "grad_norm": 0.0048829601146280766, "learning_rate": 1.438470370840001e-05, "loss": 8.083799184532836e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00008, "step": 444, "tokens/total": 15448448, "tokens/train_per_sec_per_gpu": 26.95, "tokens/trainable": 201851} +{"epoch": 1.73828125, "grad_norm": 0.01981750875711441, "learning_rate": 1.4260562119706606e-05, "loss": 0.00018735197954811156, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.00019, "step": 445, "tokens/total": 15481024, "tokens/train_per_sec_per_gpu": 30.26, "tokens/trainable": 202325} +{"epoch": 1.7421875, "grad_norm": 0.021114468574523926, "learning_rate": 1.413811586531508e-05, "loss": 0.00023567129392176867, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.00024, "step": 446, "tokens/total": 15514064, "tokens/train_per_sec_per_gpu": 28.76, "tokens/trainable": 202766} +{"epoch": 1.74609375, "grad_norm": 0.30760258436203003, "learning_rate": 1.4017370040713884e-05, "loss": 0.0037974196020513773, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.0038, "step": 447, "tokens/total": 15551200, "tokens/train_per_sec_per_gpu": 23.62, "tokens/trainable": 203215} +{"epoch": 1.75, "grad_norm": 0.19040031731128693, "learning_rate": 1.3898329670629645e-05, "loss": 0.0011992794461548328, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.0012, "step": 448, "tokens/total": 15591040, "tokens/train_per_sec_per_gpu": 21.03, "tokens/trainable": 203660} +{"epoch": 1.75390625, "grad_norm": 0.06951410323381424, "learning_rate": 1.3780999708818058e-05, "loss": 0.0006639776984229684, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.00066, "step": 449, "tokens/total": 15622864, "tokens/train_per_sec_per_gpu": 28.58, "tokens/trainable": 204110} +{"epoch": 1.7578125, "grad_norm": 0.359775185585022, "learning_rate": 1.3665385037857758e-05, "loss": 0.004150236491113901, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.96, "memory/max_allocated (GiB)": 36.96, "ppl": 1.00416, "step": 450, "tokens/total": 15656144, "tokens/train_per_sec_per_gpu": 24.84, "tokens/trainable": 204562} +{"epoch": 1.76171875, "grad_norm": 0.032830625772476196, "learning_rate": 1.3551490468947126e-05, "loss": 0.0002510029589757323, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.33, "memory/max_allocated (GiB)": 36.33, "ppl": 1.00025, "step": 451, "tokens/total": 15688000, "tokens/train_per_sec_per_gpu": 27.68, "tokens/trainable": 205008} +{"epoch": 1.765625, "grad_norm": 0.21395793557167053, "learning_rate": 1.3439320741704075e-05, "loss": 0.00216006045229733, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00216, "step": 452, "tokens/total": 15723104, "tokens/train_per_sec_per_gpu": 25.22, "tokens/trainable": 205432} +{"epoch": 1.76953125, "grad_norm": 0.10554767400026321, "learning_rate": 1.3328880523968808e-05, "loss": 0.0005892113549634814, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.07, "memory/max_allocated (GiB)": 37.07, "ppl": 1.00059, "step": 453, "tokens/total": 15756752, "tokens/train_per_sec_per_gpu": 26.35, "tokens/trainable": 205895} +{"epoch": 1.7734375, "grad_norm": 0.0023844747338443995, "learning_rate": 1.3220174411609587e-05, "loss": 3.291782195447013e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.00003, "step": 454, "tokens/total": 15790576, "tokens/train_per_sec_per_gpu": 26.81, "tokens/trainable": 206334} +{"epoch": 1.77734375, "grad_norm": 0.04232781380414963, "learning_rate": 1.3113206928331471e-05, "loss": 0.00013443417265079916, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 35.55, "memory/max_allocated (GiB)": 35.55, "ppl": 1.00013, "step": 455, "tokens/total": 15819344, "tokens/train_per_sec_per_gpu": 30.78, "tokens/trainable": 206796} +{"epoch": 1.78125, "grad_norm": 0.15394103527069092, "learning_rate": 1.300798252548806e-05, "loss": 0.0007882431382313371, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.00079, "step": 456, "tokens/total": 15853776, "tokens/train_per_sec_per_gpu": 27.49, "tokens/trainable": 207248} +{"epoch": 1.78515625, "grad_norm": 0.8043767213821411, "learning_rate": 1.2904505581896265e-05, "loss": 0.02176298014819622, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.022, "step": 457, "tokens/total": 15892768, "tokens/train_per_sec_per_gpu": 23.42, "tokens/trainable": 207721} +{"epoch": 1.7890625, "grad_norm": 0.26657694578170776, "learning_rate": 1.2802780403654082e-05, "loss": 0.0017039180966094136, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00171, "step": 458, "tokens/total": 15926656, "tokens/train_per_sec_per_gpu": 26.75, "tokens/trainable": 208168} +{"epoch": 1.79296875, "grad_norm": 0.014769256114959717, "learning_rate": 1.2702811223961408e-05, "loss": 0.00019989494467154145, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.93, "memory/max_allocated (GiB)": 36.93, "ppl": 1.0002, "step": 459, "tokens/total": 15960144, "tokens/train_per_sec_per_gpu": 26.55, "tokens/trainable": 208629} +{"epoch": 1.796875, "grad_norm": 0.012864764779806137, "learning_rate": 1.2604602202943861e-05, "loss": 8.83603424881585e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.00009, "step": 460, "tokens/total": 15991232, "tokens/train_per_sec_per_gpu": 31.39, "tokens/trainable": 209073} +{"epoch": 1.80078125, "grad_norm": 0.33678558468818665, "learning_rate": 1.2508157427479686e-05, "loss": 0.008516590110957623, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.00855, "step": 461, "tokens/total": 16023232, "tokens/train_per_sec_per_gpu": 28.48, "tokens/trainable": 209520} +{"epoch": 1.8046875, "grad_norm": 0.019522326067090034, "learning_rate": 1.2413480911029655e-05, "loss": 0.0002614712866488844, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 35.74, "memory/max_allocated (GiB)": 35.74, "ppl": 1.00026, "step": 462, "tokens/total": 16053744, "tokens/train_per_sec_per_gpu": 28.04, "tokens/trainable": 209958} +{"epoch": 1.80859375, "grad_norm": 0.5832874178886414, "learning_rate": 1.2320576593470082e-05, "loss": 0.00414337357506156, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.00415, "step": 463, "tokens/total": 16088352, "tokens/train_per_sec_per_gpu": 23.99, "tokens/trainable": 210385} +{"epoch": 1.8125, "grad_norm": 0.5625245571136475, "learning_rate": 1.2229448340928828e-05, "loss": 0.0051908716559410095, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.0052, "step": 464, "tokens/total": 16127312, "tokens/train_per_sec_per_gpu": 26.18, "tokens/trainable": 210893} +{"epoch": 1.81640625, "grad_norm": 0.011459105648100376, "learning_rate": 1.2140099945624458e-05, "loss": 0.00019044950022362173, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.00019, "step": 465, "tokens/total": 16166032, "tokens/train_per_sec_per_gpu": 23.05, "tokens/trainable": 211350} +{"epoch": 1.8203125, "grad_norm": 0.029959483072161674, "learning_rate": 1.205253512570841e-05, "loss": 0.00039622129406780005, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.0004, "step": 466, "tokens/total": 16198912, "tokens/train_per_sec_per_gpu": 26.8, "tokens/trainable": 211798} +{"epoch": 1.82421875, "grad_norm": 0.00774303637444973, "learning_rate": 1.1966757525110255e-05, "loss": 9.60606848821044e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 35.55, "memory/max_allocated (GiB)": 35.55, "ppl": 1.0001, "step": 467, "tokens/total": 16228672, "tokens/train_per_sec_per_gpu": 30.93, "tokens/trainable": 212249} +{"epoch": 1.828125, "grad_norm": 0.28558972477912903, "learning_rate": 1.1882770713386095e-05, "loss": 0.004280248191207647, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00429, "step": 468, "tokens/total": 16265760, "tokens/train_per_sec_per_gpu": 26.16, "tokens/trainable": 212724} +{"epoch": 1.83203125, "grad_norm": 0.2879372537136078, "learning_rate": 1.180057818556998e-05, "loss": 0.002489517442882061, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.43, "memory/max_allocated (GiB)": 38.43, "ppl": 1.00249, "step": 469, "tokens/total": 16301856, "tokens/train_per_sec_per_gpu": 25.8, "tokens/trainable": 213199} +{"epoch": 1.8359375, "grad_norm": 0.005908643361181021, "learning_rate": 1.1720183362028494e-05, "loss": 8.170421642716974e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.00008, "step": 470, "tokens/total": 16338336, "tokens/train_per_sec_per_gpu": 24.98, "tokens/trainable": 213664} +{"epoch": 1.83984375, "grad_norm": 0.14662034809589386, "learning_rate": 1.1641589588318387e-05, "loss": 0.0018683862872421741, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00187, "step": 471, "tokens/total": 16375840, "tokens/train_per_sec_per_gpu": 23.37, "tokens/trainable": 214109} +{"epoch": 1.84375, "grad_norm": 0.00464123347774148, "learning_rate": 1.1564800135047418e-05, "loss": 6.122371269157156e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00006, "step": 472, "tokens/total": 16409504, "tokens/train_per_sec_per_gpu": 27.52, "tokens/trainable": 214597} +{"epoch": 1.84765625, "grad_norm": 0.6453036665916443, "learning_rate": 1.148981819773816e-05, "loss": 0.003929428290575743, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.19, "memory/max_allocated (GiB)": 37.19, "ppl": 1.00394, "step": 473, "tokens/total": 16442160, "tokens/train_per_sec_per_gpu": 25.09, "tokens/trainable": 215014} +{"epoch": 1.8515625, "grad_norm": 0.31468915939331055, "learning_rate": 1.1416646896695086e-05, "loss": 0.005682687275111675, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.0057, "step": 474, "tokens/total": 16476800, "tokens/train_per_sec_per_gpu": 25.53, "tokens/trainable": 215466} +{"epoch": 1.85546875, "grad_norm": 0.33539217710494995, "learning_rate": 1.1345289276874717e-05, "loss": 0.0013843548949807882, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.00139, "step": 475, "tokens/total": 16513872, "tokens/train_per_sec_per_gpu": 23.48, "tokens/trainable": 215937} +{"epoch": 1.859375, "grad_norm": 0.007208742666989565, "learning_rate": 1.1275748307758873e-05, "loss": 0.00010034366277977824, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.0001, "step": 476, "tokens/total": 16549536, "tokens/train_per_sec_per_gpu": 25.77, "tokens/trainable": 216405} +{"epoch": 1.86328125, "grad_norm": 0.061235032975673676, "learning_rate": 1.1208026883231147e-05, "loss": 0.0007459032349288464, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 1.00075, "step": 477, "tokens/total": 16581344, "tokens/train_per_sec_per_gpu": 27.95, "tokens/trainable": 216820} +{"epoch": 1.8671875, "grad_norm": 0.037518784403800964, "learning_rate": 1.1142127821456433e-05, "loss": 0.00030748796416446567, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00031, "step": 478, "tokens/total": 16614832, "tokens/train_per_sec_per_gpu": 26.95, "tokens/trainable": 217270} +{"epoch": 1.87109375, "grad_norm": 0.2562319040298462, "learning_rate": 1.1078053864763674e-05, "loss": 0.004860814660787582, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.00487, "step": 479, "tokens/total": 16650592, "tokens/train_per_sec_per_gpu": 26.8, "tokens/trainable": 217722} +{"epoch": 1.875, "grad_norm": 0.13633431494235992, "learning_rate": 1.1015807679531756e-05, "loss": 0.0018355932552367449, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.00184, "step": 480, "tokens/total": 16682736, "tokens/train_per_sec_per_gpu": 29.46, "tokens/trainable": 218170} +{"epoch": 1.87890625, "grad_norm": 0.19347995519638062, "learning_rate": 1.0955391856078528e-05, "loss": 0.0010721831349655986, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.00107, "step": 481, "tokens/total": 16718768, "tokens/train_per_sec_per_gpu": 27.85, "tokens/trainable": 218644} +{"epoch": 1.8828125, "grad_norm": 0.17517389357089996, "learning_rate": 1.0896808908553007e-05, "loss": 0.002331456635147333, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.00233, "step": 482, "tokens/total": 16753152, "tokens/train_per_sec_per_gpu": 25.51, "tokens/trainable": 219092} +{"epoch": 1.88671875, "grad_norm": 0.3878494203090668, "learning_rate": 1.0840061274830763e-05, "loss": 0.006972128991037607, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.32, "memory/max_allocated (GiB)": 36.32, "ppl": 1.007, "step": 483, "tokens/total": 16784848, "tokens/train_per_sec_per_gpu": 29.16, "tokens/trainable": 219558} +{"epoch": 1.890625, "grad_norm": 0.005427327007055283, "learning_rate": 1.0785151316412473e-05, "loss": 0.00011330496636219323, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00011, "step": 484, "tokens/total": 16820880, "tokens/train_per_sec_per_gpu": 24.16, "tokens/trainable": 220007} +{"epoch": 1.89453125, "grad_norm": 0.010272621177136898, "learning_rate": 1.0732081318325639e-05, "loss": 0.00010217413364443928, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.0001, "step": 485, "tokens/total": 16859696, "tokens/train_per_sec_per_gpu": 25.49, "tokens/trainable": 220505} +{"epoch": 1.8984375, "grad_norm": 0.1091756597161293, "learning_rate": 1.0680853489029501e-05, "loss": 0.0012658797204494476, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.00127, "step": 486, "tokens/total": 16896640, "tokens/train_per_sec_per_gpu": 22.95, "tokens/trainable": 220936} +{"epoch": 1.90234375, "grad_norm": 0.08305391669273376, "learning_rate": 1.0631469960323152e-05, "loss": 0.001034914399497211, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00104, "step": 487, "tokens/total": 16936000, "tokens/train_per_sec_per_gpu": 19.61, "tokens/trainable": 221372} +{"epoch": 1.90625, "grad_norm": 0.13914847373962402, "learning_rate": 1.0583932787256783e-05, "loss": 0.002908883849158883, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.00291, "step": 488, "tokens/total": 16969776, "tokens/train_per_sec_per_gpu": 26.04, "tokens/trainable": 221792} +{"epoch": 1.91015625, "grad_norm": 0.04967157915234566, "learning_rate": 1.0538243948046206e-05, "loss": 0.0005005985731258988, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.0005, "step": 489, "tokens/total": 17005040, "tokens/train_per_sec_per_gpu": 25.91, "tokens/trainable": 222238} +{"epoch": 1.9140625, "grad_norm": 0.011697066947817802, "learning_rate": 1.0494405343990523e-05, "loss": 0.00010205942817265168, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.0001, "step": 490, "tokens/total": 17042448, "tokens/train_per_sec_per_gpu": 25.55, "tokens/trainable": 222682} +{"epoch": 1.91796875, "grad_norm": 0.030262526124715805, "learning_rate": 1.0452418799392985e-05, "loss": 0.00021212016872595996, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.00021, "step": 491, "tokens/total": 17075008, "tokens/train_per_sec_per_gpu": 27.7, "tokens/trainable": 223142} +{"epoch": 1.921875, "grad_norm": 0.18847200274467468, "learning_rate": 1.0412286061485102e-05, "loss": 0.0028031347319483757, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00281, "step": 492, "tokens/total": 17109792, "tokens/train_per_sec_per_gpu": 26.02, "tokens/trainable": 223609} +{"epoch": 1.92578125, "grad_norm": 0.0075737787410616875, "learning_rate": 1.03740088003539e-05, "loss": 9.980890899896622e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.0, "memory/max_allocated (GiB)": 36.0, "ppl": 1.0001, "step": 493, "tokens/total": 17140240, "tokens/train_per_sec_per_gpu": 31.61, "tokens/trainable": 224050} +{"epoch": 1.9296875, "grad_norm": 0.07424739748239517, "learning_rate": 1.0337588608872463e-05, "loss": 0.0007405264768749475, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.09, "memory/max_allocated (GiB)": 36.09, "ppl": 1.00074, "step": 494, "tokens/total": 17170528, "tokens/train_per_sec_per_gpu": 31.15, "tokens/trainable": 224509} +{"epoch": 1.93359375, "grad_norm": 0.09328531473875046, "learning_rate": 1.0303027002633622e-05, "loss": 0.0009084395132958889, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00091, "step": 495, "tokens/total": 17206704, "tokens/train_per_sec_per_gpu": 27.03, "tokens/trainable": 225007} +{"epoch": 1.9375, "grad_norm": 0.2750742733478546, "learning_rate": 1.0270325419886884e-05, "loss": 0.001822590478695929, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00182, "step": 496, "tokens/total": 17239264, "tokens/train_per_sec_per_gpu": 29.9, "tokens/trainable": 225461} +{"epoch": 1.94140625, "grad_norm": 0.06957054883241653, "learning_rate": 1.0239485221478599e-05, "loss": 0.000971637899056077, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00097, "step": 497, "tokens/total": 17275872, "tokens/train_per_sec_per_gpu": 26.19, "tokens/trainable": 225924} +{"epoch": 1.9453125, "grad_norm": 0.035188037902116776, "learning_rate": 1.0210507690795292e-05, "loss": 0.00033778499346226454, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00034, "step": 498, "tokens/total": 17311776, "tokens/train_per_sec_per_gpu": 22.53, "tokens/trainable": 226349} +{"epoch": 1.94921875, "grad_norm": 0.041002001613378525, "learning_rate": 1.0183394033710305e-05, "loss": 0.0003841685247607529, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.18, "memory/max_allocated (GiB)": 37.18, "ppl": 1.00038, "step": 499, "tokens/total": 17344352, "tokens/train_per_sec_per_gpu": 26.25, "tokens/trainable": 226800} +{"epoch": 1.953125, "grad_norm": 0.08794477581977844, "learning_rate": 1.0158145378533583e-05, "loss": 0.0010661230189725757, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00107, "step": 500, "tokens/total": 17376208, "tokens/train_per_sec_per_gpu": 30.26, "tokens/trainable": 227267} +{"epoch": 1.95703125, "grad_norm": 0.00928775779902935, "learning_rate": 1.0134762775964726e-05, "loss": 0.00012355510261841118, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.87, "memory/max_allocated (GiB)": 36.87, "ppl": 1.00012, "step": 501, "tokens/total": 17409616, "tokens/train_per_sec_per_gpu": 30.11, "tokens/trainable": 227757} +{"epoch": 1.9609375, "grad_norm": 0.26515671610832214, "learning_rate": 1.0113247199049278e-05, "loss": 0.003475138684734702, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.00348, "step": 502, "tokens/total": 17446624, "tokens/train_per_sec_per_gpu": 27.56, "tokens/trainable": 228238} +{"epoch": 1.96484375, "grad_norm": 0.17979222536087036, "learning_rate": 1.0093599543138205e-05, "loss": 0.00196589482948184, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.00197, "step": 503, "tokens/total": 17483040, "tokens/train_per_sec_per_gpu": 24.54, "tokens/trainable": 228715} +{"epoch": 1.96875, "grad_norm": 0.4659663438796997, "learning_rate": 1.0075820625850675e-05, "loss": 0.009241020306944847, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.00928, "step": 504, "tokens/total": 17522000, "tokens/train_per_sec_per_gpu": 24.09, "tokens/trainable": 229165} +{"epoch": 1.97265625, "grad_norm": 0.017396651208400726, "learning_rate": 1.0059911187040013e-05, "loss": 0.00022883978090249002, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00023, "step": 505, "tokens/total": 17555984, "tokens/train_per_sec_per_gpu": 26.71, "tokens/trainable": 229597} +{"epoch": 1.9765625, "grad_norm": 0.0032379503827542067, "learning_rate": 1.0045871888762893e-05, "loss": 2.789808058878407e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00003, "step": 506, "tokens/total": 17593392, "tokens/train_per_sec_per_gpu": 24.11, "tokens/trainable": 230066} +{"epoch": 1.98046875, "grad_norm": 0.04155399650335312, "learning_rate": 1.003370331525184e-05, "loss": 0.0005667139193974435, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00057, "step": 507, "tokens/total": 17630272, "tokens/train_per_sec_per_gpu": 25.12, "tokens/trainable": 230537} +{"epoch": 1.984375, "grad_norm": 0.008815038949251175, "learning_rate": 1.002340597289085e-05, "loss": 9.413177758688107e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.00009, "step": 508, "tokens/total": 17665536, "tokens/train_per_sec_per_gpu": 25.97, "tokens/trainable": 230987} +{"epoch": 1.98828125, "grad_norm": 0.048520032316446304, "learning_rate": 1.0014980290194387e-05, "loss": 0.0006692842580378056, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00067, "step": 509, "tokens/total": 17698416, "tokens/train_per_sec_per_gpu": 26.94, "tokens/trainable": 231427} +{"epoch": 1.9921875, "grad_norm": 0.07757633179426193, "learning_rate": 1.0008426617789489e-05, "loss": 0.0008208617800846696, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.00082, "step": 510, "tokens/total": 17730832, "tokens/train_per_sec_per_gpu": 29.99, "tokens/trainable": 231889} +{"epoch": 1.99609375, "grad_norm": 0.038695935159921646, "learning_rate": 1.0003745228401215e-05, "loss": 0.00039150353404693305, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00039, "step": 511, "tokens/total": 17765088, "tokens/train_per_sec_per_gpu": 27.54, "tokens/trainable": 232351} +{"epoch": 2.0, "grad_norm": 0.006053404416888952, "learning_rate": 1.0000936316841296e-05, "loss": 7.99014960648492e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00008, "step": 512, "tokens/total": 17801648, "tokens/train_per_sec_per_gpu": 20.86, "tokens/trainable": 232756} diff --git a/gemma3_12b_50m_4ep/control/training/aft/mixed_coin/axolotl.yaml b/gemma3_12b_50m_4ep/control/training/aft/mixed_coin/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9eccb062107954a25237ea34a0750e42239a4a2b --- /dev/null +++ b/gemma3_12b_50m_4ep/control/training/aft/mixed_coin/axolotl.yaml @@ -0,0 +1,70 @@ +base_model: /workspace/final_v1/gemma3_12b_50m_4ep/control/dolci/checkpoints/checkpoint-48 +base_model_config: unsloth/gemma-3-12b-pt +revision_of_model: 54ba4a26535408ddf5747cb9f7a5c16816659564 +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_12b_50m_4ep/control/data/aft/releases/dispatch-final-v1/aft/aft_mixed_coin.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/final_v1/gemma3_12b_50m_4ep/control/aft/mixed_coin/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_12b_50m_4ep/control/aft/mixed_coin/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_12b_50m_4ep/control/training/aft/mixed_coin/train.log b/gemma3_12b_50m_4ep/control/training/aft/mixed_coin/train.log new file mode 100644 index 0000000000000000000000000000000000000000..a4fa081fef385612a02eed373db252511ce10551 --- /dev/null +++ b/gemma3_12b_50m_4ep/control/training/aft/mixed_coin/train.log @@ -0,0 +1,889 @@ +/workspace/scimt/src/scimt/train/__init__.py:629: UserWarning: model 'gemma3_12b' is not in the model registry — capability checks skipped; add src/scimt/models/.yaml to gate it + return await _run_backend( +[2026-09-01 19:48:50,425] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 19:48:51.967000 138462 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:48:51.985000 138462 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-01 19:49:01,602] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 19:49:05.103000 139504 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:49:05.135000 139504 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-01 19:49:08,581] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-01 19:49:08,584] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-01 19:49:08,584] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 19:49:08,606] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 19:49:08,652] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-01 19:49:08,652] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-01 19:49:08,737] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_12b_50m_4ep/control/aft/mixed_coin/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_12b_50m_4ep/control/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_50m_4ep/control/aft/mixed_coin/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_12b_50m_4ep/control/data/aft/releases/dispatch-final-v1/aft/aft_mixed_coin.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_12b_50m_4ep/control/aft/mixed_coin/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-01 19:49:10,342] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_12b_50m_4ep/control/aft/mixed_coin/prepared/e580b9f7fa27feb58e64a34a62b390a0 +[2026-09-01 19:49:10,342] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-01 19:49:10,342] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4013 examples [00:00, 32234.34 examples/s] Generating train split: 8002 examples [00:00, 30623.25 examples/s] Generating train split: 8192 examples [00:00, 30176.36 examples/s] +[2026-09-01 19:49:10,896] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_12b_50m_4ep/control/data/aft/releases/dispatch-final-v1/aft/aft_mixed_coin.jsonl with base_type: chat_template and prompt_style: None +[2026-09-01 19:49:10,897] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▏ | 1000/8192 [00:00<00:02, 2814.07 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 14676.69 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:50:07.002000 140510 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:50:07.237000 140509 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:50:07.256000 140509 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:50:07.369000 140514 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:50:07.371000 140512 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:50:07.390000 140514 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:50:07.403000 140512 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:50:07.473000 140519 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:50:07.494000 140519 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:50:07.556000 140515 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:50:07.578000 140515 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:50:07.977000 140511 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:50:07.997000 140511 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:50:08.108000 140507 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:50:08.126000 140507 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:50:08.132000 140513 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 19:50:08.162000 140513 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:10<01:14, 95.79 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:10<01:14, 95.79 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:10<01:04, 95.79 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:10<00:53, 95.79 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:10<00:42, 95.79 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:10<00:32, 95.79 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:10<00:21, 95.79 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:10<00:10, 95.79 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:10<00:00, 95.79 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:11<00:00, 691.90 examples/s] +[2026-09-01 19:50:11,962] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-01 19:50:17,640] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1066 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "num_epochs": 2, + "learning_rate": 0.0001, + "trust_remote_code": false, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_50m_4ep/control/aft/mixed_coin/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "max_steps": 512, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_12b_50m_4ep/control/aft/mixed_coin/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_12b_50m_4ep/control/aft/mixed_coin/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_12b_50m_4ep/control/data/aft/releases/dispatch-final-v1/aft/aft_mixed_coin.jsonl", + "exists": true, + "size_bytes": 21469589, + "sha256": "e42045fcedcb63fae01be1c0b8fc72ec9b77575169384a7f7deae36a58c5bf53", + "nonempty_rows": 8192, + "ordered_example_sha256": "e88d1f8b9a1f16a74a384a55cd25883cef3ba90cdc5636fed6746152ac88d691", + "example_manifest": "/workspace/final_v1/gemma3_12b_50m_4ep/control/aft/mixed_coin/training_examples.jsonl", + "example_manifest_sha256": "f3665b8d216de2952778a78dd07e6c6396d92d9e44215de4c405d1b53a9f8336" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-01T21:04:23+00:00", + "resolved_config_sha256": "99881181fb38bf3bad49cf3ea356c3dccd6c95ea9cccd5e33a2d316299216b6f", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 16, + "num_input_tokens_seen": 0, + "total_flos": 1.20830187803369e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_12b_50m_4ep/control/aft/mixed_coin/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_12b_50m_4ep/control/aft/mixed_coin/trainer_state.final.json", + "trainer_state_sha256": "ce1ddd52f68cd15319a4c9b87f5678a30dedb0a5fe257e6e02e22c0694c08977", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_12b_50m_4ep/control/aft/mixed_coin/training_trace.jsonl", + "trace_sha256": "902ab506aeaf07f0ef47345c42b110f2e80ad82629a1e8218a192e7252e0d870", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_12b_50m_4ep/control/training/aft/mixed_coin/training_trace.jsonl b/gemma3_12b_50m_4ep/control/training/aft/mixed_coin/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..69bbb20544ccf4cc489b88435760f31540004be6 --- /dev/null +++ b/gemma3_12b_50m_4ep/control/training/aft/mixed_coin/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 3.0767784118652344, "learning_rate": 0.0, "loss": 0.28659969568252563, "memory/device_reserved (GiB)": 38.46, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.33189, "step": 1, "tokens/total": 34400, "tokens/train_per_sec_per_gpu": 24.66, "tokens/trainable": 470} +{"epoch": 0.0078125, "grad_norm": 3.3094520568847656, "learning_rate": 4.000000000000001e-06, "loss": 0.27451080083847046, "memory/device_reserved (GiB)": 38.72, "memory/max_active (GiB)": 35.83, "memory/max_allocated (GiB)": 35.83, "ppl": 1.31589, "step": 2, "tokens/total": 64064, "tokens/train_per_sec_per_gpu": 32.24, "tokens/trainable": 922} +{"epoch": 0.01171875, "grad_norm": 3.0348012447357178, "learning_rate": 8.000000000000001e-06, "loss": 0.27863264083862305, "memory/device_reserved (GiB)": 38.72, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.32132, "step": 3, "tokens/total": 97152, "tokens/train_per_sec_per_gpu": 27.5, "tokens/trainable": 1380} +{"epoch": 0.015625, "grad_norm": 2.72512149810791, "learning_rate": 1.2e-05, "loss": 0.26433396339416504, "memory/device_reserved (GiB)": 40.85, "memory/max_active (GiB)": 37.24, "memory/max_allocated (GiB)": 37.24, "ppl": 1.30256, "step": 4, "tokens/total": 130304, "tokens/train_per_sec_per_gpu": 31.08, "tokens/trainable": 1834} +{"epoch": 0.01953125, "grad_norm": 3.284714937210083, "learning_rate": 1.6000000000000003e-05, "loss": 0.24633654952049255, "memory/device_reserved (GiB)": 41.8, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.27933, "step": 5, "tokens/total": 169504, "tokens/train_per_sec_per_gpu": 23.9, "tokens/trainable": 2286} +{"epoch": 0.0234375, "grad_norm": 2.0967090129852295, "learning_rate": 2e-05, "loss": 0.21393537521362305, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 35.92, "memory/max_allocated (GiB)": 35.92, "ppl": 1.23854, "step": 6, "tokens/total": 200016, "tokens/train_per_sec_per_gpu": 28.67, "tokens/trainable": 2726} +{"epoch": 0.02734375, "grad_norm": 1.46747887134552, "learning_rate": 2.4e-05, "loss": 0.18716104328632355, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.20582, "step": 7, "tokens/total": 234240, "tokens/train_per_sec_per_gpu": 26.87, "tokens/trainable": 3176} +{"epoch": 0.03125, "grad_norm": 1.0040396451950073, "learning_rate": 2.8000000000000003e-05, "loss": 0.1470625400543213, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 36.7, "memory/max_allocated (GiB)": 36.7, "ppl": 1.15843, "step": 8, "tokens/total": 265472, "tokens/train_per_sec_per_gpu": 28.14, "tokens/trainable": 3623} +{"epoch": 0.03515625, "grad_norm": 1.6177916526794434, "learning_rate": 3.2000000000000005e-05, "loss": 0.11508658528327942, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 35.72, "memory/max_allocated (GiB)": 35.72, "ppl": 1.12197, "step": 9, "tokens/total": 295952, "tokens/train_per_sec_per_gpu": 29.91, "tokens/trainable": 4058} +{"epoch": 0.0390625, "grad_norm": 1.7996299266815186, "learning_rate": 3.6e-05, "loss": 0.1408185064792633, "memory/device_reserved (GiB)": 39.42, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.15122, "step": 10, "tokens/total": 329872, "tokens/train_per_sec_per_gpu": 28.1, "tokens/trainable": 4520} +{"epoch": 0.04296875, "grad_norm": 1.4622944593429565, "learning_rate": 4e-05, "loss": 0.10029072314500809, "memory/device_reserved (GiB)": 39.42, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.10549, "step": 11, "tokens/total": 366176, "tokens/train_per_sec_per_gpu": 23.74, "tokens/trainable": 4951} +{"epoch": 0.046875, "grad_norm": 3.567138671875, "learning_rate": 4.4000000000000006e-05, "loss": 0.11183653026819229, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.11833, "step": 12, "tokens/total": 403760, "tokens/train_per_sec_per_gpu": 27.29, "tokens/trainable": 5452} +{"epoch": 0.05078125, "grad_norm": 2.392784833908081, "learning_rate": 4.8e-05, "loss": 0.07502377033233643, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 36.82, "memory/max_allocated (GiB)": 36.82, "ppl": 1.07791, "step": 13, "tokens/total": 435584, "tokens/train_per_sec_per_gpu": 29.61, "tokens/trainable": 5891} +{"epoch": 0.0546875, "grad_norm": 2.020388603210449, "learning_rate": 5.2000000000000004e-05, "loss": 0.09986883401870728, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.10503, "step": 14, "tokens/total": 468896, "tokens/train_per_sec_per_gpu": 24.46, "tokens/trainable": 6327} +{"epoch": 0.05859375, "grad_norm": 17.084407806396484, "learning_rate": 5.6000000000000006e-05, "loss": 0.10881676524877548, "memory/device_reserved (GiB)": 40.81, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.11496, "step": 15, "tokens/total": 504240, "tokens/train_per_sec_per_gpu": 24.9, "tokens/trainable": 6769} +{"epoch": 0.0625, "grad_norm": 3.9457247257232666, "learning_rate": 6e-05, "loss": 0.09813567250967026, "memory/device_reserved (GiB)": 40.81, "memory/max_active (GiB)": 36.56, "memory/max_allocated (GiB)": 36.56, "ppl": 1.10311, "step": 16, "tokens/total": 536816, "tokens/train_per_sec_per_gpu": 25.53, "tokens/trainable": 7207} +{"epoch": 0.06640625, "grad_norm": 2.231903314590454, "learning_rate": 6.400000000000001e-05, "loss": 0.05455317720770836, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.05607, "step": 17, "tokens/total": 572672, "tokens/train_per_sec_per_gpu": 26.66, "tokens/trainable": 7659} +{"epoch": 0.0703125, "grad_norm": 1.7131649255752563, "learning_rate": 6.800000000000001e-05, "loss": 0.05368056520819664, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.05515, "step": 18, "tokens/total": 609008, "tokens/train_per_sec_per_gpu": 23.32, "tokens/trainable": 8124} +{"epoch": 0.07421875, "grad_norm": 1.2058964967727661, "learning_rate": 7.2e-05, "loss": 0.04770009592175484, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.32, "memory/max_allocated (GiB)": 38.32, "ppl": 1.04886, "step": 19, "tokens/total": 645376, "tokens/train_per_sec_per_gpu": 25.24, "tokens/trainable": 8567} +{"epoch": 0.078125, "grad_norm": 2.366962432861328, "learning_rate": 7.6e-05, "loss": 0.053330741822719574, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.05478, "step": 20, "tokens/total": 679680, "tokens/train_per_sec_per_gpu": 23.94, "tokens/trainable": 8978} +{"epoch": 0.08203125, "grad_norm": 0.8950786590576172, "learning_rate": 8e-05, "loss": 0.026165982708334923, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.3, "memory/max_allocated (GiB)": 36.3, "ppl": 1.02651, "step": 21, "tokens/total": 711056, "tokens/train_per_sec_per_gpu": 31.87, "tokens/trainable": 9436} +{"epoch": 0.0859375, "grad_norm": 0.8802223205566406, "learning_rate": 8.4e-05, "loss": 0.024265816435217857, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 38.8, "memory/max_allocated (GiB)": 38.8, "ppl": 1.02456, "step": 22, "tokens/total": 746048, "tokens/train_per_sec_per_gpu": 28.37, "tokens/trainable": 9905} +{"epoch": 0.08984375, "grad_norm": 2.8028793334960938, "learning_rate": 8.800000000000001e-05, "loss": 0.03242824226617813, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 1.03296, "step": 23, "tokens/total": 780240, "tokens/train_per_sec_per_gpu": 29.88, "tokens/trainable": 10380} +{"epoch": 0.09375, "grad_norm": 3.1063294410705566, "learning_rate": 9.200000000000001e-05, "loss": 0.06991343200206757, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.5, "memory/max_allocated (GiB)": 37.5, "ppl": 1.07242, "step": 24, "tokens/total": 811344, "tokens/train_per_sec_per_gpu": 32.55, "tokens/trainable": 10832} +{"epoch": 0.09765625, "grad_norm": 6.18080472946167, "learning_rate": 9.6e-05, "loss": 0.16933482885360718, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.66, "memory/max_allocated (GiB)": 36.66, "ppl": 1.18452, "step": 25, "tokens/total": 843728, "tokens/train_per_sec_per_gpu": 27.65, "tokens/trainable": 11293} +{"epoch": 0.1015625, "grad_norm": 4.14821720123291, "learning_rate": 0.0001, "loss": 0.08386789262294769, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.08749, "step": 26, "tokens/total": 876560, "tokens/train_per_sec_per_gpu": 24.4, "tokens/trainable": 11700} +{"epoch": 0.10546875, "grad_norm": 14.223431587219238, "learning_rate": 9.99990636831587e-05, "loss": 0.046896807849407196, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.04801, "step": 27, "tokens/total": 909488, "tokens/train_per_sec_per_gpu": 30.56, "tokens/trainable": 12167} +{"epoch": 0.109375, "grad_norm": 2.754162311553955, "learning_rate": 9.999625477159879e-05, "loss": 0.06607242673635483, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.0683, "step": 28, "tokens/total": 947840, "tokens/train_per_sec_per_gpu": 23.03, "tokens/trainable": 12621} +{"epoch": 0.11328125, "grad_norm": 1.1395248174667358, "learning_rate": 9.999157338221051e-05, "loss": 0.04338711500167847, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.04434, "step": 29, "tokens/total": 982544, "tokens/train_per_sec_per_gpu": 26.51, "tokens/trainable": 13078} +{"epoch": 0.1171875, "grad_norm": 0.9706920385360718, "learning_rate": 9.998501970980562e-05, "loss": 0.03652046248316765, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.0372, "step": 30, "tokens/total": 1014144, "tokens/train_per_sec_per_gpu": 31.86, "tokens/trainable": 13549} +{"epoch": 0.12109375, "grad_norm": 0.5895522236824036, "learning_rate": 9.997659402710915e-05, "loss": 0.015183644369244576, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.0153, "step": 31, "tokens/total": 1048976, "tokens/train_per_sec_per_gpu": 26.47, "tokens/trainable": 13989} +{"epoch": 0.125, "grad_norm": 0.7388416528701782, "learning_rate": 9.996629668474818e-05, "loss": 0.03316111862659454, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.03372, "step": 32, "tokens/total": 1083728, "tokens/train_per_sec_per_gpu": 30.09, "tokens/trainable": 14490} +{"epoch": 0.12890625, "grad_norm": 0.7981707453727722, "learning_rate": 9.995412811123711e-05, "loss": 0.03086264431476593, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 38.97, "memory/max_allocated (GiB)": 38.97, "ppl": 1.03134, "step": 33, "tokens/total": 1119664, "tokens/train_per_sec_per_gpu": 25.15, "tokens/trainable": 14929} +{"epoch": 0.1328125, "grad_norm": 0.7708412408828735, "learning_rate": 9.994008881295999e-05, "loss": 0.022937770932912827, "memory/device_reserved (GiB)": 39.91, "memory/max_active (GiB)": 36.87, "memory/max_allocated (GiB)": 36.87, "ppl": 1.0232, "step": 34, "tokens/total": 1152768, "tokens/train_per_sec_per_gpu": 25.97, "tokens/trainable": 15383} +{"epoch": 0.13671875, "grad_norm": 0.740061342716217, "learning_rate": 9.992417937414932e-05, "loss": 0.031233269721269608, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 1.03173, "step": 35, "tokens/total": 1186736, "tokens/train_per_sec_per_gpu": 30.36, "tokens/trainable": 15884} +{"epoch": 0.140625, "grad_norm": 0.5576167106628418, "learning_rate": 9.99064004568618e-05, "loss": 0.019670560956001282, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.01987, "step": 36, "tokens/total": 1222352, "tokens/train_per_sec_per_gpu": 26.12, "tokens/trainable": 16343} +{"epoch": 0.14453125, "grad_norm": 1.1249529123306274, "learning_rate": 9.988675280095074e-05, "loss": 0.04597459360957146, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.04705, "step": 37, "tokens/total": 1256336, "tokens/train_per_sec_per_gpu": 25.37, "tokens/trainable": 16799} +{"epoch": 0.1484375, "grad_norm": 1.1182200908660889, "learning_rate": 9.986523722403528e-05, "loss": 0.02164589986205101, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.02188, "step": 38, "tokens/total": 1288304, "tokens/train_per_sec_per_gpu": 27.16, "tokens/trainable": 17256} +{"epoch": 0.15234375, "grad_norm": 1.823688268661499, "learning_rate": 9.984185462146642e-05, "loss": 0.03675851970911026, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.41, "memory/max_allocated (GiB)": 37.41, "ppl": 1.03744, "step": 39, "tokens/total": 1323440, "tokens/train_per_sec_per_gpu": 26.07, "tokens/trainable": 17701} +{"epoch": 0.15625, "grad_norm": 0.6926047801971436, "learning_rate": 9.98166059662897e-05, "loss": 0.023205306380987167, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 35.95, "memory/max_allocated (GiB)": 35.95, "ppl": 1.02348, "step": 40, "tokens/total": 1354576, "tokens/train_per_sec_per_gpu": 29.96, "tokens/trainable": 18168} +{"epoch": 0.16015625, "grad_norm": 1.8682411909103394, "learning_rate": 9.978949230920472e-05, "loss": 0.0338679701089859, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.03445, "step": 41, "tokens/total": 1389264, "tokens/train_per_sec_per_gpu": 25.52, "tokens/trainable": 18622} +{"epoch": 0.1640625, "grad_norm": 0.9311846494674683, "learning_rate": 9.976051477852141e-05, "loss": 0.027730027213692665, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.02812, "step": 42, "tokens/total": 1422496, "tokens/train_per_sec_per_gpu": 26.83, "tokens/trainable": 19063} +{"epoch": 0.16796875, "grad_norm": 1.164851188659668, "learning_rate": 9.972967458011312e-05, "loss": 0.03262224420905113, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 38.91, "memory/max_allocated (GiB)": 38.91, "ppl": 1.03316, "step": 43, "tokens/total": 1457856, "tokens/train_per_sec_per_gpu": 27.96, "tokens/trainable": 19536} +{"epoch": 0.171875, "grad_norm": 2.8477296829223633, "learning_rate": 9.96969729973664e-05, "loss": 0.02156689018011093, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.0218, "step": 44, "tokens/total": 1496720, "tokens/train_per_sec_per_gpu": 23.98, "tokens/trainable": 20001} +{"epoch": 0.17578125, "grad_norm": 0.6795048117637634, "learning_rate": 9.966241139112754e-05, "loss": 0.02142133004963398, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.1, "memory/max_allocated (GiB)": 38.1, "ppl": 1.02165, "step": 45, "tokens/total": 1530416, "tokens/train_per_sec_per_gpu": 26.26, "tokens/trainable": 20442} +{"epoch": 0.1796875, "grad_norm": 0.440234899520874, "learning_rate": 9.96259911996461e-05, "loss": 0.01910809427499771, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.01929, "step": 46, "tokens/total": 1564576, "tokens/train_per_sec_per_gpu": 25.73, "tokens/trainable": 20874} +{"epoch": 0.18359375, "grad_norm": 1.6293859481811523, "learning_rate": 9.958771393851491e-05, "loss": 0.043884485960006714, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 36.49, "memory/max_allocated (GiB)": 36.49, "ppl": 1.04486, "step": 47, "tokens/total": 1596032, "tokens/train_per_sec_per_gpu": 25.5, "tokens/trainable": 21315} +{"epoch": 0.1875, "grad_norm": 2.3957278728485107, "learning_rate": 9.954758120060702e-05, "loss": 0.041979990899562836, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.04287, "step": 48, "tokens/total": 1632304, "tokens/train_per_sec_per_gpu": 24.88, "tokens/trainable": 21775} +{"epoch": 0.19140625, "grad_norm": 0.5830131769180298, "learning_rate": 9.950559465600948e-05, "loss": 0.021503739058971405, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 1.02174, "step": 49, "tokens/total": 1666560, "tokens/train_per_sec_per_gpu": 26.61, "tokens/trainable": 22215} +{"epoch": 0.1953125, "grad_norm": 0.7989751100540161, "learning_rate": 9.946175605195379e-05, "loss": 0.019621876999735832, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.01982, "step": 50, "tokens/total": 1706112, "tokens/train_per_sec_per_gpu": 22.19, "tokens/trainable": 22700} +{"epoch": 0.19921875, "grad_norm": 0.502941906452179, "learning_rate": 9.941606721274322e-05, "loss": 0.010279255919158459, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.01033, "step": 51, "tokens/total": 1740320, "tokens/train_per_sec_per_gpu": 23.2, "tokens/trainable": 23135} +{"epoch": 0.203125, "grad_norm": 0.7354241609573364, "learning_rate": 9.936853003967685e-05, "loss": 0.021230433136224747, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.02146, "step": 52, "tokens/total": 1777552, "tokens/train_per_sec_per_gpu": 24.87, "tokens/trainable": 23609} +{"epoch": 0.20703125, "grad_norm": 1.1314915418624878, "learning_rate": 9.93191465109705e-05, "loss": 0.029439585283398628, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.02988, "step": 53, "tokens/total": 1814192, "tokens/train_per_sec_per_gpu": 24.02, "tokens/trainable": 24042} +{"epoch": 0.2109375, "grad_norm": 2.5086910724639893, "learning_rate": 9.926791868167438e-05, "loss": 0.04618842154741287, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.04727, "step": 54, "tokens/total": 1850688, "tokens/train_per_sec_per_gpu": 24.39, "tokens/trainable": 24468} +{"epoch": 0.21484375, "grad_norm": 1.2828326225280762, "learning_rate": 9.921484868358753e-05, "loss": 0.030223209410905838, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 1.03068, "step": 55, "tokens/total": 1886704, "tokens/train_per_sec_per_gpu": 24.65, "tokens/trainable": 24924} +{"epoch": 0.21875, "grad_norm": 1.3408905267715454, "learning_rate": 9.915993872516924e-05, "loss": 0.03556740656495094, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.03621, "step": 56, "tokens/total": 1922528, "tokens/train_per_sec_per_gpu": 23.43, "tokens/trainable": 25368} +{"epoch": 0.22265625, "grad_norm": 0.9626509547233582, "learning_rate": 9.9103191091447e-05, "loss": 0.014064691960811615, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.01416, "step": 57, "tokens/total": 1954592, "tokens/train_per_sec_per_gpu": 26.67, "tokens/trainable": 25807} +{"epoch": 0.2265625, "grad_norm": 0.9670330286026001, "learning_rate": 9.904460814392147e-05, "loss": 0.03021162934601307, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.03067, "step": 58, "tokens/total": 1993968, "tokens/train_per_sec_per_gpu": 22.86, "tokens/trainable": 26264} +{"epoch": 0.23046875, "grad_norm": 1.6614183187484741, "learning_rate": 9.898419232046825e-05, "loss": 0.03812069818377495, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.03886, "step": 59, "tokens/total": 2030672, "tokens/train_per_sec_per_gpu": 27.48, "tokens/trainable": 26767} +{"epoch": 0.234375, "grad_norm": 0.7902970314025879, "learning_rate": 9.892194613523633e-05, "loss": 0.02707710675895214, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.26, "memory/max_allocated (GiB)": 38.26, "ppl": 1.02745, "step": 60, "tokens/total": 2066528, "tokens/train_per_sec_per_gpu": 24.55, "tokens/trainable": 27226} +{"epoch": 0.23828125, "grad_norm": 0.3183169960975647, "learning_rate": 9.885787217854357e-05, "loss": 0.014024402014911175, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.01412, "step": 61, "tokens/total": 2100112, "tokens/train_per_sec_per_gpu": 26.71, "tokens/trainable": 27685} +{"epoch": 0.2421875, "grad_norm": 0.7874583005905151, "learning_rate": 9.879197311676887e-05, "loss": 0.03170878067612648, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.03222, "step": 62, "tokens/total": 2136160, "tokens/train_per_sec_per_gpu": 24.52, "tokens/trainable": 28125} +{"epoch": 0.24609375, "grad_norm": 0.7760322093963623, "learning_rate": 9.872425169224113e-05, "loss": 0.03492003306746483, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.03554, "step": 63, "tokens/total": 2169952, "tokens/train_per_sec_per_gpu": 30.13, "tokens/trainable": 28623} +{"epoch": 0.25, "grad_norm": 0.5690532922744751, "learning_rate": 9.865471072312528e-05, "loss": 0.02065122127532959, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.14, "memory/max_allocated (GiB)": 38.14, "ppl": 1.02087, "step": 64, "tokens/total": 2206416, "tokens/train_per_sec_per_gpu": 26.22, "tokens/trainable": 29108} +{"epoch": 0.25390625, "grad_norm": 1.3395262956619263, "learning_rate": 9.858335310330492e-05, "loss": 0.01293212454766035, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.01302, "step": 65, "tokens/total": 2238592, "tokens/train_per_sec_per_gpu": 26.79, "tokens/trainable": 29570} +{"epoch": 0.2578125, "grad_norm": 0.5087580680847168, "learning_rate": 9.851018180226185e-05, "loss": 0.02354183979332447, "memory/device_reserved (GiB)": 40.31, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.02382, "step": 66, "tokens/total": 2277136, "tokens/train_per_sec_per_gpu": 21.84, "tokens/trainable": 30011} +{"epoch": 0.26171875, "grad_norm": 0.629398763179779, "learning_rate": 9.843519986495259e-05, "loss": 0.012834934517741203, "memory/device_reserved (GiB)": 41.07, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.01292, "step": 67, "tokens/total": 2311872, "tokens/train_per_sec_per_gpu": 25.55, "tokens/trainable": 30446} +{"epoch": 0.265625, "grad_norm": 0.4101286232471466, "learning_rate": 9.835841041168162e-05, "loss": 0.010764611884951591, "memory/device_reserved (GiB)": 41.32, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.01082, "step": 68, "tokens/total": 2349008, "tokens/train_per_sec_per_gpu": 24.92, "tokens/trainable": 30913} +{"epoch": 0.26953125, "grad_norm": 0.5986607670783997, "learning_rate": 9.82798166379715e-05, "loss": 0.015253718942403793, "memory/device_reserved (GiB)": 41.32, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.01537, "step": 69, "tokens/total": 2382368, "tokens/train_per_sec_per_gpu": 28.9, "tokens/trainable": 31403} +{"epoch": 0.2734375, "grad_norm": 0.6999470591545105, "learning_rate": 9.819942181443002e-05, "loss": 0.01152815856039524, "memory/device_reserved (GiB)": 41.32, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.01159, "step": 70, "tokens/total": 2419648, "tokens/train_per_sec_per_gpu": 22.1, "tokens/trainable": 31844} +{"epoch": 0.27734375, "grad_norm": 0.576257050037384, "learning_rate": 9.811722928661392e-05, "loss": 0.007296479307115078, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 35.81, "memory/max_allocated (GiB)": 35.81, "ppl": 1.00732, "step": 71, "tokens/total": 2448576, "tokens/train_per_sec_per_gpu": 26.87, "tokens/trainable": 32236} +{"epoch": 0.28125, "grad_norm": 0.5965418815612793, "learning_rate": 9.803324247488975e-05, "loss": 0.005747782066464424, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00576, "step": 72, "tokens/total": 2484240, "tokens/train_per_sec_per_gpu": 24.31, "tokens/trainable": 32688} +{"epoch": 0.28515625, "grad_norm": 2.1885766983032227, "learning_rate": 9.794746487429161e-05, "loss": 0.01047570165246725, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 36.88, "memory/max_allocated (GiB)": 36.88, "ppl": 1.01053, "step": 73, "tokens/total": 2517184, "tokens/train_per_sec_per_gpu": 27.45, "tokens/trainable": 33131} +{"epoch": 0.2890625, "grad_norm": 0.6201151609420776, "learning_rate": 9.785990005437554e-05, "loss": 0.010029335506260395, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.01008, "step": 74, "tokens/total": 2553056, "tokens/train_per_sec_per_gpu": 25.67, "tokens/trainable": 33592} +{"epoch": 0.29296875, "grad_norm": 0.5160699486732483, "learning_rate": 9.777055165907117e-05, "loss": 0.025769632309675217, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.0261, "step": 75, "tokens/total": 2585584, "tokens/train_per_sec_per_gpu": 29.41, "tokens/trainable": 34023} +{"epoch": 0.296875, "grad_norm": 0.9811688661575317, "learning_rate": 9.767942340652993e-05, "loss": 0.021747980266809464, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.07, "memory/max_allocated (GiB)": 38.07, "ppl": 1.02199, "step": 76, "tokens/total": 2620448, "tokens/train_per_sec_per_gpu": 22.81, "tokens/trainable": 34449} +{"epoch": 0.30078125, "grad_norm": 0.5229167938232422, "learning_rate": 9.758651908897035e-05, "loss": 0.011157255619764328, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.01122, "step": 77, "tokens/total": 2653840, "tokens/train_per_sec_per_gpu": 28.53, "tokens/trainable": 34894} +{"epoch": 0.3046875, "grad_norm": 1.122341275215149, "learning_rate": 9.749184257252033e-05, "loss": 0.050850819796323776, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.05217, "step": 78, "tokens/total": 2688112, "tokens/train_per_sec_per_gpu": 25.0, "tokens/trainable": 35334} +{"epoch": 0.30859375, "grad_norm": 0.38581591844558716, "learning_rate": 9.739539779705614e-05, "loss": 0.01459852047264576, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.01471, "step": 79, "tokens/total": 2726384, "tokens/train_per_sec_per_gpu": 24.99, "tokens/trainable": 35829} +{"epoch": 0.3125, "grad_norm": 0.38980618119239807, "learning_rate": 9.729718877603861e-05, "loss": 0.016370553523302078, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.01651, "step": 80, "tokens/total": 2760176, "tokens/train_per_sec_per_gpu": 27.23, "tokens/trainable": 36260} +{"epoch": 0.31640625, "grad_norm": 0.6745172739028931, "learning_rate": 9.719721959634592e-05, "loss": 0.03491232916712761, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.03553, "step": 81, "tokens/total": 2795392, "tokens/train_per_sec_per_gpu": 23.71, "tokens/trainable": 36697} +{"epoch": 0.3203125, "grad_norm": 0.3183346092700958, "learning_rate": 9.709549441810375e-05, "loss": 0.011508136987686157, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.55, "memory/max_allocated (GiB)": 39.55, "ppl": 1.01157, "step": 82, "tokens/total": 2833008, "tokens/train_per_sec_per_gpu": 27.49, "tokens/trainable": 37181} +{"epoch": 0.32421875, "grad_norm": 1.3744608163833618, "learning_rate": 9.699201747451195e-05, "loss": 0.019207235425710678, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.01939, "step": 83, "tokens/total": 2865600, "tokens/train_per_sec_per_gpu": 25.53, "tokens/trainable": 37629} +{"epoch": 0.328125, "grad_norm": 0.16014264523983002, "learning_rate": 9.688679307166854e-05, "loss": 0.0061916159465909, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00621, "step": 84, "tokens/total": 2898400, "tokens/train_per_sec_per_gpu": 28.9, "tokens/trainable": 38092} +{"epoch": 0.33203125, "grad_norm": 0.7409338355064392, "learning_rate": 9.677982558839042e-05, "loss": 0.029516642913222313, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.02996, "step": 85, "tokens/total": 2931696, "tokens/train_per_sec_per_gpu": 27.66, "tokens/trainable": 38571} +{"epoch": 0.3359375, "grad_norm": 0.30602309107780457, "learning_rate": 9.66711194760312e-05, "loss": 0.009220009669661522, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.00926, "step": 86, "tokens/total": 2969104, "tokens/train_per_sec_per_gpu": 26.07, "tokens/trainable": 39059} +{"epoch": 0.33984375, "grad_norm": 0.46287861466407776, "learning_rate": 9.656067925829593e-05, "loss": 0.01410167571157217, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.0142, "step": 87, "tokens/total": 2999856, "tokens/train_per_sec_per_gpu": 31.9, "tokens/trainable": 39519} +{"epoch": 0.34375, "grad_norm": 0.24755224585533142, "learning_rate": 9.644850953105288e-05, "loss": 0.008136758580803871, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 35.78, "memory/max_allocated (GiB)": 35.78, "ppl": 1.00817, "step": 88, "tokens/total": 3030208, "tokens/train_per_sec_per_gpu": 26.73, "tokens/trainable": 39961} +{"epoch": 0.34765625, "grad_norm": 1.1408194303512573, "learning_rate": 9.633461496214225e-05, "loss": 0.03234165534377098, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.03287, "step": 89, "tokens/total": 3062912, "tokens/train_per_sec_per_gpu": 27.06, "tokens/trainable": 40431} +{"epoch": 0.3515625, "grad_norm": 0.852928102016449, "learning_rate": 9.621900029118195e-05, "loss": 0.035508234053850174, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.04, "memory/max_allocated (GiB)": 36.04, "ppl": 1.03615, "step": 90, "tokens/total": 3092592, "tokens/train_per_sec_per_gpu": 26.15, "tokens/trainable": 40854} +{"epoch": 0.35546875, "grad_norm": 0.11538046598434448, "learning_rate": 9.610167032937036e-05, "loss": 0.0013292465591803193, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00133, "step": 91, "tokens/total": 3130880, "tokens/train_per_sec_per_gpu": 26.66, "tokens/trainable": 41373} +{"epoch": 0.359375, "grad_norm": 0.6617401838302612, "learning_rate": 9.598262995928611e-05, "loss": 0.02893206477165222, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.02935, "step": 92, "tokens/total": 3162080, "tokens/train_per_sec_per_gpu": 30.79, "tokens/trainable": 41842} +{"epoch": 0.36328125, "grad_norm": 0.4819481372833252, "learning_rate": 9.586188413468492e-05, "loss": 0.009772080928087234, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00982, "step": 93, "tokens/total": 3200272, "tokens/train_per_sec_per_gpu": 23.4, "tokens/trainable": 42288} +{"epoch": 0.3671875, "grad_norm": 0.2565758228302002, "learning_rate": 9.57394378802934e-05, "loss": 0.008706117048859596, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.00874, "step": 94, "tokens/total": 3233584, "tokens/train_per_sec_per_gpu": 28.02, "tokens/trainable": 42740} +{"epoch": 0.37109375, "grad_norm": 0.8564949631690979, "learning_rate": 9.56152962916e-05, "loss": 0.031086154282093048, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.26, "memory/max_allocated (GiB)": 36.26, "ppl": 1.03157, "step": 95, "tokens/total": 3264144, "tokens/train_per_sec_per_gpu": 27.32, "tokens/trainable": 43138} +{"epoch": 0.375, "grad_norm": 0.33518096804618835, "learning_rate": 9.548946453464296e-05, "loss": 0.008292345330119133, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.00833, "step": 96, "tokens/total": 3298240, "tokens/train_per_sec_per_gpu": 27.46, "tokens/trainable": 43623} +{"epoch": 0.37890625, "grad_norm": 0.500891923904419, "learning_rate": 9.53619478457953e-05, "loss": 0.016781670972704887, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.01692, "step": 97, "tokens/total": 3333136, "tokens/train_per_sec_per_gpu": 24.55, "tokens/trainable": 44079} +{"epoch": 0.3828125, "grad_norm": 0.40820643305778503, "learning_rate": 9.523275153154695e-05, "loss": 0.012674050405621529, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.01275, "step": 98, "tokens/total": 3365536, "tokens/train_per_sec_per_gpu": 25.74, "tokens/trainable": 44529} +{"epoch": 0.38671875, "grad_norm": 0.7298224568367004, "learning_rate": 9.51018809682839e-05, "loss": 0.027986666187644005, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.02838, "step": 99, "tokens/total": 3403200, "tokens/train_per_sec_per_gpu": 26.17, "tokens/trainable": 44976} +{"epoch": 0.390625, "grad_norm": 0.6046676635742188, "learning_rate": 9.49693416020645e-05, "loss": 0.028806906193494797, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.53, "memory/max_allocated (GiB)": 38.53, "ppl": 1.02923, "step": 100, "tokens/total": 3440944, "tokens/train_per_sec_per_gpu": 25.05, "tokens/trainable": 45429} +{"epoch": 0.39453125, "grad_norm": 0.21802762150764465, "learning_rate": 9.483513894839276e-05, "loss": 0.009221297688782215, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.00926, "step": 101, "tokens/total": 3474992, "tokens/train_per_sec_per_gpu": 25.85, "tokens/trainable": 45876} +{"epoch": 0.3984375, "grad_norm": 0.35168248414993286, "learning_rate": 9.469927859198888e-05, "loss": 0.011101609095931053, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.01116, "step": 102, "tokens/total": 3510912, "tokens/train_per_sec_per_gpu": 29.68, "tokens/trainable": 46363} +{"epoch": 0.40234375, "grad_norm": 0.34059131145477295, "learning_rate": 9.456176618655689e-05, "loss": 0.018328920006752014, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.47, "memory/max_allocated (GiB)": 36.47, "ppl": 1.0185, "step": 103, "tokens/total": 3542960, "tokens/train_per_sec_per_gpu": 28.1, "tokens/trainable": 46810} +{"epoch": 0.40625, "grad_norm": 0.25447607040405273, "learning_rate": 9.442260745454927e-05, "loss": 0.011509202420711517, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.01158, "step": 104, "tokens/total": 3578336, "tokens/train_per_sec_per_gpu": 26.44, "tokens/trainable": 47287} +{"epoch": 0.41015625, "grad_norm": 0.2630482614040375, "learning_rate": 9.428180818692884e-05, "loss": 0.012222552672028542, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.45, "memory/max_allocated (GiB)": 38.45, "ppl": 1.0123, "step": 105, "tokens/total": 3614496, "tokens/train_per_sec_per_gpu": 25.24, "tokens/trainable": 47788} +{"epoch": 0.4140625, "grad_norm": 0.4290602505207062, "learning_rate": 9.413937424292791e-05, "loss": 0.01731049083173275, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.01746, "step": 106, "tokens/total": 3653632, "tokens/train_per_sec_per_gpu": 22.1, "tokens/trainable": 48243} +{"epoch": 0.41796875, "grad_norm": 0.17030790448188782, "learning_rate": 9.399531154980424e-05, "loss": 0.006193030625581741, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 1.00621, "step": 107, "tokens/total": 3690208, "tokens/train_per_sec_per_gpu": 24.59, "tokens/trainable": 48692} +{"epoch": 0.421875, "grad_norm": 0.37133318185806274, "learning_rate": 9.384962610259455e-05, "loss": 0.009373048320412636, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.00942, "step": 108, "tokens/total": 3724864, "tokens/train_per_sec_per_gpu": 24.68, "tokens/trainable": 49142} +{"epoch": 0.42578125, "grad_norm": 0.4365437924861908, "learning_rate": 9.370232396386494e-05, "loss": 0.006701692473143339, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.22, "memory/max_allocated (GiB)": 37.22, "ppl": 1.00672, "step": 109, "tokens/total": 3757952, "tokens/train_per_sec_per_gpu": 26.93, "tokens/trainable": 49583} +{"epoch": 0.4296875, "grad_norm": 0.4275747835636139, "learning_rate": 9.355341126345868e-05, "loss": 0.00965808890759945, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.0097, "step": 110, "tokens/total": 3793232, "tokens/train_per_sec_per_gpu": 25.99, "tokens/trainable": 50055} +{"epoch": 0.43359375, "grad_norm": 0.234165221452713, "learning_rate": 9.340289419824107e-05, "loss": 0.005667372606694698, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.00568, "step": 111, "tokens/total": 3825552, "tokens/train_per_sec_per_gpu": 28.68, "tokens/trainable": 50500} +{"epoch": 0.4375, "grad_norm": 0.5043003559112549, "learning_rate": 9.325077903184159e-05, "loss": 0.004459582269191742, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00447, "step": 112, "tokens/total": 3860544, "tokens/train_per_sec_per_gpu": 24.4, "tokens/trainable": 50950} +{"epoch": 0.44140625, "grad_norm": 0.03909856453537941, "learning_rate": 9.30970720943932e-05, "loss": 0.00041493101161904633, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.15, "memory/max_allocated (GiB)": 38.15, "ppl": 1.00042, "step": 113, "tokens/total": 3895552, "tokens/train_per_sec_per_gpu": 24.97, "tokens/trainable": 51389} +{"epoch": 0.4453125, "grad_norm": 0.5978719592094421, "learning_rate": 9.2941779782269e-05, "loss": 0.011705869808793068, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.01177, "step": 114, "tokens/total": 3929152, "tokens/train_per_sec_per_gpu": 25.97, "tokens/trainable": 51862} +{"epoch": 0.44921875, "grad_norm": 0.72988361120224, "learning_rate": 9.278490855781596e-05, "loss": 0.018697654828429222, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.24, "memory/max_allocated (GiB)": 37.24, "ppl": 1.01887, "step": 115, "tokens/total": 3961472, "tokens/train_per_sec_per_gpu": 30.52, "tokens/trainable": 52301} +{"epoch": 0.453125, "grad_norm": 0.5918133854866028, "learning_rate": 9.262646494908604e-05, "loss": 0.011454951018095016, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.01152, "step": 116, "tokens/total": 4000128, "tokens/train_per_sec_per_gpu": 24.11, "tokens/trainable": 52755} +{"epoch": 0.45703125, "grad_norm": 0.5881994366645813, "learning_rate": 9.246645554956457e-05, "loss": 0.017478536814451218, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.01763, "step": 117, "tokens/total": 4035152, "tokens/train_per_sec_per_gpu": 25.52, "tokens/trainable": 53224} +{"epoch": 0.4609375, "grad_norm": 0.38013505935668945, "learning_rate": 9.230488701789578e-05, "loss": 0.0068869441747665405, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.00691, "step": 118, "tokens/total": 4072208, "tokens/train_per_sec_per_gpu": 23.93, "tokens/trainable": 53666} +{"epoch": 0.46484375, "grad_norm": 0.13333360850811005, "learning_rate": 9.214176607760577e-05, "loss": 0.003277395386248827, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.00328, "step": 119, "tokens/total": 4110464, "tokens/train_per_sec_per_gpu": 24.5, "tokens/trainable": 54129} +{"epoch": 0.46875, "grad_norm": 0.491417795419693, "learning_rate": 9.197709951682268e-05, "loss": 0.022560179233551025, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.02282, "step": 120, "tokens/total": 4142896, "tokens/train_per_sec_per_gpu": 29.54, "tokens/trainable": 54587} +{"epoch": 0.47265625, "grad_norm": 0.5677105784416199, "learning_rate": 9.181089418799428e-05, "loss": 0.029425840824842453, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.02986, "step": 121, "tokens/total": 4179360, "tokens/train_per_sec_per_gpu": 27.75, "tokens/trainable": 55070} +{"epoch": 0.4765625, "grad_norm": 0.375547856092453, "learning_rate": 9.164315700760271e-05, "loss": 0.016245262697339058, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.01638, "step": 122, "tokens/total": 4215040, "tokens/train_per_sec_per_gpu": 23.65, "tokens/trainable": 55494} +{"epoch": 0.48046875, "grad_norm": 0.28384554386138916, "learning_rate": 9.147389495587671e-05, "loss": 0.011233808472752571, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.0113, "step": 123, "tokens/total": 4246448, "tokens/train_per_sec_per_gpu": 31.36, "tokens/trainable": 55940} +{"epoch": 0.484375, "grad_norm": 0.2076566368341446, "learning_rate": 9.130311507650116e-05, "loss": 0.009959067218005657, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.01001, "step": 124, "tokens/total": 4278832, "tokens/train_per_sec_per_gpu": 27.31, "tokens/trainable": 56404} +{"epoch": 0.48828125, "grad_norm": 0.2582095265388489, "learning_rate": 9.113082447632394e-05, "loss": 0.011642523109912872, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.01171, "step": 125, "tokens/total": 4310896, "tokens/train_per_sec_per_gpu": 33.48, "tokens/trainable": 56878} +{"epoch": 0.4921875, "grad_norm": 0.21890223026275635, "learning_rate": 9.09570303250602e-05, "loss": 0.007427360862493515, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.00746, "step": 126, "tokens/total": 4342912, "tokens/train_per_sec_per_gpu": 30.89, "tokens/trainable": 57362} +{"epoch": 0.49609375, "grad_norm": 0.07802467048168182, "learning_rate": 9.078173985499394e-05, "loss": 0.002844002330675721, "memory/device_reserved (GiB)": 42.05, "memory/max_active (GiB)": 35.95, "memory/max_allocated (GiB)": 35.95, "ppl": 1.00285, "step": 127, "tokens/total": 4373312, "tokens/train_per_sec_per_gpu": 28.69, "tokens/trainable": 57792} +{"epoch": 0.5, "grad_norm": 0.28861889243125916, "learning_rate": 9.060496036067713e-05, "loss": 0.01030177902430296, "memory/device_reserved (GiB)": 42.05, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.01036, "step": 128, "tokens/total": 4410816, "tokens/train_per_sec_per_gpu": 22.68, "tokens/trainable": 58216} +{"epoch": 0.50390625, "grad_norm": 0.4341903030872345, "learning_rate": 9.042669919862615e-05, "loss": 0.01349002867937088, "memory/device_reserved (GiB)": 42.05, "memory/max_active (GiB)": 36.77, "memory/max_allocated (GiB)": 36.77, "ppl": 1.01358, "step": 129, "tokens/total": 4442704, "tokens/train_per_sec_per_gpu": 30.34, "tokens/trainable": 58680} +{"epoch": 0.5078125, "grad_norm": 0.22787874937057495, "learning_rate": 9.024696378701557e-05, "loss": 0.003789752721786499, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 39.48, "memory/max_allocated (GiB)": 39.48, "ppl": 1.0038, "step": 130, "tokens/total": 4478384, "tokens/train_per_sec_per_gpu": 24.09, "tokens/trainable": 59113} +{"epoch": 0.51171875, "grad_norm": 0.6023702025413513, "learning_rate": 9.006576160536948e-05, "loss": 0.018769558519124985, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.01895, "step": 131, "tokens/total": 4513328, "tokens/train_per_sec_per_gpu": 24.53, "tokens/trainable": 59562} +{"epoch": 0.515625, "grad_norm": 1.4625134468078613, "learning_rate": 8.988310019425035e-05, "loss": 0.010394902899861336, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.01045, "step": 132, "tokens/total": 4550832, "tokens/train_per_sec_per_gpu": 24.36, "tokens/trainable": 60006} +{"epoch": 0.51953125, "grad_norm": 0.4159144461154938, "learning_rate": 8.969898715494506e-05, "loss": 0.006678587291389704, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.0067, "step": 133, "tokens/total": 4587968, "tokens/train_per_sec_per_gpu": 25.95, "tokens/trainable": 60487} +{"epoch": 0.5234375, "grad_norm": 15.838319778442383, "learning_rate": 8.951343014914869e-05, "loss": 0.02954277954995632, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.02998, "step": 134, "tokens/total": 4624544, "tokens/train_per_sec_per_gpu": 23.75, "tokens/trainable": 60953} +{"epoch": 0.52734375, "grad_norm": 0.6038540005683899, "learning_rate": 8.932643689864568e-05, "loss": 0.01193074882030487, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.012, "step": 135, "tokens/total": 4661296, "tokens/train_per_sec_per_gpu": 27.5, "tokens/trainable": 61440} +{"epoch": 0.53125, "grad_norm": 0.3945084810256958, "learning_rate": 8.913801518498845e-05, "loss": 0.004984225146472454, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.005, "step": 136, "tokens/total": 4700128, "tokens/train_per_sec_per_gpu": 21.5, "tokens/trainable": 61901} +{"epoch": 0.53515625, "grad_norm": 0.38089239597320557, "learning_rate": 8.894817284917364e-05, "loss": 0.0119123924523592, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.01198, "step": 137, "tokens/total": 4736912, "tokens/train_per_sec_per_gpu": 22.97, "tokens/trainable": 62355} +{"epoch": 0.5390625, "grad_norm": 0.44026851654052734, "learning_rate": 8.875691779131569e-05, "loss": 0.012182775884866714, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.01226, "step": 138, "tokens/total": 4774192, "tokens/train_per_sec_per_gpu": 20.94, "tokens/trainable": 62786} +{"epoch": 0.54296875, "grad_norm": 0.035368744283914566, "learning_rate": 8.856425797031829e-05, "loss": 0.0007888172985985875, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00079, "step": 139, "tokens/total": 4809344, "tokens/train_per_sec_per_gpu": 26.0, "tokens/trainable": 63228} +{"epoch": 0.546875, "grad_norm": 0.2630954384803772, "learning_rate": 8.837020140354295e-05, "loss": 0.00839592982083559, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.00843, "step": 140, "tokens/total": 4845712, "tokens/train_per_sec_per_gpu": 27.07, "tokens/trainable": 63693} +{"epoch": 0.55078125, "grad_norm": 0.43906936049461365, "learning_rate": 8.817475616647554e-05, "loss": 0.004174365196377039, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.00418, "step": 141, "tokens/total": 4880368, "tokens/train_per_sec_per_gpu": 26.78, "tokens/trainable": 64159} +{"epoch": 0.5546875, "grad_norm": 0.09508957713842392, "learning_rate": 8.797793039239017e-05, "loss": 0.00276487460359931, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00277, "step": 142, "tokens/total": 4915424, "tokens/train_per_sec_per_gpu": 26.26, "tokens/trainable": 64608} +{"epoch": 0.55859375, "grad_norm": 0.15554048120975494, "learning_rate": 8.777973227201069e-05, "loss": 0.005121240392327309, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.00513, "step": 143, "tokens/total": 4951744, "tokens/train_per_sec_per_gpu": 28.03, "tokens/trainable": 65117} +{"epoch": 0.5625, "grad_norm": 0.3856823444366455, "learning_rate": 8.758017005316988e-05, "loss": 0.014600001275539398, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.01471, "step": 144, "tokens/total": 4985152, "tokens/train_per_sec_per_gpu": 28.51, "tokens/trainable": 65542} +{"epoch": 0.56640625, "grad_norm": 0.4464718699455261, "learning_rate": 8.737925204046629e-05, "loss": 0.01893303357064724, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.01911, "step": 145, "tokens/total": 5019936, "tokens/train_per_sec_per_gpu": 24.22, "tokens/trainable": 65992} +{"epoch": 0.5703125, "grad_norm": 0.186944842338562, "learning_rate": 8.717698659491851e-05, "loss": 0.006051606498658657, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00607, "step": 146, "tokens/total": 5057024, "tokens/train_per_sec_per_gpu": 22.56, "tokens/trainable": 66449} +{"epoch": 0.57421875, "grad_norm": 0.3332504332065582, "learning_rate": 8.697338213361735e-05, "loss": 0.009953278116881847, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.01, "step": 147, "tokens/total": 5088320, "tokens/train_per_sec_per_gpu": 26.32, "tokens/trainable": 66897} +{"epoch": 0.578125, "grad_norm": 0.16099554300308228, "learning_rate": 8.676844712937552e-05, "loss": 0.0039850580506026745, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.39, "memory/max_allocated (GiB)": 39.39, "ppl": 1.00399, "step": 148, "tokens/total": 5128064, "tokens/train_per_sec_per_gpu": 24.92, "tokens/trainable": 67369} +{"epoch": 0.58203125, "grad_norm": 0.16350099444389343, "learning_rate": 8.656219011037509e-05, "loss": 0.003052587155252695, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00306, "step": 149, "tokens/total": 5165104, "tokens/train_per_sec_per_gpu": 23.61, "tokens/trainable": 67857} +{"epoch": 0.5859375, "grad_norm": 0.29921823740005493, "learning_rate": 8.63546196598125e-05, "loss": 0.0034504360519349575, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00346, "step": 150, "tokens/total": 5202336, "tokens/train_per_sec_per_gpu": 25.25, "tokens/trainable": 68305} +{"epoch": 0.58984375, "grad_norm": 0.3413047194480896, "learning_rate": 8.614574441554145e-05, "loss": 0.012254754081368446, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.13, "memory/max_allocated (GiB)": 37.13, "ppl": 1.01233, "step": 151, "tokens/total": 5235440, "tokens/train_per_sec_per_gpu": 27.51, "tokens/trainable": 68730} +{"epoch": 0.59375, "grad_norm": 0.254293292760849, "learning_rate": 8.593557306971349e-05, "loss": 0.005805077496916056, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.44, "memory/max_allocated (GiB)": 37.44, "ppl": 1.00582, "step": 152, "tokens/total": 5268416, "tokens/train_per_sec_per_gpu": 26.1, "tokens/trainable": 69185} +{"epoch": 0.59765625, "grad_norm": 0.03500900790095329, "learning_rate": 8.572411436841618e-05, "loss": 0.00041209947085008025, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00041, "step": 153, "tokens/total": 5302128, "tokens/train_per_sec_per_gpu": 26.66, "tokens/trainable": 69660} +{"epoch": 0.6015625, "grad_norm": 0.36331841349601746, "learning_rate": 8.551137711130922e-05, "loss": 0.004842349793761969, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.2, "memory/max_allocated (GiB)": 38.2, "ppl": 1.00485, "step": 154, "tokens/total": 5333520, "tokens/train_per_sec_per_gpu": 24.01, "tokens/trainable": 70066} +{"epoch": 0.60546875, "grad_norm": 0.30322447419166565, "learning_rate": 8.529737015125824e-05, "loss": 0.004301430657505989, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00431, "step": 155, "tokens/total": 5367968, "tokens/train_per_sec_per_gpu": 25.95, "tokens/trainable": 70532} +{"epoch": 0.609375, "grad_norm": 0.4194088280200958, "learning_rate": 8.508210239396639e-05, "loss": 0.017566315829753876, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.01772, "step": 156, "tokens/total": 5405536, "tokens/train_per_sec_per_gpu": 24.34, "tokens/trainable": 70981} +{"epoch": 0.61328125, "grad_norm": 0.03025585226714611, "learning_rate": 8.486558279760375e-05, "loss": 0.00045174139086157084, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 36.07, "memory/max_allocated (GiB)": 36.07, "ppl": 1.00045, "step": 157, "tokens/total": 5436672, "tokens/train_per_sec_per_gpu": 26.81, "tokens/trainable": 71407} +{"epoch": 0.6171875, "grad_norm": 0.31769412755966187, "learning_rate": 8.464782037243449e-05, "loss": 0.008668814785778522, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.00871, "step": 158, "tokens/total": 5469280, "tokens/train_per_sec_per_gpu": 27.54, "tokens/trainable": 71848} +{"epoch": 0.62109375, "grad_norm": 0.6208626627922058, "learning_rate": 8.442882418044202e-05, "loss": 0.008168932981789112, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.0082, "step": 159, "tokens/total": 5501904, "tokens/train_per_sec_per_gpu": 29.53, "tokens/trainable": 72325} +{"epoch": 0.625, "grad_norm": 0.327109694480896, "learning_rate": 8.420860333495179e-05, "loss": 0.01694602705538273, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.01709, "step": 160, "tokens/total": 5534944, "tokens/train_per_sec_per_gpu": 26.89, "tokens/trainable": 72739} +{"epoch": 0.62890625, "grad_norm": 0.2830430865287781, "learning_rate": 8.398716700025208e-05, "loss": 0.007847893051803112, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.00788, "step": 161, "tokens/total": 5568160, "tokens/train_per_sec_per_gpu": 25.83, "tokens/trainable": 73191} +{"epoch": 0.6328125, "grad_norm": 0.7779407501220703, "learning_rate": 8.376452439121266e-05, "loss": 0.03291098773479462, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 35.89, "memory/max_allocated (GiB)": 35.89, "ppl": 1.03346, "step": 162, "tokens/total": 5598256, "tokens/train_per_sec_per_gpu": 31.35, "tokens/trainable": 73642} +{"epoch": 0.63671875, "grad_norm": 0.13102814555168152, "learning_rate": 8.354068477290124e-05, "loss": 0.003101293696090579, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00311, "step": 163, "tokens/total": 5632560, "tokens/train_per_sec_per_gpu": 28.0, "tokens/trainable": 74101} +{"epoch": 0.640625, "grad_norm": 0.14522644877433777, "learning_rate": 8.331565746019807e-05, "loss": 0.004611078184098005, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.52, "memory/max_allocated (GiB)": 36.52, "ppl": 1.00462, "step": 164, "tokens/total": 5664464, "tokens/train_per_sec_per_gpu": 27.34, "tokens/trainable": 74558} +{"epoch": 0.64453125, "grad_norm": 0.5858204364776611, "learning_rate": 8.308945181740812e-05, "loss": 0.0197240449488163, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.01992, "step": 165, "tokens/total": 5699040, "tokens/train_per_sec_per_gpu": 27.48, "tokens/trainable": 75000} +{"epoch": 0.6484375, "grad_norm": 0.24589231610298157, "learning_rate": 8.286207725787153e-05, "loss": 0.008771466091275215, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.00881, "step": 166, "tokens/total": 5735968, "tokens/train_per_sec_per_gpu": 22.65, "tokens/trainable": 75445} +{"epoch": 0.65234375, "grad_norm": 0.5058696269989014, "learning_rate": 8.263354324357182e-05, "loss": 0.01946869306266308, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.01966, "step": 167, "tokens/total": 5770688, "tokens/train_per_sec_per_gpu": 25.61, "tokens/trainable": 75880} +{"epoch": 0.65625, "grad_norm": 0.20219309628009796, "learning_rate": 8.240385928474219e-05, "loss": 0.006444904021918774, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00647, "step": 168, "tokens/total": 5803824, "tokens/train_per_sec_per_gpu": 27.31, "tokens/trainable": 76294} +{"epoch": 0.66015625, "grad_norm": 0.1849479377269745, "learning_rate": 8.217303493946967e-05, "loss": 0.007921035401523113, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.00795, "step": 169, "tokens/total": 5838256, "tokens/train_per_sec_per_gpu": 26.54, "tokens/trainable": 76742} +{"epoch": 0.6640625, "grad_norm": 0.207477405667305, "learning_rate": 8.194107981329746e-05, "loss": 0.00674483785405755, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00677, "step": 170, "tokens/total": 5876224, "tokens/train_per_sec_per_gpu": 23.58, "tokens/trainable": 77196} +{"epoch": 0.66796875, "grad_norm": 0.06854303926229477, "learning_rate": 8.170800355882518e-05, "loss": 0.001922383438795805, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.24, "memory/max_allocated (GiB)": 37.24, "ppl": 1.00192, "step": 171, "tokens/total": 5910448, "tokens/train_per_sec_per_gpu": 30.75, "tokens/trainable": 77663} +{"epoch": 0.671875, "grad_norm": 0.3741186261177063, "learning_rate": 8.147381587530713e-05, "loss": 0.010948240756988525, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.01101, "step": 172, "tokens/total": 5944256, "tokens/train_per_sec_per_gpu": 26.8, "tokens/trainable": 78117} +{"epoch": 0.67578125, "grad_norm": 0.33622583746910095, "learning_rate": 8.123852650824877e-05, "loss": 0.018746662884950638, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.01892, "step": 173, "tokens/total": 5978288, "tokens/train_per_sec_per_gpu": 28.9, "tokens/trainable": 78601} +{"epoch": 0.6796875, "grad_norm": 0.2962580621242523, "learning_rate": 8.100214524900103e-05, "loss": 0.009059429168701172, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.52, "memory/max_allocated (GiB)": 38.52, "ppl": 1.0091, "step": 174, "tokens/total": 6013680, "tokens/train_per_sec_per_gpu": 22.48, "tokens/trainable": 79032} +{"epoch": 0.68359375, "grad_norm": 0.1525028795003891, "learning_rate": 8.076468193435301e-05, "loss": 0.004882441833615303, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.00489, "step": 175, "tokens/total": 6045072, "tokens/train_per_sec_per_gpu": 26.29, "tokens/trainable": 79456} +{"epoch": 0.6875, "grad_norm": 0.09203991293907166, "learning_rate": 8.052614644612253e-05, "loss": 0.0027973109390586615, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.0028, "step": 176, "tokens/total": 6082336, "tokens/train_per_sec_per_gpu": 23.59, "tokens/trainable": 79917} +{"epoch": 0.69140625, "grad_norm": 0.25318458676338196, "learning_rate": 8.028654871074489e-05, "loss": 0.00639927014708519, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00642, "step": 177, "tokens/total": 6117024, "tokens/train_per_sec_per_gpu": 25.78, "tokens/trainable": 80358} +{"epoch": 0.6953125, "grad_norm": 0.10227160155773163, "learning_rate": 8.004589869885986e-05, "loss": 0.0016681416891515255, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.49, "memory/max_allocated (GiB)": 36.49, "ppl": 1.00167, "step": 178, "tokens/total": 6149136, "tokens/train_per_sec_per_gpu": 26.96, "tokens/trainable": 80777} +{"epoch": 0.69921875, "grad_norm": 0.719673752784729, "learning_rate": 7.980420642489674e-05, "loss": 0.017802022397518158, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.01796, "step": 179, "tokens/total": 6187776, "tokens/train_per_sec_per_gpu": 23.65, "tokens/trainable": 81250} +{"epoch": 0.703125, "grad_norm": 0.13491567969322205, "learning_rate": 7.95614819466576e-05, "loss": 0.002687457948923111, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00269, "step": 180, "tokens/total": 6221728, "tokens/train_per_sec_per_gpu": 28.65, "tokens/trainable": 81715} +{"epoch": 0.70703125, "grad_norm": 0.7875140309333801, "learning_rate": 7.931773536489872e-05, "loss": 0.02840462327003479, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.02881, "step": 181, "tokens/total": 6255920, "tokens/train_per_sec_per_gpu": 25.21, "tokens/trainable": 82160} +{"epoch": 0.7109375, "grad_norm": 0.4137536287307739, "learning_rate": 7.907297682291035e-05, "loss": 0.014628499746322632, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.01474, "step": 182, "tokens/total": 6291104, "tokens/train_per_sec_per_gpu": 26.87, "tokens/trainable": 82624} +{"epoch": 0.71484375, "grad_norm": 0.4043141305446625, "learning_rate": 7.882721650609442e-05, "loss": 0.008878855034708977, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.00892, "step": 183, "tokens/total": 6330976, "tokens/train_per_sec_per_gpu": 23.5, "tokens/trainable": 83125} +{"epoch": 0.71875, "grad_norm": 0.7706875801086426, "learning_rate": 7.85804646415409e-05, "loss": 0.02860451489686966, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.02902, "step": 184, "tokens/total": 6366160, "tokens/train_per_sec_per_gpu": 27.1, "tokens/trainable": 83581} +{"epoch": 0.72265625, "grad_norm": 0.1162385493516922, "learning_rate": 7.833273149760207e-05, "loss": 0.003058674046769738, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.00306, "step": 185, "tokens/total": 6397280, "tokens/train_per_sec_per_gpu": 29.07, "tokens/trainable": 84051} +{"epoch": 0.7265625, "grad_norm": 0.5512031316757202, "learning_rate": 7.808402738346527e-05, "loss": 0.018928907811641693, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.01911, "step": 186, "tokens/total": 6435632, "tokens/train_per_sec_per_gpu": 23.7, "tokens/trainable": 84500} +{"epoch": 0.73046875, "grad_norm": 0.13949114084243774, "learning_rate": 7.783436264872382e-05, "loss": 0.0037838718853890896, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.00379, "step": 187, "tokens/total": 6472752, "tokens/train_per_sec_per_gpu": 25.13, "tokens/trainable": 84986} +{"epoch": 0.734375, "grad_norm": 0.21509163081645966, "learning_rate": 7.758374768294647e-05, "loss": 0.008132916875183582, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.05, "memory/max_allocated (GiB)": 38.05, "ppl": 1.00817, "step": 188, "tokens/total": 6508528, "tokens/train_per_sec_per_gpu": 27.31, "tokens/trainable": 85465} +{"epoch": 0.73828125, "grad_norm": 0.264432817697525, "learning_rate": 7.733219291524489e-05, "loss": 0.010627533309161663, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.59, "memory/max_allocated (GiB)": 37.59, "ppl": 1.01068, "step": 189, "tokens/total": 6542048, "tokens/train_per_sec_per_gpu": 27.84, "tokens/trainable": 85896} +{"epoch": 0.7421875, "grad_norm": 0.2909233570098877, "learning_rate": 7.707970881383977e-05, "loss": 0.01483793556690216, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.57, "memory/max_allocated (GiB)": 36.57, "ppl": 1.01495, "step": 190, "tokens/total": 6573792, "tokens/train_per_sec_per_gpu": 28.43, "tokens/trainable": 86346} +{"epoch": 0.74609375, "grad_norm": 0.4723002314567566, "learning_rate": 7.682630588562518e-05, "loss": 0.012286916375160217, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.01236, "step": 191, "tokens/total": 6608848, "tokens/train_per_sec_per_gpu": 24.61, "tokens/trainable": 86777} +{"epoch": 0.75, "grad_norm": 0.20634613931179047, "learning_rate": 7.657199467573129e-05, "loss": 0.010780619457364082, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.01084, "step": 192, "tokens/total": 6642560, "tokens/train_per_sec_per_gpu": 26.45, "tokens/trainable": 87249} +{"epoch": 0.75390625, "grad_norm": 0.18323977291584015, "learning_rate": 7.631678576708561e-05, "loss": 0.006329106166958809, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00635, "step": 193, "tokens/total": 6676272, "tokens/train_per_sec_per_gpu": 29.34, "tokens/trainable": 87741} +{"epoch": 0.7578125, "grad_norm": 0.16646911203861237, "learning_rate": 7.606068977997255e-05, "loss": 0.006958503741770983, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00698, "step": 194, "tokens/total": 6713568, "tokens/train_per_sec_per_gpu": 25.99, "tokens/trainable": 88219} +{"epoch": 0.76171875, "grad_norm": 0.6024181246757507, "learning_rate": 7.580371737159148e-05, "loss": 0.02390211634337902, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.02419, "step": 195, "tokens/total": 6749296, "tokens/train_per_sec_per_gpu": 23.23, "tokens/trainable": 88630} +{"epoch": 0.765625, "grad_norm": 0.19809697568416595, "learning_rate": 7.554587923561324e-05, "loss": 0.004085099790245295, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00409, "step": 196, "tokens/total": 6783952, "tokens/train_per_sec_per_gpu": 24.93, "tokens/trainable": 89068} +{"epoch": 0.76953125, "grad_norm": 0.6952179670333862, "learning_rate": 7.528718610173511e-05, "loss": 0.018139110878109932, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.48, "memory/max_allocated (GiB)": 37.48, "ppl": 1.0183, "step": 197, "tokens/total": 6819152, "tokens/train_per_sec_per_gpu": 23.1, "tokens/trainable": 89489} +{"epoch": 0.7734375, "grad_norm": 0.07048558443784714, "learning_rate": 7.502764873523431e-05, "loss": 0.0014573907246813178, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.00146, "step": 198, "tokens/total": 6853440, "tokens/train_per_sec_per_gpu": 27.68, "tokens/trainable": 89928} +{"epoch": 0.77734375, "grad_norm": 0.17578265070915222, "learning_rate": 7.476727793652011e-05, "loss": 0.003963840194046497, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.71, "memory/max_allocated (GiB)": 36.71, "ppl": 1.00397, "step": 199, "tokens/total": 6886272, "tokens/train_per_sec_per_gpu": 28.47, "tokens/trainable": 90407} +{"epoch": 0.78125, "grad_norm": 0.3413652777671814, "learning_rate": 7.450608454068415e-05, "loss": 0.011968707665801048, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.01204, "step": 200, "tokens/total": 6920624, "tokens/train_per_sec_per_gpu": 27.01, "tokens/trainable": 90849} +{"epoch": 0.78515625, "grad_norm": 0.2301051914691925, "learning_rate": 7.424407941704987e-05, "loss": 0.006275743246078491, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.0063, "step": 201, "tokens/total": 6957328, "tokens/train_per_sec_per_gpu": 27.65, "tokens/trainable": 91350} +{"epoch": 0.7890625, "grad_norm": 0.7015486359596252, "learning_rate": 7.398127346871986e-05, "loss": 0.02864483930170536, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.02906, "step": 202, "tokens/total": 6994048, "tokens/train_per_sec_per_gpu": 24.68, "tokens/trainable": 91805} +{"epoch": 0.79296875, "grad_norm": 0.10557533800601959, "learning_rate": 7.371767763212238e-05, "loss": 0.0026492816396057606, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.00265, "step": 203, "tokens/total": 7029040, "tokens/train_per_sec_per_gpu": 26.85, "tokens/trainable": 92258} +{"epoch": 0.796875, "grad_norm": 0.4139856994152069, "learning_rate": 7.345330287655617e-05, "loss": 0.016636144369840622, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.01678, "step": 204, "tokens/total": 7067104, "tokens/train_per_sec_per_gpu": 22.99, "tokens/trainable": 92721} +{"epoch": 0.80078125, "grad_norm": 0.4842893183231354, "learning_rate": 7.31881602037339e-05, "loss": 0.016384506598114967, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.01652, "step": 205, "tokens/total": 7099136, "tokens/train_per_sec_per_gpu": 26.67, "tokens/trainable": 93160} +{"epoch": 0.8046875, "grad_norm": 0.24110686779022217, "learning_rate": 7.29222606473245e-05, "loss": 0.009393470361828804, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.00944, "step": 206, "tokens/total": 7132208, "tokens/train_per_sec_per_gpu": 26.8, "tokens/trainable": 93605} +{"epoch": 0.80859375, "grad_norm": 0.3036360442638397, "learning_rate": 7.265561527249383e-05, "loss": 0.009917547926306725, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.00997, "step": 207, "tokens/total": 7166256, "tokens/train_per_sec_per_gpu": 28.06, "tokens/trainable": 94041} +{"epoch": 0.8125, "grad_norm": 0.2490842491388321, "learning_rate": 7.238823517544436e-05, "loss": 0.007975293323397636, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.00801, "step": 208, "tokens/total": 7203152, "tokens/train_per_sec_per_gpu": 28.69, "tokens/trainable": 94512} +{"epoch": 0.81640625, "grad_norm": 0.09001977741718292, "learning_rate": 7.212013148295333e-05, "loss": 0.0020909998565912247, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.00209, "step": 209, "tokens/total": 7237264, "tokens/train_per_sec_per_gpu": 30.11, "tokens/trainable": 94994} +{"epoch": 0.8203125, "grad_norm": 0.2809375524520874, "learning_rate": 7.185131535190975e-05, "loss": 0.0049909064546227455, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.005, "step": 210, "tokens/total": 7269664, "tokens/train_per_sec_per_gpu": 30.39, "tokens/trainable": 95503} +{"epoch": 0.82421875, "grad_norm": 0.1591213494539261, "learning_rate": 7.158179796885005e-05, "loss": 0.004285926930606365, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.0043, "step": 211, "tokens/total": 7304080, "tokens/train_per_sec_per_gpu": 27.83, "tokens/trainable": 95954} +{"epoch": 0.828125, "grad_norm": 0.12394823133945465, "learning_rate": 7.131159054949273e-05, "loss": 0.004582830239087343, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00459, "step": 212, "tokens/total": 7342480, "tokens/train_per_sec_per_gpu": 22.61, "tokens/trainable": 96428} +{"epoch": 0.83203125, "grad_norm": 0.37879785895347595, "learning_rate": 7.104070433827139e-05, "loss": 0.014133693650364876, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.01423, "step": 213, "tokens/total": 7378032, "tokens/train_per_sec_per_gpu": 23.32, "tokens/trainable": 96828} +{"epoch": 0.8359375, "grad_norm": 0.2919084429740906, "learning_rate": 7.076915060786705e-05, "loss": 0.006862573325634003, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.96, "memory/max_allocated (GiB)": 37.96, "ppl": 1.00689, "step": 214, "tokens/total": 7410928, "tokens/train_per_sec_per_gpu": 24.56, "tokens/trainable": 97270} +{"epoch": 0.83984375, "grad_norm": 0.2852591872215271, "learning_rate": 7.049694065873882e-05, "loss": 0.013444975018501282, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.01354, "step": 215, "tokens/total": 7445664, "tokens/train_per_sec_per_gpu": 28.3, "tokens/trainable": 97727} +{"epoch": 0.84375, "grad_norm": 0.3492720127105713, "learning_rate": 7.022408581865382e-05, "loss": 0.015668835490942, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.69, "memory/max_allocated (GiB)": 38.69, "ppl": 1.01579, "step": 216, "tokens/total": 7479984, "tokens/train_per_sec_per_gpu": 25.75, "tokens/trainable": 98201} +{"epoch": 0.84765625, "grad_norm": 0.020630132406949997, "learning_rate": 6.99505974422157e-05, "loss": 0.0004898316110484302, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.00049, "step": 217, "tokens/total": 7514144, "tokens/train_per_sec_per_gpu": 28.74, "tokens/trainable": 98674} +{"epoch": 0.8515625, "grad_norm": 0.09152205288410187, "learning_rate": 6.967648691039213e-05, "loss": 0.0017674528062343597, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.73, "memory/max_allocated (GiB)": 36.73, "ppl": 1.00177, "step": 218, "tokens/total": 7546256, "tokens/train_per_sec_per_gpu": 28.29, "tokens/trainable": 99108} +{"epoch": 0.85546875, "grad_norm": 0.11286962777376175, "learning_rate": 6.940176563004123e-05, "loss": 0.0029908716678619385, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.003, "step": 219, "tokens/total": 7580432, "tokens/train_per_sec_per_gpu": 26.27, "tokens/trainable": 99540} +{"epoch": 0.859375, "grad_norm": 0.1738736778497696, "learning_rate": 6.912644503343682e-05, "loss": 0.005620469339191914, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.00564, "step": 220, "tokens/total": 7617904, "tokens/train_per_sec_per_gpu": 23.73, "tokens/trainable": 99978} +{"epoch": 0.86328125, "grad_norm": 0.30923405289649963, "learning_rate": 6.885053657779273e-05, "loss": 0.011393813416361809, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.01146, "step": 221, "tokens/total": 7649696, "tokens/train_per_sec_per_gpu": 28.74, "tokens/trainable": 100450} +{"epoch": 0.8671875, "grad_norm": 0.14390479028224945, "learning_rate": 6.857405174478604e-05, "loss": 0.0016583045944571495, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.00166, "step": 222, "tokens/total": 7684464, "tokens/train_per_sec_per_gpu": 24.62, "tokens/trainable": 100903} +{"epoch": 0.87109375, "grad_norm": 0.23503725230693817, "learning_rate": 6.82970020400792e-05, "loss": 0.013573935255408287, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.01367, "step": 223, "tokens/total": 7721936, "tokens/train_per_sec_per_gpu": 22.58, "tokens/trainable": 101351} +{"epoch": 0.875, "grad_norm": 0.256751149892807, "learning_rate": 6.801939899284132e-05, "loss": 0.00669925706461072, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.34, "memory/max_allocated (GiB)": 38.34, "ppl": 1.00672, "step": 224, "tokens/total": 7758160, "tokens/train_per_sec_per_gpu": 26.8, "tokens/trainable": 101857} +{"epoch": 0.87890625, "grad_norm": 0.12211651355028152, "learning_rate": 6.774125415526827e-05, "loss": 0.002582470653578639, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00259, "step": 225, "tokens/total": 7791216, "tokens/train_per_sec_per_gpu": 28.1, "tokens/trainable": 102328} +{"epoch": 0.8828125, "grad_norm": 0.06090810149908066, "learning_rate": 6.746257910210214e-05, "loss": 0.0013585272245109081, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.12, "memory/max_allocated (GiB)": 37.12, "ppl": 1.00136, "step": 226, "tokens/total": 7824400, "tokens/train_per_sec_per_gpu": 28.48, "tokens/trainable": 102779} +{"epoch": 0.88671875, "grad_norm": 0.1719846874475479, "learning_rate": 6.718338543014937e-05, "loss": 0.004362316802144051, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 35.75, "memory/max_allocated (GiB)": 35.75, "ppl": 1.00437, "step": 227, "tokens/total": 7854672, "tokens/train_per_sec_per_gpu": 31.23, "tokens/trainable": 103228} +{"epoch": 0.890625, "grad_norm": 0.05666934326291084, "learning_rate": 6.69036847577983e-05, "loss": 0.001159785664640367, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.00116, "step": 228, "tokens/total": 7888592, "tokens/train_per_sec_per_gpu": 27.06, "tokens/trainable": 103684} +{"epoch": 0.89453125, "grad_norm": 0.20365233719348907, "learning_rate": 6.662348872453553e-05, "loss": 0.005394858308136463, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.33, "memory/max_allocated (GiB)": 38.33, "ppl": 1.00541, "step": 229, "tokens/total": 7924848, "tokens/train_per_sec_per_gpu": 27.8, "tokens/trainable": 104145} +{"epoch": 0.8984375, "grad_norm": 0.19191378355026245, "learning_rate": 6.63428089904618e-05, "loss": 0.0022528928238898516, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.00226, "step": 230, "tokens/total": 7963856, "tokens/train_per_sec_per_gpu": 24.47, "tokens/trainable": 104598} +{"epoch": 0.90234375, "grad_norm": 0.20152772963047028, "learning_rate": 6.60616572358065e-05, "loss": 0.0021832333877682686, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.00219, "step": 231, "tokens/total": 7998320, "tokens/train_per_sec_per_gpu": 25.87, "tokens/trainable": 104998} +{"epoch": 0.90625, "grad_norm": 0.25930115580558777, "learning_rate": 6.578004516044172e-05, "loss": 0.003996685147285461, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.004, "step": 232, "tokens/total": 8034096, "tokens/train_per_sec_per_gpu": 27.84, "tokens/trainable": 105465} +{"epoch": 0.91015625, "grad_norm": 0.45257464051246643, "learning_rate": 6.549798448339548e-05, "loss": 0.01601148396730423, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.59, "memory/max_allocated (GiB)": 38.59, "ppl": 1.01614, "step": 233, "tokens/total": 8071360, "tokens/train_per_sec_per_gpu": 25.3, "tokens/trainable": 105928} +{"epoch": 0.9140625, "grad_norm": 0.02797914482653141, "learning_rate": 6.521548694236384e-05, "loss": 0.00035640608984977007, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00036, "step": 234, "tokens/total": 8108560, "tokens/train_per_sec_per_gpu": 23.58, "tokens/trainable": 106365} +{"epoch": 0.91796875, "grad_norm": 0.12456994503736496, "learning_rate": 6.493256429322259e-05, "loss": 0.0023241452872753143, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.00233, "step": 235, "tokens/total": 8144816, "tokens/train_per_sec_per_gpu": 22.43, "tokens/trainable": 106796} +{"epoch": 0.921875, "grad_norm": 0.3182932436466217, "learning_rate": 6.464922830953799e-05, "loss": 0.01117429044097662, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.01124, "step": 236, "tokens/total": 8183680, "tokens/train_per_sec_per_gpu": 22.5, "tokens/trainable": 107227} +{"epoch": 0.92578125, "grad_norm": 0.8628256320953369, "learning_rate": 6.436549078207688e-05, "loss": 0.04397165775299072, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.04495, "step": 237, "tokens/total": 8216688, "tokens/train_per_sec_per_gpu": 29.06, "tokens/trainable": 107687} +{"epoch": 0.9296875, "grad_norm": 0.5041798949241638, "learning_rate": 6.408136351831592e-05, "loss": 0.004540415946394205, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.00455, "step": 238, "tokens/total": 8249104, "tokens/train_per_sec_per_gpu": 24.38, "tokens/trainable": 108127} +{"epoch": 0.93359375, "grad_norm": 0.1622237116098404, "learning_rate": 6.379685834195036e-05, "loss": 0.0018906828481703997, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00189, "step": 239, "tokens/total": 8288528, "tokens/train_per_sec_per_gpu": 24.77, "tokens/trainable": 108633} +{"epoch": 0.9375, "grad_norm": 0.08833803981542587, "learning_rate": 6.351198709240186e-05, "loss": 0.001825703657232225, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.00183, "step": 240, "tokens/total": 8328464, "tokens/train_per_sec_per_gpu": 21.21, "tokens/trainable": 109085} +{"epoch": 0.94140625, "grad_norm": 0.21734321117401123, "learning_rate": 6.32267616243259e-05, "loss": 0.004739041440188885, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.00475, "step": 241, "tokens/total": 8361264, "tokens/train_per_sec_per_gpu": 29.02, "tokens/trainable": 109548} +{"epoch": 0.9453125, "grad_norm": 0.09377095848321915, "learning_rate": 6.294119380711849e-05, "loss": 0.0015565806534141302, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.39, "memory/max_allocated (GiB)": 38.39, "ppl": 1.00156, "step": 242, "tokens/total": 8395552, "tokens/train_per_sec_per_gpu": 24.02, "tokens/trainable": 109975} +{"epoch": 0.94921875, "grad_norm": 0.4244003891944885, "learning_rate": 6.265529552442209e-05, "loss": 0.016060730442404747, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.01619, "step": 243, "tokens/total": 8430656, "tokens/train_per_sec_per_gpu": 28.29, "tokens/trainable": 110436} +{"epoch": 0.953125, "grad_norm": 0.12588731944561005, "learning_rate": 6.236907867363127e-05, "loss": 0.0015841057756915689, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.17, "memory/max_allocated (GiB)": 38.17, "ppl": 1.00159, "step": 244, "tokens/total": 8465232, "tokens/train_per_sec_per_gpu": 26.86, "tokens/trainable": 110882} +{"epoch": 0.95703125, "grad_norm": 0.20038087666034698, "learning_rate": 6.208255516539749e-05, "loss": 0.003118037013337016, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.00312, "step": 245, "tokens/total": 8497552, "tokens/train_per_sec_per_gpu": 26.82, "tokens/trainable": 111355} +{"epoch": 0.9609375, "grad_norm": 0.14364290237426758, "learning_rate": 6.179573692313344e-05, "loss": 0.0025180529337376356, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.8, "memory/max_allocated (GiB)": 38.8, "ppl": 1.00252, "step": 246, "tokens/total": 8534336, "tokens/train_per_sec_per_gpu": 26.25, "tokens/trainable": 111803} +{"epoch": 0.96484375, "grad_norm": 0.12907195091247559, "learning_rate": 6.150863588251694e-05, "loss": 0.00408803578466177, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.0041, "step": 247, "tokens/total": 8574224, "tokens/train_per_sec_per_gpu": 22.86, "tokens/trainable": 112270} +{"epoch": 0.96875, "grad_norm": 0.6586377024650574, "learning_rate": 6.122126399099419e-05, "loss": 0.011401331052184105, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.01147, "step": 248, "tokens/total": 8613616, "tokens/train_per_sec_per_gpu": 27.1, "tokens/trainable": 112778} +{"epoch": 0.97265625, "grad_norm": 0.6748576164245605, "learning_rate": 6.0933633207282615e-05, "loss": 0.012606333941221237, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.01269, "step": 249, "tokens/total": 8650048, "tokens/train_per_sec_per_gpu": 27.6, "tokens/trainable": 113284} +{"epoch": 0.9765625, "grad_norm": 0.14774703979492188, "learning_rate": 6.064575550087316e-05, "loss": 0.005306069739162922, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00532, "step": 250, "tokens/total": 8686688, "tokens/train_per_sec_per_gpu": 23.91, "tokens/trainable": 113722} +{"epoch": 0.98046875, "grad_norm": 0.3517683446407318, "learning_rate": 6.0357642851532245e-05, "loss": 0.008184859529137611, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00822, "step": 251, "tokens/total": 8723376, "tokens/train_per_sec_per_gpu": 25.99, "tokens/trainable": 114213} +{"epoch": 0.984375, "grad_norm": 0.4393784701824188, "learning_rate": 6.0069307248803294e-05, "loss": 0.013888435438275337, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.01399, "step": 252, "tokens/total": 8758736, "tokens/train_per_sec_per_gpu": 22.11, "tokens/trainable": 114634} +{"epoch": 0.98828125, "grad_norm": 0.28601863980293274, "learning_rate": 5.9780760691507635e-05, "loss": 0.006867052521556616, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.33, "memory/max_allocated (GiB)": 36.33, "ppl": 1.00689, "step": 253, "tokens/total": 8790480, "tokens/train_per_sec_per_gpu": 26.95, "tokens/trainable": 115066} +{"epoch": 0.9921875, "grad_norm": 0.05788040533661842, "learning_rate": 5.9492015187245334e-05, "loss": 0.0012266368139535189, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00123, "step": 254, "tokens/total": 8828224, "tokens/train_per_sec_per_gpu": 23.47, "tokens/trainable": 115520} +{"epoch": 0.99609375, "grad_norm": 0.2933010458946228, "learning_rate": 5.920308275189541e-05, "loss": 0.0051932320930063725, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.00521, "step": 255, "tokens/total": 8860400, "tokens/train_per_sec_per_gpu": 25.74, "tokens/trainable": 115921} +{"epoch": 1.0, "grad_norm": 0.21706163883209229, "learning_rate": 5.8913975409115874e-05, "loss": 0.008472070097923279, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.00851, "step": 256, "tokens/total": 8893760, "tokens/train_per_sec_per_gpu": 25.59, "tokens/trainable": 116384} +{"epoch": 1.00390625, "grad_norm": 0.2550739049911499, "learning_rate": 5.8624705189843395e-05, "loss": 0.007156818639487028, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 1.00718, "step": 257, "tokens/total": 8930976, "tokens/train_per_sec_per_gpu": 24.78, "tokens/trainable": 116871} +{"epoch": 1.0078125, "grad_norm": 0.15252681076526642, "learning_rate": 5.833528413179249e-05, "loss": 0.0022481554187834263, "memory/device_reserved (GiB)": 42.25, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00225, "step": 258, "tokens/total": 8967472, "tokens/train_per_sec_per_gpu": 28.02, "tokens/trainable": 117369} +{"epoch": 1.01171875, "grad_norm": 0.24202542006969452, "learning_rate": 5.80457242789548e-05, "loss": 0.006266828626394272, "memory/device_reserved (GiB)": 42.31, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.00629, "step": 259, "tokens/total": 9007456, "tokens/train_per_sec_per_gpu": 23.99, "tokens/trainable": 117866} +{"epoch": 1.015625, "grad_norm": 0.17992764711380005, "learning_rate": 5.77560376810977e-05, "loss": 0.0027958517894148827, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 1.0028, "step": 260, "tokens/total": 9041104, "tokens/train_per_sec_per_gpu": 29.43, "tokens/trainable": 118330} +{"epoch": 1.01953125, "grad_norm": 0.45435965061187744, "learning_rate": 5.7466236393263005e-05, "loss": 0.001130607444792986, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.00113, "step": 261, "tokens/total": 9076176, "tokens/train_per_sec_per_gpu": 25.72, "tokens/trainable": 118776} +{"epoch": 1.0234375, "grad_norm": 0.09589112550020218, "learning_rate": 5.717633247526522e-05, "loss": 0.0024520030710846186, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00246, "step": 262, "tokens/total": 9107472, "tokens/train_per_sec_per_gpu": 27.91, "tokens/trainable": 119211} +{"epoch": 1.02734375, "grad_norm": 0.12382327765226364, "learning_rate": 5.688633799118971e-05, "loss": 0.002382563892751932, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.00239, "step": 263, "tokens/total": 9139552, "tokens/train_per_sec_per_gpu": 23.59, "tokens/trainable": 119616} +{"epoch": 1.03125, "grad_norm": 0.19539223611354828, "learning_rate": 5.659626500889066e-05, "loss": 0.0034907683730125427, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.0035, "step": 264, "tokens/total": 9173424, "tokens/train_per_sec_per_gpu": 26.01, "tokens/trainable": 120086} +{"epoch": 1.03515625, "grad_norm": 0.0883532464504242, "learning_rate": 5.6306125599488905e-05, "loss": 0.0008758740150369704, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.00088, "step": 265, "tokens/total": 9203984, "tokens/train_per_sec_per_gpu": 32.4, "tokens/trainable": 120525} +{"epoch": 1.0390625, "grad_norm": 0.10653501003980637, "learning_rate": 5.601593183686955e-05, "loss": 0.002780267270281911, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.69, "memory/max_allocated (GiB)": 38.69, "ppl": 1.00278, "step": 266, "tokens/total": 9241248, "tokens/train_per_sec_per_gpu": 26.15, "tokens/trainable": 121001} +{"epoch": 1.04296875, "grad_norm": 0.23462294042110443, "learning_rate": 5.572569579717961e-05, "loss": 0.004946558736264706, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.18, "memory/max_allocated (GiB)": 39.18, "ppl": 1.00496, "step": 267, "tokens/total": 9276768, "tokens/train_per_sec_per_gpu": 24.69, "tokens/trainable": 121465} +{"epoch": 1.046875, "grad_norm": 0.04873845353722572, "learning_rate": 5.543542955832538e-05, "loss": 0.0003125692601315677, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00031, "step": 268, "tokens/total": 9316528, "tokens/train_per_sec_per_gpu": 21.84, "tokens/trainable": 121908} +{"epoch": 1.05078125, "grad_norm": 0.29227694869041443, "learning_rate": 5.514514519946986e-05, "loss": 0.003915291279554367, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.00392, "step": 269, "tokens/total": 9356352, "tokens/train_per_sec_per_gpu": 25.9, "tokens/trainable": 122406} +{"epoch": 1.0546875, "grad_norm": 0.17162470519542694, "learning_rate": 5.485485480053015e-05, "loss": 0.0016307509504258633, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.00163, "step": 270, "tokens/total": 9394576, "tokens/train_per_sec_per_gpu": 22.05, "tokens/trainable": 122812} +{"epoch": 1.05859375, "grad_norm": 0.6890908479690552, "learning_rate": 5.4564570441674645e-05, "loss": 0.008029540069401264, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.92, "memory/max_allocated (GiB)": 36.92, "ppl": 1.00806, "step": 271, "tokens/total": 9427840, "tokens/train_per_sec_per_gpu": 25.71, "tokens/trainable": 123252} +{"epoch": 1.0625, "grad_norm": 0.2048344761133194, "learning_rate": 5.42743042028204e-05, "loss": 0.002246133517473936, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.00225, "step": 272, "tokens/total": 9463536, "tokens/train_per_sec_per_gpu": 20.78, "tokens/trainable": 123692} +{"epoch": 1.06640625, "grad_norm": 0.33452561497688293, "learning_rate": 5.3984068163130464e-05, "loss": 0.005725300870835781, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.09, "memory/max_allocated (GiB)": 37.09, "ppl": 1.00574, "step": 273, "tokens/total": 9497248, "tokens/train_per_sec_per_gpu": 28.36, "tokens/trainable": 124162} +{"epoch": 1.0703125, "grad_norm": 0.2757226228713989, "learning_rate": 5.369387440051111e-05, "loss": 0.006774573586881161, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.0068, "step": 274, "tokens/total": 9530784, "tokens/train_per_sec_per_gpu": 25.62, "tokens/trainable": 124625} +{"epoch": 1.07421875, "grad_norm": 0.0022927923128008842, "learning_rate": 5.340373499110935e-05, "loss": 5.248050365480594e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00005, "step": 275, "tokens/total": 9566656, "tokens/train_per_sec_per_gpu": 26.71, "tokens/trainable": 125119} +{"epoch": 1.078125, "grad_norm": 0.01345912367105484, "learning_rate": 5.3113662008810304e-05, "loss": 0.00020729006791952997, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00021, "step": 276, "tokens/total": 9606672, "tokens/train_per_sec_per_gpu": 25.2, "tokens/trainable": 125601} +{"epoch": 1.08203125, "grad_norm": 0.03346296027302742, "learning_rate": 5.282366752473479e-05, "loss": 0.00029993191128596663, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 1.0003, "step": 277, "tokens/total": 9643680, "tokens/train_per_sec_per_gpu": 24.7, "tokens/trainable": 126075} +{"epoch": 1.0859375, "grad_norm": 0.05970214307308197, "learning_rate": 5.2533763606737005e-05, "loss": 0.0003895902482327074, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00039, "step": 278, "tokens/total": 9683200, "tokens/train_per_sec_per_gpu": 19.52, "tokens/trainable": 126491} +{"epoch": 1.08984375, "grad_norm": 0.02426113933324814, "learning_rate": 5.224396231890232e-05, "loss": 0.00023556684027425945, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.00024, "step": 279, "tokens/total": 9714672, "tokens/train_per_sec_per_gpu": 30.34, "tokens/trainable": 126940} +{"epoch": 1.09375, "grad_norm": 0.4579313397407532, "learning_rate": 5.195427572104522e-05, "loss": 0.004314976278692484, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00432, "step": 280, "tokens/total": 9749104, "tokens/train_per_sec_per_gpu": 23.98, "tokens/trainable": 127376} +{"epoch": 1.09765625, "grad_norm": 0.01172641571611166, "learning_rate": 5.166471586820751e-05, "loss": 0.0001057953923009336, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.35, "memory/max_allocated (GiB)": 39.35, "ppl": 1.00011, "step": 281, "tokens/total": 9785712, "tokens/train_per_sec_per_gpu": 24.02, "tokens/trainable": 127841} +{"epoch": 1.1015625, "grad_norm": 0.3204192519187927, "learning_rate": 5.1375294810156615e-05, "loss": 0.011134465225040913, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.0112, "step": 282, "tokens/total": 9820544, "tokens/train_per_sec_per_gpu": 26.15, "tokens/trainable": 128288} +{"epoch": 1.10546875, "grad_norm": 0.006660244893282652, "learning_rate": 5.1086024590884144e-05, "loss": 0.00011100003030151129, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00011, "step": 283, "tokens/total": 9857952, "tokens/train_per_sec_per_gpu": 25.25, "tokens/trainable": 128737} +{"epoch": 1.109375, "grad_norm": 0.1843951940536499, "learning_rate": 5.079691724810461e-05, "loss": 0.0013875709846615791, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.33, "memory/max_allocated (GiB)": 38.33, "ppl": 1.00139, "step": 284, "tokens/total": 9894128, "tokens/train_per_sec_per_gpu": 21.99, "tokens/trainable": 129172} +{"epoch": 1.11328125, "grad_norm": 0.07134410738945007, "learning_rate": 5.0507984812754684e-05, "loss": 0.0005857737269252539, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.17, "memory/max_allocated (GiB)": 37.17, "ppl": 1.00059, "step": 285, "tokens/total": 9927568, "tokens/train_per_sec_per_gpu": 29.07, "tokens/trainable": 129669} +{"epoch": 1.1171875, "grad_norm": 0.2542164623737335, "learning_rate": 5.021923930849237e-05, "loss": 0.0036372989416122437, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.00364, "step": 286, "tokens/total": 9961888, "tokens/train_per_sec_per_gpu": 28.27, "tokens/trainable": 130155} +{"epoch": 1.12109375, "grad_norm": 0.013902081176638603, "learning_rate": 4.99306927511967e-05, "loss": 0.0001379086315864697, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.31, "memory/max_allocated (GiB)": 37.31, "ppl": 1.00014, "step": 287, "tokens/total": 9995696, "tokens/train_per_sec_per_gpu": 28.19, "tokens/trainable": 130595} +{"epoch": 1.125, "grad_norm": 0.18814300000667572, "learning_rate": 4.964235714846775e-05, "loss": 0.0023441596422344446, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.72, "memory/max_allocated (GiB)": 36.72, "ppl": 1.00235, "step": 288, "tokens/total": 10028000, "tokens/train_per_sec_per_gpu": 27.47, "tokens/trainable": 131015} +{"epoch": 1.12890625, "grad_norm": 0.002427003812044859, "learning_rate": 4.9354244499126866e-05, "loss": 4.867167444899678e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.16, "memory/max_allocated (GiB)": 36.16, "ppl": 1.00005, "step": 289, "tokens/total": 10059728, "tokens/train_per_sec_per_gpu": 29.7, "tokens/trainable": 131481} +{"epoch": 1.1328125, "grad_norm": 0.11797486245632172, "learning_rate": 4.90663667927174e-05, "loss": 0.001414420548826456, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.00142, "step": 290, "tokens/total": 10092576, "tokens/train_per_sec_per_gpu": 29.26, "tokens/trainable": 131930} +{"epoch": 1.13671875, "grad_norm": 0.003663543611764908, "learning_rate": 4.877873600900581e-05, "loss": 6.615153688471764e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.00007, "step": 291, "tokens/total": 10128608, "tokens/train_per_sec_per_gpu": 21.93, "tokens/trainable": 132388} +{"epoch": 1.140625, "grad_norm": 0.015041554346680641, "learning_rate": 4.849136411748306e-05, "loss": 0.00019204874115530401, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.00019, "step": 292, "tokens/total": 10166192, "tokens/train_per_sec_per_gpu": 25.2, "tokens/trainable": 132860} +{"epoch": 1.14453125, "grad_norm": 0.16559572517871857, "learning_rate": 4.8204263076866574e-05, "loss": 0.006702376529574394, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00672, "step": 293, "tokens/total": 10201840, "tokens/train_per_sec_per_gpu": 29.99, "tokens/trainable": 133360} +{"epoch": 1.1484375, "grad_norm": 0.17392498254776, "learning_rate": 4.791744483460251e-05, "loss": 0.0013287472538650036, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00133, "step": 294, "tokens/total": 10236048, "tokens/train_per_sec_per_gpu": 24.53, "tokens/trainable": 133793} +{"epoch": 1.15234375, "grad_norm": 0.0026634999085217714, "learning_rate": 4.7630921326368736e-05, "loss": 5.9752677771029994e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.00006, "step": 295, "tokens/total": 10274688, "tokens/train_per_sec_per_gpu": 20.27, "tokens/trainable": 134218} +{"epoch": 1.15625, "grad_norm": 0.029364775866270065, "learning_rate": 4.7344704475577916e-05, "loss": 0.0002793333260342479, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 35.66, "memory/max_allocated (GiB)": 35.66, "ppl": 1.00028, "step": 296, "tokens/total": 10304576, "tokens/train_per_sec_per_gpu": 30.67, "tokens/trainable": 134654} +{"epoch": 1.16015625, "grad_norm": 0.523063600063324, "learning_rate": 4.705880619288153e-05, "loss": 0.0035161436535418034, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.21, "memory/max_allocated (GiB)": 38.21, "ppl": 1.00352, "step": 297, "tokens/total": 10340720, "tokens/train_per_sec_per_gpu": 22.28, "tokens/trainable": 135056} +{"epoch": 1.1640625, "grad_norm": 0.3645828068256378, "learning_rate": 4.677323837567412e-05, "loss": 0.0034707190934568644, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00348, "step": 298, "tokens/total": 10373376, "tokens/train_per_sec_per_gpu": 25.67, "tokens/trainable": 135483} +{"epoch": 1.16796875, "grad_norm": 0.004196534398943186, "learning_rate": 4.6488012907598146e-05, "loss": 6.752751505700871e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00007, "step": 299, "tokens/total": 10406128, "tokens/train_per_sec_per_gpu": 28.34, "tokens/trainable": 135957} +{"epoch": 1.171875, "grad_norm": 0.022116737440228462, "learning_rate": 4.620314165804964e-05, "loss": 0.0002539573470130563, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.3, "memory/max_allocated (GiB)": 38.3, "ppl": 1.00025, "step": 300, "tokens/total": 10441792, "tokens/train_per_sec_per_gpu": 26.01, "tokens/trainable": 136430} +{"epoch": 1.17578125, "grad_norm": 0.4474564790725708, "learning_rate": 4.591863648168407e-05, "loss": 0.00526253180578351, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00528, "step": 301, "tokens/total": 10476128, "tokens/train_per_sec_per_gpu": 26.69, "tokens/trainable": 136896} +{"epoch": 1.1796875, "grad_norm": 0.07129482924938202, "learning_rate": 4.5634509217923135e-05, "loss": 0.0008131759241223335, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00081, "step": 302, "tokens/total": 10509472, "tokens/train_per_sec_per_gpu": 26.12, "tokens/trainable": 137331} +{"epoch": 1.18359375, "grad_norm": 1.5436980724334717, "learning_rate": 4.535077169046201e-05, "loss": 0.0022148056887090206, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00222, "step": 303, "tokens/total": 10546032, "tokens/train_per_sec_per_gpu": 25.45, "tokens/trainable": 137768} +{"epoch": 1.1875, "grad_norm": 0.4574410319328308, "learning_rate": 4.506743570677743e-05, "loss": 0.005447423551231623, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.00546, "step": 304, "tokens/total": 10576624, "tokens/train_per_sec_per_gpu": 31.75, "tokens/trainable": 138200} +{"epoch": 1.19140625, "grad_norm": 0.08121448755264282, "learning_rate": 4.478451305763618e-05, "loss": 0.0009549845126457512, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.95, "memory/max_allocated (GiB)": 36.95, "ppl": 1.00096, "step": 305, "tokens/total": 10609760, "tokens/train_per_sec_per_gpu": 30.96, "tokens/trainable": 138660} +{"epoch": 1.1953125, "grad_norm": 0.07748699188232422, "learning_rate": 4.450201551660454e-05, "loss": 0.0007151119061745703, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.51, "memory/max_allocated (GiB)": 36.51, "ppl": 1.00072, "step": 306, "tokens/total": 10641472, "tokens/train_per_sec_per_gpu": 26.6, "tokens/trainable": 139092} +{"epoch": 1.19921875, "grad_norm": 0.39893022179603577, "learning_rate": 4.4219954839558276e-05, "loss": 0.010442938655614853, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.0105, "step": 307, "tokens/total": 10676160, "tokens/train_per_sec_per_gpu": 25.69, "tokens/trainable": 139533} +{"epoch": 1.203125, "grad_norm": 0.3132190406322479, "learning_rate": 4.393834276419352e-05, "loss": 0.003176536178216338, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.83, "memory/max_allocated (GiB)": 35.83, "ppl": 1.00318, "step": 308, "tokens/total": 10706384, "tokens/train_per_sec_per_gpu": 32.76, "tokens/trainable": 140020} +{"epoch": 1.20703125, "grad_norm": 0.6307603716850281, "learning_rate": 4.36571910095382e-05, "loss": 0.004229922778904438, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.00424, "step": 309, "tokens/total": 10744496, "tokens/train_per_sec_per_gpu": 25.77, "tokens/trainable": 140465} +{"epoch": 1.2109375, "grad_norm": 0.09459506720304489, "learning_rate": 4.337651127546448e-05, "loss": 0.0009815557859838009, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00098, "step": 310, "tokens/total": 10780176, "tokens/train_per_sec_per_gpu": 27.88, "tokens/trainable": 140915} +{"epoch": 1.21484375, "grad_norm": 0.029859120026230812, "learning_rate": 4.3096315242201736e-05, "loss": 0.00022185369743965566, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.61, "memory/max_allocated (GiB)": 36.61, "ppl": 1.00022, "step": 311, "tokens/total": 10813136, "tokens/train_per_sec_per_gpu": 26.6, "tokens/trainable": 141350} +{"epoch": 1.21875, "grad_norm": 0.008029108867049217, "learning_rate": 4.2816614569850635e-05, "loss": 0.00011434729094617069, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00011, "step": 312, "tokens/total": 10848912, "tokens/train_per_sec_per_gpu": 24.62, "tokens/trainable": 141789} +{"epoch": 1.22265625, "grad_norm": 0.2866860628128052, "learning_rate": 4.2537420897897864e-05, "loss": 0.00429800758138299, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.00431, "step": 313, "tokens/total": 10884320, "tokens/train_per_sec_per_gpu": 27.37, "tokens/trainable": 142266} +{"epoch": 1.2265625, "grad_norm": 0.022199038416147232, "learning_rate": 4.225874584473174e-05, "loss": 0.0002912393829319626, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.00029, "step": 314, "tokens/total": 10917152, "tokens/train_per_sec_per_gpu": 26.31, "tokens/trainable": 142704} +{"epoch": 1.23046875, "grad_norm": 0.1639176607131958, "learning_rate": 4.19806010071587e-05, "loss": 0.0015631124842911959, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.00156, "step": 315, "tokens/total": 10955024, "tokens/train_per_sec_per_gpu": 24.71, "tokens/trainable": 143168} +{"epoch": 1.234375, "grad_norm": 0.174128919839859, "learning_rate": 4.170299795992081e-05, "loss": 0.001598567352630198, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.0016, "step": 316, "tokens/total": 10988416, "tokens/train_per_sec_per_gpu": 27.37, "tokens/trainable": 143615} +{"epoch": 1.23828125, "grad_norm": 0.11350835859775543, "learning_rate": 4.142594825521398e-05, "loss": 0.0017983278958126903, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.0018, "step": 317, "tokens/total": 11027712, "tokens/train_per_sec_per_gpu": 26.28, "tokens/trainable": 144096} +{"epoch": 1.2421875, "grad_norm": 0.020274458453059196, "learning_rate": 4.114946342220728e-05, "loss": 0.00025956390891224146, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.00026, "step": 318, "tokens/total": 11063920, "tokens/train_per_sec_per_gpu": 24.21, "tokens/trainable": 144543} +{"epoch": 1.24609375, "grad_norm": 0.16120970249176025, "learning_rate": 4.087355496656321e-05, "loss": 0.0019062783103436232, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00191, "step": 319, "tokens/total": 11097952, "tokens/train_per_sec_per_gpu": 28.86, "tokens/trainable": 145016} +{"epoch": 1.25, "grad_norm": 0.620114803314209, "learning_rate": 4.05982343699588e-05, "loss": 0.012367453426122665, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.01244, "step": 320, "tokens/total": 11137168, "tokens/train_per_sec_per_gpu": 25.51, "tokens/trainable": 145511} +{"epoch": 1.25390625, "grad_norm": 0.024067921563982964, "learning_rate": 4.0323513089607876e-05, "loss": 0.0002486950543243438, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.00025, "step": 321, "tokens/total": 11172224, "tokens/train_per_sec_per_gpu": 24.58, "tokens/trainable": 145978} +{"epoch": 1.2578125, "grad_norm": 0.15742127597332, "learning_rate": 4.004940255778431e-05, "loss": 0.00304510653950274, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00305, "step": 322, "tokens/total": 11210832, "tokens/train_per_sec_per_gpu": 24.21, "tokens/trainable": 146422} +{"epoch": 1.26171875, "grad_norm": 0.03611810505390167, "learning_rate": 3.977591418134619e-05, "loss": 0.0005555619718506932, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.00056, "step": 323, "tokens/total": 11245984, "tokens/train_per_sec_per_gpu": 26.27, "tokens/trainable": 146863} +{"epoch": 1.265625, "grad_norm": 0.10045048594474792, "learning_rate": 3.95030593412612e-05, "loss": 0.0022798755671828985, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.00228, "step": 324, "tokens/total": 11281360, "tokens/train_per_sec_per_gpu": 24.89, "tokens/trainable": 147286} +{"epoch": 1.26953125, "grad_norm": 0.22728759050369263, "learning_rate": 3.923084939213296e-05, "loss": 0.002287969458848238, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.88, "memory/max_allocated (GiB)": 35.88, "ppl": 1.00229, "step": 325, "tokens/total": 11312176, "tokens/train_per_sec_per_gpu": 28.08, "tokens/trainable": 147708} +{"epoch": 1.2734375, "grad_norm": 0.14370062947273254, "learning_rate": 3.895929566172861e-05, "loss": 0.0018756273202598095, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.00188, "step": 326, "tokens/total": 11348192, "tokens/train_per_sec_per_gpu": 25.51, "tokens/trainable": 148172} +{"epoch": 1.27734375, "grad_norm": 0.5271180868148804, "learning_rate": 3.868840945050728e-05, "loss": 0.010484442114830017, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.05, "memory/max_allocated (GiB)": 39.05, "ppl": 1.01054, "step": 327, "tokens/total": 11386704, "tokens/train_per_sec_per_gpu": 21.81, "tokens/trainable": 148604} +{"epoch": 1.28125, "grad_norm": 0.0393427237868309, "learning_rate": 3.841820203114995e-05, "loss": 0.0002005074784392491, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.0002, "step": 328, "tokens/total": 11418448, "tokens/train_per_sec_per_gpu": 29.5, "tokens/trainable": 149051} +{"epoch": 1.28515625, "grad_norm": 0.050891146063804626, "learning_rate": 3.814868464809027e-05, "loss": 0.0004916652687825263, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.00049, "step": 329, "tokens/total": 11450304, "tokens/train_per_sec_per_gpu": 26.16, "tokens/trainable": 149453} +{"epoch": 1.2890625, "grad_norm": 0.5734721422195435, "learning_rate": 3.787986851704667e-05, "loss": 0.008043204434216022, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.00808, "step": 330, "tokens/total": 11484048, "tokens/train_per_sec_per_gpu": 26.02, "tokens/trainable": 149883} +{"epoch": 1.29296875, "grad_norm": 0.01727970875799656, "learning_rate": 3.7611764824555654e-05, "loss": 0.00032211365760304034, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00032, "step": 331, "tokens/total": 11516032, "tokens/train_per_sec_per_gpu": 31.36, "tokens/trainable": 150357} +{"epoch": 1.296875, "grad_norm": 0.06230725347995758, "learning_rate": 3.734438472750619e-05, "loss": 0.0007744263857603073, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.00077, "step": 332, "tokens/total": 11555968, "tokens/train_per_sec_per_gpu": 23.43, "tokens/trainable": 150828} +{"epoch": 1.30078125, "grad_norm": 0.02598424069583416, "learning_rate": 3.707773935267552e-05, "loss": 0.0002701167541090399, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.46, "memory/max_allocated (GiB)": 38.46, "ppl": 1.00027, "step": 333, "tokens/total": 11592880, "tokens/train_per_sec_per_gpu": 27.24, "tokens/trainable": 151314} +{"epoch": 1.3046875, "grad_norm": 0.011625629849731922, "learning_rate": 3.68118397962661e-05, "loss": 0.00022004164929967374, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.00022, "step": 334, "tokens/total": 11627792, "tokens/train_per_sec_per_gpu": 23.58, "tokens/trainable": 151722} +{"epoch": 1.30859375, "grad_norm": 0.03982725739479065, "learning_rate": 3.654669712344384e-05, "loss": 0.0005912472843192518, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 1.00059, "step": 335, "tokens/total": 11658832, "tokens/train_per_sec_per_gpu": 32.13, "tokens/trainable": 152178} +{"epoch": 1.3125, "grad_norm": 0.00262474431656301, "learning_rate": 3.628232236787763e-05, "loss": 4.5215507270768285e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.96, "memory/max_allocated (GiB)": 37.96, "ppl": 1.00005, "step": 336, "tokens/total": 11694464, "tokens/train_per_sec_per_gpu": 22.96, "tokens/trainable": 152617} +{"epoch": 1.31640625, "grad_norm": 0.07218950986862183, "learning_rate": 3.6018726531280144e-05, "loss": 0.00045364024117588997, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00045, "step": 337, "tokens/total": 11732176, "tokens/train_per_sec_per_gpu": 28.67, "tokens/trainable": 153108} +{"epoch": 1.3203125, "grad_norm": 0.23905178904533386, "learning_rate": 3.575592058295017e-05, "loss": 0.003075775457546115, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.00308, "step": 338, "tokens/total": 11766032, "tokens/train_per_sec_per_gpu": 28.96, "tokens/trainable": 153589} +{"epoch": 1.32421875, "grad_norm": 0.009214645251631737, "learning_rate": 3.549391545931585e-05, "loss": 6.700362428091466e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00007, "step": 339, "tokens/total": 11800640, "tokens/train_per_sec_per_gpu": 28.1, "tokens/trainable": 154049} +{"epoch": 1.328125, "grad_norm": 0.021615060046315193, "learning_rate": 3.5232722063479914e-05, "loss": 0.00022907440143171698, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.00023, "step": 340, "tokens/total": 11834352, "tokens/train_per_sec_per_gpu": 24.47, "tokens/trainable": 154498} +{"epoch": 1.33203125, "grad_norm": 0.011503382585942745, "learning_rate": 3.49723512647657e-05, "loss": 0.00020055499044246972, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.0002, "step": 341, "tokens/total": 11873056, "tokens/train_per_sec_per_gpu": 25.11, "tokens/trainable": 154998} +{"epoch": 1.3359375, "grad_norm": 0.14886975288391113, "learning_rate": 3.471281389826491e-05, "loss": 0.0021605361253023148, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.00216, "step": 342, "tokens/total": 11904192, "tokens/train_per_sec_per_gpu": 26.35, "tokens/trainable": 155403} +{"epoch": 1.33984375, "grad_norm": 0.05687630549073219, "learning_rate": 3.4454120764386764e-05, "loss": 0.0005302417557686567, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.66, "memory/max_allocated (GiB)": 38.66, "ppl": 1.00053, "step": 343, "tokens/total": 11942384, "tokens/train_per_sec_per_gpu": 24.64, "tokens/trainable": 155858} +{"epoch": 1.34375, "grad_norm": 0.43057695031166077, "learning_rate": 3.4196282628408526e-05, "loss": 0.009709179401397705, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.00976, "step": 344, "tokens/total": 11976624, "tokens/train_per_sec_per_gpu": 25.68, "tokens/trainable": 156303} +{"epoch": 1.34765625, "grad_norm": 0.014733171090483665, "learning_rate": 3.3939310220027456e-05, "loss": 0.0001609918981557712, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00016, "step": 345, "tokens/total": 12010768, "tokens/train_per_sec_per_gpu": 29.22, "tokens/trainable": 156798} +{"epoch": 1.3515625, "grad_norm": 0.0028556296601891518, "learning_rate": 3.3683214232914404e-05, "loss": 5.962311115581542e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.00006, "step": 346, "tokens/total": 12046224, "tokens/train_per_sec_per_gpu": 26.52, "tokens/trainable": 157271} +{"epoch": 1.35546875, "grad_norm": 0.03930514678359032, "learning_rate": 3.342800532426873e-05, "loss": 0.0002837421197909862, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.00028, "step": 347, "tokens/total": 12082160, "tokens/train_per_sec_per_gpu": 25.77, "tokens/trainable": 157737} +{"epoch": 1.359375, "grad_norm": 0.3733222782611847, "learning_rate": 3.317369411437484e-05, "loss": 0.0040243323892354965, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.00403, "step": 348, "tokens/total": 12118128, "tokens/train_per_sec_per_gpu": 26.49, "tokens/trainable": 158205} +{"epoch": 1.36328125, "grad_norm": 0.007541185710579157, "learning_rate": 3.292029118616024e-05, "loss": 0.00010774863767437637, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.00011, "step": 349, "tokens/total": 12151264, "tokens/train_per_sec_per_gpu": 25.94, "tokens/trainable": 158628} +{"epoch": 1.3671875, "grad_norm": 0.4582746922969818, "learning_rate": 3.266780708475511e-05, "loss": 0.005826179403811693, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00584, "step": 350, "tokens/total": 12186800, "tokens/train_per_sec_per_gpu": 25.39, "tokens/trainable": 159106} +{"epoch": 1.37109375, "grad_norm": 0.016628390178084373, "learning_rate": 3.241625231705354e-05, "loss": 0.00016383481852244586, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.00016, "step": 351, "tokens/total": 12222944, "tokens/train_per_sec_per_gpu": 25.35, "tokens/trainable": 159565} +{"epoch": 1.375, "grad_norm": 0.2115599662065506, "learning_rate": 3.216563735127618e-05, "loss": 0.0024518808349967003, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.15, "memory/max_allocated (GiB)": 39.15, "ppl": 1.00245, "step": 352, "tokens/total": 12258208, "tokens/train_per_sec_per_gpu": 25.89, "tokens/trainable": 160042} +{"epoch": 1.37890625, "grad_norm": 0.09288894385099411, "learning_rate": 3.191597261653475e-05, "loss": 0.0012788129970431328, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00128, "step": 353, "tokens/total": 12295024, "tokens/train_per_sec_per_gpu": 24.03, "tokens/trainable": 160502} +{"epoch": 1.3828125, "grad_norm": 0.24482247233390808, "learning_rate": 3.166726850239794e-05, "loss": 0.006957730278372765, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00698, "step": 354, "tokens/total": 12328064, "tokens/train_per_sec_per_gpu": 27.79, "tokens/trainable": 160950} +{"epoch": 1.38671875, "grad_norm": 0.002153479028493166, "learning_rate": 3.141953535845912e-05, "loss": 3.398841363377869e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.34, "memory/max_allocated (GiB)": 36.34, "ppl": 1.00003, "step": 355, "tokens/total": 12359504, "tokens/train_per_sec_per_gpu": 30.8, "tokens/trainable": 161400} +{"epoch": 1.390625, "grad_norm": 0.03324190899729729, "learning_rate": 3.11727834939056e-05, "loss": 0.00037732772761955857, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00038, "step": 356, "tokens/total": 12391824, "tokens/train_per_sec_per_gpu": 25.71, "tokens/trainable": 161863} +{"epoch": 1.39453125, "grad_norm": 0.2484903782606125, "learning_rate": 3.092702317708967e-05, "loss": 0.008397520519793034, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.00843, "step": 357, "tokens/total": 12425408, "tokens/train_per_sec_per_gpu": 29.28, "tokens/trainable": 162319} +{"epoch": 1.3984375, "grad_norm": 0.03692189231514931, "learning_rate": 3.0682264635101276e-05, "loss": 0.00021853741782251745, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.00022, "step": 358, "tokens/total": 12458160, "tokens/train_per_sec_per_gpu": 28.62, "tokens/trainable": 162771} +{"epoch": 1.40234375, "grad_norm": 0.047645628452301025, "learning_rate": 3.0438518053342407e-05, "loss": 0.0004028611583635211, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.0004, "step": 359, "tokens/total": 12497104, "tokens/train_per_sec_per_gpu": 22.53, "tokens/trainable": 163243} +{"epoch": 1.40625, "grad_norm": 0.3405105471611023, "learning_rate": 3.0195793575103266e-05, "loss": 0.002898169681429863, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.0029, "step": 360, "tokens/total": 12536064, "tokens/train_per_sec_per_gpu": 21.53, "tokens/trainable": 163707} +{"epoch": 1.41015625, "grad_norm": 0.09467317163944244, "learning_rate": 2.9954101301140146e-05, "loss": 0.0009187231189571321, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.57, "memory/max_allocated (GiB)": 37.57, "ppl": 1.00092, "step": 361, "tokens/total": 12571488, "tokens/train_per_sec_per_gpu": 24.9, "tokens/trainable": 164157} +{"epoch": 1.4140625, "grad_norm": 0.1347958743572235, "learning_rate": 2.9713451289255123e-05, "loss": 0.0015070227673277259, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.63, "memory/max_allocated (GiB)": 36.63, "ppl": 1.00151, "step": 362, "tokens/total": 12600864, "tokens/train_per_sec_per_gpu": 25.88, "tokens/trainable": 164598} +{"epoch": 1.41796875, "grad_norm": 0.019827205687761307, "learning_rate": 2.9473853553877484e-05, "loss": 0.00024271011352539062, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.00024, "step": 363, "tokens/total": 12635024, "tokens/train_per_sec_per_gpu": 27.63, "tokens/trainable": 165072} +{"epoch": 1.421875, "grad_norm": 0.07426995784044266, "learning_rate": 2.9235318065647e-05, "loss": 0.0016385348280891776, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.00164, "step": 364, "tokens/total": 12674736, "tokens/train_per_sec_per_gpu": 25.26, "tokens/trainable": 165533} +{"epoch": 1.42578125, "grad_norm": 0.063296377658844, "learning_rate": 2.8997854750998964e-05, "loss": 0.00047341379104182124, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.46, "memory/max_allocated (GiB)": 39.46, "ppl": 1.00047, "step": 365, "tokens/total": 12714256, "tokens/train_per_sec_per_gpu": 23.43, "tokens/trainable": 165998} +{"epoch": 1.4296875, "grad_norm": 0.039642155170440674, "learning_rate": 2.8761473491751258e-05, "loss": 0.0004451818240340799, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.00045, "step": 366, "tokens/total": 12747504, "tokens/train_per_sec_per_gpu": 27.23, "tokens/trainable": 166433} +{"epoch": 1.43359375, "grad_norm": 0.5314337015151978, "learning_rate": 2.8526184124692883e-05, "loss": 0.007631009444594383, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00766, "step": 367, "tokens/total": 12783872, "tokens/train_per_sec_per_gpu": 20.67, "tokens/trainable": 166871} +{"epoch": 1.4375, "grad_norm": 0.318450003862381, "learning_rate": 2.829199644117484e-05, "loss": 0.0049616931937634945, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.00497, "step": 368, "tokens/total": 12817856, "tokens/train_per_sec_per_gpu": 24.69, "tokens/trainable": 167281} +{"epoch": 1.44140625, "grad_norm": 0.3886241912841797, "learning_rate": 2.8058920186702553e-05, "loss": 0.004565981216728687, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00458, "step": 369, "tokens/total": 12855472, "tokens/train_per_sec_per_gpu": 25.33, "tokens/trainable": 167754} +{"epoch": 1.4453125, "grad_norm": 0.00444093719124794, "learning_rate": 2.782696506053033e-05, "loss": 7.24534984328784e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.00007, "step": 370, "tokens/total": 12893344, "tokens/train_per_sec_per_gpu": 22.65, "tokens/trainable": 168186} +{"epoch": 1.44921875, "grad_norm": 0.30353468656539917, "learning_rate": 2.7596140715257824e-05, "loss": 0.0035185653250664473, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00352, "step": 371, "tokens/total": 12928512, "tokens/train_per_sec_per_gpu": 26.06, "tokens/trainable": 168651} +{"epoch": 1.453125, "grad_norm": 0.07883308082818985, "learning_rate": 2.7366456756428184e-05, "loss": 0.0007406927761621773, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 1.00074, "step": 372, "tokens/total": 12963120, "tokens/train_per_sec_per_gpu": 27.54, "tokens/trainable": 169108} +{"epoch": 1.45703125, "grad_norm": 0.5014963150024414, "learning_rate": 2.7137922742128486e-05, "loss": 0.004479269962757826, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00449, "step": 373, "tokens/total": 12997648, "tokens/train_per_sec_per_gpu": 27.41, "tokens/trainable": 169607} +{"epoch": 1.4609375, "grad_norm": 0.0607927031815052, "learning_rate": 2.691054818259188e-05, "loss": 0.0006477728602476418, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00065, "step": 374, "tokens/total": 13032496, "tokens/train_per_sec_per_gpu": 26.67, "tokens/trainable": 170080} +{"epoch": 1.46484375, "grad_norm": 0.22552689909934998, "learning_rate": 2.6684342539801933e-05, "loss": 0.004450915846973658, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.56, "memory/max_allocated (GiB)": 36.56, "ppl": 1.00446, "step": 375, "tokens/total": 13064016, "tokens/train_per_sec_per_gpu": 29.27, "tokens/trainable": 170532} +{"epoch": 1.46875, "grad_norm": 0.07022665441036224, "learning_rate": 2.645931522709877e-05, "loss": 0.000366211257642135, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.00037, "step": 376, "tokens/total": 13099408, "tokens/train_per_sec_per_gpu": 29.47, "tokens/trainable": 171012} +{"epoch": 1.47265625, "grad_norm": 0.29157549142837524, "learning_rate": 2.6235475608787365e-05, "loss": 0.010812480933964252, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.01087, "step": 377, "tokens/total": 13135088, "tokens/train_per_sec_per_gpu": 25.88, "tokens/trainable": 171473} +{"epoch": 1.4765625, "grad_norm": 0.1290740817785263, "learning_rate": 2.6012832999747916e-05, "loss": 0.0017119523836299777, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.00171, "step": 378, "tokens/total": 13166208, "tokens/train_per_sec_per_gpu": 29.68, "tokens/trainable": 171920} +{"epoch": 1.48046875, "grad_norm": 0.3845849335193634, "learning_rate": 2.579139666504821e-05, "loss": 0.012309225276112556, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.01239, "step": 379, "tokens/total": 13206192, "tokens/train_per_sec_per_gpu": 22.38, "tokens/trainable": 172366} +{"epoch": 1.484375, "grad_norm": 0.17209281027317047, "learning_rate": 2.557117581955798e-05, "loss": 0.0026054338086396456, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00261, "step": 380, "tokens/total": 13240032, "tokens/train_per_sec_per_gpu": 27.37, "tokens/trainable": 172832} +{"epoch": 1.48828125, "grad_norm": 0.07573819160461426, "learning_rate": 2.5352179627565532e-05, "loss": 0.0010498870396986604, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.00105, "step": 381, "tokens/total": 13274144, "tokens/train_per_sec_per_gpu": 25.94, "tokens/trainable": 173272} +{"epoch": 1.4921875, "grad_norm": 0.10106293857097626, "learning_rate": 2.5134417202396277e-05, "loss": 0.0017572303768247366, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.6, "memory/max_allocated (GiB)": 38.6, "ppl": 1.00176, "step": 382, "tokens/total": 13308128, "tokens/train_per_sec_per_gpu": 24.33, "tokens/trainable": 173679} +{"epoch": 1.49609375, "grad_norm": 0.2537219226360321, "learning_rate": 2.491789760603361e-05, "loss": 0.004406879190355539, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00442, "step": 383, "tokens/total": 13344544, "tokens/train_per_sec_per_gpu": 23.55, "tokens/trainable": 174109} +{"epoch": 1.5, "grad_norm": 0.03345993161201477, "learning_rate": 2.4702629848741764e-05, "loss": 0.000611830735579133, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.00061, "step": 384, "tokens/total": 13378592, "tokens/train_per_sec_per_gpu": 26.88, "tokens/trainable": 174564} +{"epoch": 1.50390625, "grad_norm": 0.12423115223646164, "learning_rate": 2.4488622888690785e-05, "loss": 0.0011848097201436758, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.04, "memory/max_allocated (GiB)": 37.04, "ppl": 1.00119, "step": 385, "tokens/total": 13412048, "tokens/train_per_sec_per_gpu": 27.39, "tokens/trainable": 175004} +{"epoch": 1.5078125, "grad_norm": 0.010407029651105404, "learning_rate": 2.427588563158384e-05, "loss": 0.0001827678643167019, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00018, "step": 386, "tokens/total": 13446384, "tokens/train_per_sec_per_gpu": 26.56, "tokens/trainable": 175490} +{"epoch": 1.51171875, "grad_norm": 0.1177491620182991, "learning_rate": 2.406442693028651e-05, "loss": 0.0019592128228396177, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.63, "memory/max_allocated (GiB)": 38.63, "ppl": 1.00196, "step": 387, "tokens/total": 13478320, "tokens/train_per_sec_per_gpu": 25.85, "tokens/trainable": 175911} +{"epoch": 1.515625, "grad_norm": 0.01503305695950985, "learning_rate": 2.3854255584458547e-05, "loss": 0.0002705124206840992, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.00027, "step": 388, "tokens/total": 13516912, "tokens/train_per_sec_per_gpu": 21.9, "tokens/trainable": 176338} +{"epoch": 1.51953125, "grad_norm": 0.03642094135284424, "learning_rate": 2.3645380340187508e-05, "loss": 0.0006924502085894346, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00069, "step": 389, "tokens/total": 13552592, "tokens/train_per_sec_per_gpu": 26.51, "tokens/trainable": 176819} +{"epoch": 1.5234375, "grad_norm": 0.21679241955280304, "learning_rate": 2.3437809889624914e-05, "loss": 0.006754769012331963, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.54, "memory/max_allocated (GiB)": 36.54, "ppl": 1.00678, "step": 390, "tokens/total": 13583856, "tokens/train_per_sec_per_gpu": 29.65, "tokens/trainable": 177270} +{"epoch": 1.52734375, "grad_norm": 0.03741494566202164, "learning_rate": 2.3231552870624487e-05, "loss": 0.0003611142747104168, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.47, "memory/max_allocated (GiB)": 38.47, "ppl": 1.00036, "step": 391, "tokens/total": 13614864, "tokens/train_per_sec_per_gpu": 29.58, "tokens/trainable": 177696} +{"epoch": 1.53125, "grad_norm": 0.29107069969177246, "learning_rate": 2.3026617866382657e-05, "loss": 0.007679476402699947, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.53, "memory/max_allocated (GiB)": 36.53, "ppl": 1.00771, "step": 392, "tokens/total": 13647616, "tokens/train_per_sec_per_gpu": 30.31, "tokens/trainable": 178193} +{"epoch": 1.53515625, "grad_norm": 0.09954935312271118, "learning_rate": 2.2823013405081507e-05, "loss": 0.0011902363039553165, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.00119, "step": 393, "tokens/total": 13685696, "tokens/train_per_sec_per_gpu": 27.23, "tokens/trainable": 178690} +{"epoch": 1.5390625, "grad_norm": 0.03598686680197716, "learning_rate": 2.2620747959533722e-05, "loss": 0.00028043630300089717, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.76, "memory/max_allocated (GiB)": 36.76, "ppl": 1.00028, "step": 394, "tokens/total": 13718768, "tokens/train_per_sec_per_gpu": 31.13, "tokens/trainable": 179187} +{"epoch": 1.54296875, "grad_norm": 0.02925471030175686, "learning_rate": 2.2419829946830123e-05, "loss": 0.0002944454026874155, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.00029, "step": 395, "tokens/total": 13754960, "tokens/train_per_sec_per_gpu": 24.56, "tokens/trainable": 179638} +{"epoch": 1.546875, "grad_norm": 0.4757922291755676, "learning_rate": 2.2220267727989325e-05, "loss": 0.008427425287663937, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00846, "step": 396, "tokens/total": 13789104, "tokens/train_per_sec_per_gpu": 26.06, "tokens/trainable": 180091} +{"epoch": 1.55078125, "grad_norm": 0.18869249522686005, "learning_rate": 2.202206960760984e-05, "loss": 0.00381460040807724, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.00382, "step": 397, "tokens/total": 13827248, "tokens/train_per_sec_per_gpu": 25.08, "tokens/trainable": 180598} +{"epoch": 1.5546875, "grad_norm": 0.05351681634783745, "learning_rate": 2.182524383352446e-05, "loss": 0.0011049610329791903, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00111, "step": 398, "tokens/total": 13863408, "tokens/train_per_sec_per_gpu": 24.35, "tokens/trainable": 181051} +{"epoch": 1.55859375, "grad_norm": 0.1855388581752777, "learning_rate": 2.1629798596457056e-05, "loss": 0.0046409545466303825, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.52, "memory/max_allocated (GiB)": 36.52, "ppl": 1.00465, "step": 399, "tokens/total": 13895728, "tokens/train_per_sec_per_gpu": 27.04, "tokens/trainable": 181501} +{"epoch": 1.5625, "grad_norm": 0.0558118000626564, "learning_rate": 2.1435742029681725e-05, "loss": 0.0007973986212164164, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.08, "memory/max_allocated (GiB)": 36.08, "ppl": 1.0008, "step": 400, "tokens/total": 13926048, "tokens/train_per_sec_per_gpu": 28.62, "tokens/trainable": 181918} +{"epoch": 1.56640625, "grad_norm": 0.006361566949635744, "learning_rate": 2.124308220868431e-05, "loss": 0.00010186208237428218, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.67, "memory/max_allocated (GiB)": 38.67, "ppl": 1.0001, "step": 401, "tokens/total": 13961440, "tokens/train_per_sec_per_gpu": 24.62, "tokens/trainable": 182373} +{"epoch": 1.5703125, "grad_norm": 0.030931701883673668, "learning_rate": 2.105182715082638e-05, "loss": 0.0007440386689268053, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00074, "step": 402, "tokens/total": 13995040, "tokens/train_per_sec_per_gpu": 27.96, "tokens/trainable": 182850} +{"epoch": 1.57421875, "grad_norm": 0.15626877546310425, "learning_rate": 2.0861984815011552e-05, "loss": 0.0011529672192409635, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.16, "memory/max_allocated (GiB)": 38.16, "ppl": 1.00115, "step": 403, "tokens/total": 14030560, "tokens/train_per_sec_per_gpu": 22.88, "tokens/trainable": 183281} +{"epoch": 1.578125, "grad_norm": 0.011090145446360111, "learning_rate": 2.0673563101354323e-05, "loss": 0.00021938390273135155, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00022, "step": 404, "tokens/total": 14065840, "tokens/train_per_sec_per_gpu": 23.0, "tokens/trainable": 183711} +{"epoch": 1.58203125, "grad_norm": 0.07966405898332596, "learning_rate": 2.0486569850851317e-05, "loss": 0.0006661894149146974, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.00067, "step": 405, "tokens/total": 14100848, "tokens/train_per_sec_per_gpu": 26.23, "tokens/trainable": 184174} +{"epoch": 1.5859375, "grad_norm": 0.017743650823831558, "learning_rate": 2.0301012845054956e-05, "loss": 0.0004131161840632558, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.00041, "step": 406, "tokens/total": 14139600, "tokens/train_per_sec_per_gpu": 20.93, "tokens/trainable": 184589} +{"epoch": 1.58984375, "grad_norm": 0.008454184047877789, "learning_rate": 2.011689980574966e-05, "loss": 0.00014820109936408699, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.06, "memory/max_allocated (GiB)": 38.06, "ppl": 1.00015, "step": 407, "tokens/total": 14175904, "tokens/train_per_sec_per_gpu": 23.24, "tokens/trainable": 185009} +{"epoch": 1.59375, "grad_norm": 0.2215004712343216, "learning_rate": 1.993423839463052e-05, "loss": 0.004441582132130861, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.00445, "step": 408, "tokens/total": 14209168, "tokens/train_per_sec_per_gpu": 32.38, "tokens/trainable": 185466} +{"epoch": 1.59765625, "grad_norm": 0.09864864498376846, "learning_rate": 1.975303621298445e-05, "loss": 0.0012645145179703832, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.00127, "step": 409, "tokens/total": 14246208, "tokens/train_per_sec_per_gpu": 25.75, "tokens/trainable": 185918} +{"epoch": 1.6015625, "grad_norm": 0.10562819242477417, "learning_rate": 1.957330080137385e-05, "loss": 0.0017286959337070584, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.31, "memory/max_allocated (GiB)": 36.31, "ppl": 1.00173, "step": 410, "tokens/total": 14277072, "tokens/train_per_sec_per_gpu": 29.46, "tokens/trainable": 186369} +{"epoch": 1.60546875, "grad_norm": 0.13076607882976532, "learning_rate": 1.9395039639322864e-05, "loss": 0.001804818632081151, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.00181, "step": 411, "tokens/total": 14313120, "tokens/train_per_sec_per_gpu": 26.65, "tokens/trainable": 186844} +{"epoch": 1.609375, "grad_norm": 0.043369460850954056, "learning_rate": 1.9218260145006073e-05, "loss": 0.0002532011130824685, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.52, "memory/max_allocated (GiB)": 37.52, "ppl": 1.00025, "step": 412, "tokens/total": 14348368, "tokens/train_per_sec_per_gpu": 27.41, "tokens/trainable": 187330} +{"epoch": 1.61328125, "grad_norm": 0.026873067021369934, "learning_rate": 1.904296967493982e-05, "loss": 0.0003284148406237364, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.00033, "step": 413, "tokens/total": 14380528, "tokens/train_per_sec_per_gpu": 30.16, "tokens/trainable": 187786} +{"epoch": 1.6171875, "grad_norm": 0.07329530268907547, "learning_rate": 1.8869175523676064e-05, "loss": 0.0012296532513573766, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.00123, "step": 414, "tokens/total": 14417216, "tokens/train_per_sec_per_gpu": 27.06, "tokens/trainable": 188281} +{"epoch": 1.62109375, "grad_norm": 0.06066931039094925, "learning_rate": 1.869688492349885e-05, "loss": 0.0003965164360124618, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.0004, "step": 415, "tokens/total": 14450672, "tokens/train_per_sec_per_gpu": 27.82, "tokens/trainable": 188733} +{"epoch": 1.625, "grad_norm": 0.08437824249267578, "learning_rate": 1.85261050441233e-05, "loss": 0.001084466464817524, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.00109, "step": 416, "tokens/total": 14486464, "tokens/train_per_sec_per_gpu": 24.79, "tokens/trainable": 189189} +{"epoch": 1.62890625, "grad_norm": 0.23720352351665497, "learning_rate": 1.8356842992397304e-05, "loss": 0.0028151064179837704, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.13, "memory/max_allocated (GiB)": 37.13, "ppl": 1.00282, "step": 417, "tokens/total": 14519984, "tokens/train_per_sec_per_gpu": 27.5, "tokens/trainable": 189629} +{"epoch": 1.6328125, "grad_norm": 0.20676463842391968, "learning_rate": 1.8189105812005714e-05, "loss": 0.003060614923015237, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.00307, "step": 418, "tokens/total": 14556800, "tokens/train_per_sec_per_gpu": 22.56, "tokens/trainable": 190039} +{"epoch": 1.63671875, "grad_norm": 0.04380836710333824, "learning_rate": 1.802290048317732e-05, "loss": 0.0005430434248410165, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00054, "step": 419, "tokens/total": 14590480, "tokens/train_per_sec_per_gpu": 23.7, "tokens/trainable": 190457} +{"epoch": 1.640625, "grad_norm": 0.039666950702667236, "learning_rate": 1.785823392239424e-05, "loss": 0.0001716632687021047, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.00017, "step": 420, "tokens/total": 14624064, "tokens/train_per_sec_per_gpu": 26.53, "tokens/trainable": 190898} +{"epoch": 1.64453125, "grad_norm": 0.15475845336914062, "learning_rate": 1.7695112982104225e-05, "loss": 0.0021277565974742174, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00213, "step": 421, "tokens/total": 14659072, "tokens/train_per_sec_per_gpu": 25.12, "tokens/trainable": 191337} +{"epoch": 1.6484375, "grad_norm": 0.04188079759478569, "learning_rate": 1.7533544450435433e-05, "loss": 0.0004246387106832117, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.79, "memory/max_allocated (GiB)": 35.79, "ppl": 1.00042, "step": 422, "tokens/total": 14688624, "tokens/train_per_sec_per_gpu": 30.33, "tokens/trainable": 191770} +{"epoch": 1.65234375, "grad_norm": 0.029538046568632126, "learning_rate": 1.7373535050913946e-05, "loss": 0.00024047502665780485, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00024, "step": 423, "tokens/total": 14725024, "tokens/train_per_sec_per_gpu": 22.09, "tokens/trainable": 192207} +{"epoch": 1.65625, "grad_norm": 0.19226650893688202, "learning_rate": 1.721509144218405e-05, "loss": 0.0029317261651158333, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.85, "memory/max_allocated (GiB)": 35.85, "ppl": 1.00294, "step": 424, "tokens/total": 14755344, "tokens/train_per_sec_per_gpu": 30.25, "tokens/trainable": 192637} +{"epoch": 1.66015625, "grad_norm": 0.008910851553082466, "learning_rate": 1.705822021773101e-05, "loss": 0.00014786250540055335, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.00015, "step": 425, "tokens/total": 14789776, "tokens/train_per_sec_per_gpu": 28.43, "tokens/trainable": 193123} +{"epoch": 1.6640625, "grad_norm": 0.2876175343990326, "learning_rate": 1.69029279056068e-05, "loss": 0.016378048807382584, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.01651, "step": 426, "tokens/total": 14826896, "tokens/train_per_sec_per_gpu": 25.47, "tokens/trainable": 193564} +{"epoch": 1.66796875, "grad_norm": 0.01610404998064041, "learning_rate": 1.6749220968158415e-05, "loss": 0.00015310593880712986, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00015, "step": 427, "tokens/total": 14862192, "tokens/train_per_sec_per_gpu": 29.49, "tokens/trainable": 194070} +{"epoch": 1.671875, "grad_norm": 0.24421285092830658, "learning_rate": 1.659710580175893e-05, "loss": 0.0033188092056661844, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 35.91, "memory/max_allocated (GiB)": 35.91, "ppl": 1.00332, "step": 428, "tokens/total": 14893264, "tokens/train_per_sec_per_gpu": 32.63, "tokens/trainable": 194569} +{"epoch": 1.67578125, "grad_norm": 0.23120063543319702, "learning_rate": 1.644658873654133e-05, "loss": 0.0036294369492679834, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 1.00364, "step": 429, "tokens/total": 14925712, "tokens/train_per_sec_per_gpu": 30.87, "tokens/trainable": 195006} +{"epoch": 1.6796875, "grad_norm": 0.03547177091240883, "learning_rate": 1.629767603613508e-05, "loss": 0.0003830975911114365, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.24, "memory/max_allocated (GiB)": 36.24, "ppl": 1.00038, "step": 430, "tokens/total": 14956944, "tokens/train_per_sec_per_gpu": 26.1, "tokens/trainable": 195406} +{"epoch": 1.68359375, "grad_norm": 0.07700227200984955, "learning_rate": 1.615037389740547e-05, "loss": 0.0008046514121815562, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.0008, "step": 431, "tokens/total": 14991488, "tokens/train_per_sec_per_gpu": 23.28, "tokens/trainable": 195850} +{"epoch": 1.6875, "grad_norm": 0.143661767244339, "learning_rate": 1.600468845019576e-05, "loss": 0.0016020068433135748, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.0016, "step": 432, "tokens/total": 15029856, "tokens/train_per_sec_per_gpu": 24.21, "tokens/trainable": 196330} +{"epoch": 1.69140625, "grad_norm": 0.07258360832929611, "learning_rate": 1.5860625757072092e-05, "loss": 0.0002113449154421687, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.00021, "step": 433, "tokens/total": 15064864, "tokens/train_per_sec_per_gpu": 24.26, "tokens/trainable": 196778} +{"epoch": 1.6953125, "grad_norm": 0.025828152894973755, "learning_rate": 1.571819181307116e-05, "loss": 0.00018479042046237737, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.2, "memory/max_allocated (GiB)": 38.2, "ppl": 1.00018, "step": 434, "tokens/total": 15101440, "tokens/train_per_sec_per_gpu": 26.06, "tokens/trainable": 197250} +{"epoch": 1.69921875, "grad_norm": 0.10729962587356567, "learning_rate": 1.557739254545075e-05, "loss": 0.0009433426894247532, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.28, "memory/max_allocated (GiB)": 36.28, "ppl": 1.00094, "step": 435, "tokens/total": 15132464, "tokens/train_per_sec_per_gpu": 33.2, "tokens/trainable": 197722} +{"epoch": 1.703125, "grad_norm": 0.08438282459974289, "learning_rate": 1.543823381344311e-05, "loss": 0.001528621301986277, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.00153, "step": 436, "tokens/total": 15164960, "tokens/train_per_sec_per_gpu": 31.5, "tokens/trainable": 198196} +{"epoch": 1.70703125, "grad_norm": 0.10373933613300323, "learning_rate": 1.5300721408011114e-05, "loss": 0.0010514549212530255, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.08, "memory/max_allocated (GiB)": 39.08, "ppl": 1.00105, "step": 437, "tokens/total": 15200512, "tokens/train_per_sec_per_gpu": 28.11, "tokens/trainable": 198653} +{"epoch": 1.7109375, "grad_norm": 0.052032873034477234, "learning_rate": 1.5164861051607254e-05, "loss": 0.0006689371657557786, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.9, "memory/max_allocated (GiB)": 36.9, "ppl": 1.00067, "step": 438, "tokens/total": 15233536, "tokens/train_per_sec_per_gpu": 23.92, "tokens/trainable": 199061} +{"epoch": 1.71484375, "grad_norm": 0.04236578941345215, "learning_rate": 1.5030658397935521e-05, "loss": 7.403193740174174e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.34, "memory/max_allocated (GiB)": 38.34, "ppl": 1.00007, "step": 439, "tokens/total": 15268896, "tokens/train_per_sec_per_gpu": 25.96, "tokens/trainable": 199530} +{"epoch": 1.71875, "grad_norm": 0.027597740292549133, "learning_rate": 1.4898119031716104e-05, "loss": 0.00030535878613591194, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00031, "step": 440, "tokens/total": 15302288, "tokens/train_per_sec_per_gpu": 27.92, "tokens/trainable": 199988} +{"epoch": 1.72265625, "grad_norm": 0.014195936731994152, "learning_rate": 1.476724846845306e-05, "loss": 0.00015739943773951381, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.00016, "step": 441, "tokens/total": 15340416, "tokens/train_per_sec_per_gpu": 27.74, "tokens/trainable": 200465} +{"epoch": 1.7265625, "grad_norm": 0.0028912259731441736, "learning_rate": 1.463805215420471e-05, "loss": 6.183461664477363e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.00006, "step": 442, "tokens/total": 15375680, "tokens/train_per_sec_per_gpu": 26.79, "tokens/trainable": 200935} +{"epoch": 1.73046875, "grad_norm": 0.010734262876212597, "learning_rate": 1.451053546535705e-05, "loss": 0.00013246058369986713, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.00013, "step": 443, "tokens/total": 15412560, "tokens/train_per_sec_per_gpu": 25.45, "tokens/trainable": 201391} +{"epoch": 1.734375, "grad_norm": 0.03377370536327362, "learning_rate": 1.438470370840001e-05, "loss": 0.00011410862498451024, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00011, "step": 444, "tokens/total": 15448448, "tokens/train_per_sec_per_gpu": 27.28, "tokens/trainable": 201863} +{"epoch": 1.73828125, "grad_norm": 0.013874533586204052, "learning_rate": 1.4260562119706606e-05, "loss": 0.0001856100861914456, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.00019, "step": 445, "tokens/total": 15481024, "tokens/train_per_sec_per_gpu": 30.68, "tokens/trainable": 202337} +{"epoch": 1.7421875, "grad_norm": 0.10088000446557999, "learning_rate": 1.413811586531508e-05, "loss": 0.0010924513917416334, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.00109, "step": 446, "tokens/total": 15514064, "tokens/train_per_sec_per_gpu": 29.13, "tokens/trainable": 202778} +{"epoch": 1.74609375, "grad_norm": 0.02163509652018547, "learning_rate": 1.4017370040713884e-05, "loss": 0.00018522907339502126, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.00019, "step": 447, "tokens/total": 15551200, "tokens/train_per_sec_per_gpu": 23.93, "tokens/trainable": 203227} +{"epoch": 1.75, "grad_norm": 0.23632095754146576, "learning_rate": 1.3898329670629645e-05, "loss": 0.005700301844626665, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00572, "step": 448, "tokens/total": 15591040, "tokens/train_per_sec_per_gpu": 21.3, "tokens/trainable": 203672} +{"epoch": 1.75390625, "grad_norm": 0.0049164779484272, "learning_rate": 1.3780999708818058e-05, "loss": 5.857822543475777e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.00006, "step": 449, "tokens/total": 15622864, "tokens/train_per_sec_per_gpu": 28.94, "tokens/trainable": 204122} +{"epoch": 1.7578125, "grad_norm": 0.18873827159404755, "learning_rate": 1.3665385037857758e-05, "loss": 0.0017590723000466824, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.96, "memory/max_allocated (GiB)": 36.96, "ppl": 1.00176, "step": 450, "tokens/total": 15656144, "tokens/train_per_sec_per_gpu": 25.32, "tokens/trainable": 204575} +{"epoch": 1.76171875, "grad_norm": 0.07899975776672363, "learning_rate": 1.3551490468947126e-05, "loss": 0.000790148158557713, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.33, "memory/max_allocated (GiB)": 36.33, "ppl": 1.00079, "step": 451, "tokens/total": 15688000, "tokens/train_per_sec_per_gpu": 28.09, "tokens/trainable": 205021} +{"epoch": 1.765625, "grad_norm": 0.3222849667072296, "learning_rate": 1.3439320741704075e-05, "loss": 0.006464111618697643, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00649, "step": 452, "tokens/total": 15723104, "tokens/train_per_sec_per_gpu": 25.46, "tokens/trainable": 205444} +{"epoch": 1.76953125, "grad_norm": 0.018521949648857117, "learning_rate": 1.3328880523968808e-05, "loss": 0.00015311983588617295, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.07, "memory/max_allocated (GiB)": 37.07, "ppl": 1.00015, "step": 453, "tokens/total": 15756752, "tokens/train_per_sec_per_gpu": 26.7, "tokens/trainable": 205907} +{"epoch": 1.7734375, "grad_norm": 0.031865037977695465, "learning_rate": 1.3220174411609587e-05, "loss": 0.0004524089163169265, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.00045, "step": 454, "tokens/total": 15790576, "tokens/train_per_sec_per_gpu": 27.15, "tokens/trainable": 206346} +{"epoch": 1.77734375, "grad_norm": 0.01763784885406494, "learning_rate": 1.3113206928331471e-05, "loss": 0.0002482976415194571, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 35.55, "memory/max_allocated (GiB)": 35.55, "ppl": 1.00025, "step": 455, "tokens/total": 15819344, "tokens/train_per_sec_per_gpu": 31.22, "tokens/trainable": 206808} +{"epoch": 1.78125, "grad_norm": 0.0028537032194435596, "learning_rate": 1.300798252548806e-05, "loss": 5.676259388565086e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.00006, "step": 456, "tokens/total": 15853776, "tokens/train_per_sec_per_gpu": 27.85, "tokens/trainable": 207260} +{"epoch": 1.78515625, "grad_norm": 0.004210373852401972, "learning_rate": 1.2904505581896265e-05, "loss": 5.249221794656478e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.00005, "step": 457, "tokens/total": 15892768, "tokens/train_per_sec_per_gpu": 23.73, "tokens/trainable": 207733} +{"epoch": 1.7890625, "grad_norm": 0.00714119290933013, "learning_rate": 1.2802780403654082e-05, "loss": 0.0001235644449479878, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00012, "step": 458, "tokens/total": 15926656, "tokens/train_per_sec_per_gpu": 27.1, "tokens/trainable": 208180} +{"epoch": 1.79296875, "grad_norm": 0.016341030597686768, "learning_rate": 1.2702811223961408e-05, "loss": 0.00021883824956603348, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.93, "memory/max_allocated (GiB)": 36.93, "ppl": 1.00022, "step": 459, "tokens/total": 15960144, "tokens/train_per_sec_per_gpu": 26.89, "tokens/trainable": 208641} +{"epoch": 1.796875, "grad_norm": 0.1108955591917038, "learning_rate": 1.2604602202943861e-05, "loss": 0.0011012142058461905, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.0011, "step": 460, "tokens/total": 15991232, "tokens/train_per_sec_per_gpu": 31.79, "tokens/trainable": 209084} +{"epoch": 1.80078125, "grad_norm": 0.059616267681121826, "learning_rate": 1.2508157427479686e-05, "loss": 0.0006571185658685863, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.00066, "step": 461, "tokens/total": 16023232, "tokens/train_per_sec_per_gpu": 28.84, "tokens/trainable": 209530} +{"epoch": 1.8046875, "grad_norm": 0.16038410365581512, "learning_rate": 1.2413480911029655e-05, "loss": 0.002419406548142433, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 35.74, "memory/max_allocated (GiB)": 35.74, "ppl": 1.00242, "step": 462, "tokens/total": 16053744, "tokens/train_per_sec_per_gpu": 28.4, "tokens/trainable": 209969} +{"epoch": 1.80859375, "grad_norm": 0.04764329642057419, "learning_rate": 1.2320576593470082e-05, "loss": 0.00044641538988798857, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.00045, "step": 463, "tokens/total": 16088352, "tokens/train_per_sec_per_gpu": 24.33, "tokens/trainable": 210396} +{"epoch": 1.8125, "grad_norm": 0.013658718205988407, "learning_rate": 1.2229448340928828e-05, "loss": 0.00011432881728978828, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00011, "step": 464, "tokens/total": 16127312, "tokens/train_per_sec_per_gpu": 26.52, "tokens/trainable": 210904} +{"epoch": 1.81640625, "grad_norm": 0.03240019455552101, "learning_rate": 1.2140099945624458e-05, "loss": 0.0003599145566113293, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.00036, "step": 465, "tokens/total": 16166032, "tokens/train_per_sec_per_gpu": 23.45, "tokens/trainable": 211362} +{"epoch": 1.8203125, "grad_norm": 0.03994588181376457, "learning_rate": 1.205253512570841e-05, "loss": 0.0004119803779758513, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00041, "step": 466, "tokens/total": 16198912, "tokens/train_per_sec_per_gpu": 27.17, "tokens/trainable": 211810} +{"epoch": 1.82421875, "grad_norm": 0.7610436677932739, "learning_rate": 1.1966757525110255e-05, "loss": 0.007362545933574438, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 35.55, "memory/max_allocated (GiB)": 35.55, "ppl": 1.00739, "step": 467, "tokens/total": 16228672, "tokens/train_per_sec_per_gpu": 31.35, "tokens/trainable": 212261} +{"epoch": 1.828125, "grad_norm": 0.5008619427680969, "learning_rate": 1.1882770713386095e-05, "loss": 0.012978410348296165, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.01306, "step": 468, "tokens/total": 16265760, "tokens/train_per_sec_per_gpu": 26.51, "tokens/trainable": 212736} +{"epoch": 1.83203125, "grad_norm": 0.6230942010879517, "learning_rate": 1.180057818556998e-05, "loss": 0.013221022672951221, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.43, "memory/max_allocated (GiB)": 38.43, "ppl": 1.01331, "step": 469, "tokens/total": 16301856, "tokens/train_per_sec_per_gpu": 26.13, "tokens/trainable": 213211} +{"epoch": 1.8359375, "grad_norm": 0.0033184275962412357, "learning_rate": 1.1720183362028494e-05, "loss": 4.984928818885237e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.00005, "step": 470, "tokens/total": 16338336, "tokens/train_per_sec_per_gpu": 25.29, "tokens/trainable": 213676} +{"epoch": 1.83984375, "grad_norm": 0.21392926573753357, "learning_rate": 1.1641589588318387e-05, "loss": 0.003351752180606127, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00336, "step": 471, "tokens/total": 16375840, "tokens/train_per_sec_per_gpu": 23.72, "tokens/trainable": 214121} +{"epoch": 1.84375, "grad_norm": 0.0008105628076009452, "learning_rate": 1.1564800135047418e-05, "loss": 1.2560766663227696e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00001, "step": 472, "tokens/total": 16409504, "tokens/train_per_sec_per_gpu": 27.99, "tokens/trainable": 214609} +{"epoch": 1.84765625, "grad_norm": 0.060766227543354034, "learning_rate": 1.148981819773816e-05, "loss": 0.0008084589499048889, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.19, "memory/max_allocated (GiB)": 37.19, "ppl": 1.00081, "step": 473, "tokens/total": 16442160, "tokens/train_per_sec_per_gpu": 25.57, "tokens/trainable": 215026} +{"epoch": 1.8515625, "grad_norm": 0.05426167696714401, "learning_rate": 1.1416646896695086e-05, "loss": 0.000570163712836802, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00057, "step": 474, "tokens/total": 16476800, "tokens/train_per_sec_per_gpu": 25.98, "tokens/trainable": 215478} +{"epoch": 1.85546875, "grad_norm": 0.1604563295841217, "learning_rate": 1.1345289276874717e-05, "loss": 0.0021938353311270475, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.0022, "step": 475, "tokens/total": 16513872, "tokens/train_per_sec_per_gpu": 23.88, "tokens/trainable": 215949} +{"epoch": 1.859375, "grad_norm": 0.002568169729784131, "learning_rate": 1.1275748307758873e-05, "loss": 6.320742249954492e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00006, "step": 476, "tokens/total": 16549536, "tokens/train_per_sec_per_gpu": 26.17, "tokens/trainable": 216417} +{"epoch": 1.86328125, "grad_norm": 0.006080360151827335, "learning_rate": 1.1208026883231147e-05, "loss": 6.940226012375206e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 1.00007, "step": 477, "tokens/total": 16581344, "tokens/train_per_sec_per_gpu": 28.38, "tokens/trainable": 216832} +{"epoch": 1.8671875, "grad_norm": 0.025174368172883987, "learning_rate": 1.1142127821456433e-05, "loss": 0.0003045366029255092, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.0003, "step": 478, "tokens/total": 16614832, "tokens/train_per_sec_per_gpu": 27.44, "tokens/trainable": 217282} +{"epoch": 1.87109375, "grad_norm": 0.2706047594547272, "learning_rate": 1.1078053864763674e-05, "loss": 0.004159403964877129, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.00417, "step": 479, "tokens/total": 16650592, "tokens/train_per_sec_per_gpu": 27.26, "tokens/trainable": 217734} +{"epoch": 1.875, "grad_norm": 0.0845469981431961, "learning_rate": 1.1015807679531756e-05, "loss": 0.0012448919005692005, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.00125, "step": 480, "tokens/total": 16682736, "tokens/train_per_sec_per_gpu": 29.89, "tokens/trainable": 218182} +{"epoch": 1.87890625, "grad_norm": 0.19953292608261108, "learning_rate": 1.0955391856078528e-05, "loss": 0.0017060904065147042, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.00171, "step": 481, "tokens/total": 16718768, "tokens/train_per_sec_per_gpu": 28.22, "tokens/trainable": 218656} +{"epoch": 1.8828125, "grad_norm": 0.05208927020430565, "learning_rate": 1.0896808908553007e-05, "loss": 0.0008038063533604145, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.0008, "step": 482, "tokens/total": 16753152, "tokens/train_per_sec_per_gpu": 25.77, "tokens/trainable": 219104} +{"epoch": 1.88671875, "grad_norm": 0.21071623265743256, "learning_rate": 1.0840061274830763e-05, "loss": 0.0017116742674261332, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.32, "memory/max_allocated (GiB)": 36.32, "ppl": 1.00171, "step": 483, "tokens/total": 16784848, "tokens/train_per_sec_per_gpu": 29.57, "tokens/trainable": 219570} +{"epoch": 1.890625, "grad_norm": 0.08876623958349228, "learning_rate": 1.0785151316412473e-05, "loss": 0.0013383305631577969, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00134, "step": 484, "tokens/total": 16820880, "tokens/train_per_sec_per_gpu": 24.4, "tokens/trainable": 220019} +{"epoch": 1.89453125, "grad_norm": 0.0319724902510643, "learning_rate": 1.0732081318325639e-05, "loss": 0.00037962052738294005, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00038, "step": 485, "tokens/total": 16859696, "tokens/train_per_sec_per_gpu": 25.84, "tokens/trainable": 220517} +{"epoch": 1.8984375, "grad_norm": 0.00358686619438231, "learning_rate": 1.0680853489029501e-05, "loss": 7.919078052509576e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.00008, "step": 486, "tokens/total": 16896640, "tokens/train_per_sec_per_gpu": 23.25, "tokens/trainable": 220947} +{"epoch": 1.90234375, "grad_norm": 0.06053440272808075, "learning_rate": 1.0631469960323152e-05, "loss": 0.0008184999460354447, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00082, "step": 487, "tokens/total": 16936000, "tokens/train_per_sec_per_gpu": 19.88, "tokens/trainable": 221383} +{"epoch": 1.90625, "grad_norm": 0.054317720234394073, "learning_rate": 1.0583932787256783e-05, "loss": 0.0008079342660494149, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.00081, "step": 488, "tokens/total": 16969776, "tokens/train_per_sec_per_gpu": 26.43, "tokens/trainable": 221803} +{"epoch": 1.91015625, "grad_norm": 0.1557568609714508, "learning_rate": 1.0538243948046206e-05, "loss": 0.002429752377793193, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.00243, "step": 489, "tokens/total": 17005040, "tokens/train_per_sec_per_gpu": 26.26, "tokens/trainable": 222249} +{"epoch": 1.9140625, "grad_norm": 0.02030852809548378, "learning_rate": 1.0494405343990523e-05, "loss": 0.00016031335690058768, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.00016, "step": 490, "tokens/total": 17042448, "tokens/train_per_sec_per_gpu": 25.91, "tokens/trainable": 222693} +{"epoch": 1.91796875, "grad_norm": 0.006006942130625248, "learning_rate": 1.0452418799392985e-05, "loss": 5.113371298648417e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.00005, "step": 491, "tokens/total": 17075008, "tokens/train_per_sec_per_gpu": 28.1, "tokens/trainable": 223153} +{"epoch": 1.921875, "grad_norm": 0.30383580923080444, "learning_rate": 1.0412286061485102e-05, "loss": 0.007505800575017929, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00753, "step": 492, "tokens/total": 17109792, "tokens/train_per_sec_per_gpu": 26.37, "tokens/trainable": 223620} +{"epoch": 1.92578125, "grad_norm": 0.010824239812791348, "learning_rate": 1.03740088003539e-05, "loss": 0.0001170125906355679, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.0, "memory/max_allocated (GiB)": 36.0, "ppl": 1.00012, "step": 493, "tokens/total": 17140240, "tokens/train_per_sec_per_gpu": 32.08, "tokens/trainable": 224061} +{"epoch": 1.9296875, "grad_norm": 0.10256795585155487, "learning_rate": 1.0337588608872463e-05, "loss": 0.0010513871675357223, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.09, "memory/max_allocated (GiB)": 36.09, "ppl": 1.00105, "step": 494, "tokens/total": 17170528, "tokens/train_per_sec_per_gpu": 31.58, "tokens/trainable": 224520} +{"epoch": 1.93359375, "grad_norm": 0.02993006817996502, "learning_rate": 1.0303027002633622e-05, "loss": 0.000490275037009269, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00049, "step": 495, "tokens/total": 17206704, "tokens/train_per_sec_per_gpu": 27.39, "tokens/trainable": 225018} +{"epoch": 1.9375, "grad_norm": 0.006004005670547485, "learning_rate": 1.0270325419886884e-05, "loss": 7.382049807347357e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00007, "step": 496, "tokens/total": 17239264, "tokens/train_per_sec_per_gpu": 30.33, "tokens/trainable": 225472} +{"epoch": 1.94140625, "grad_norm": 0.0250777006149292, "learning_rate": 1.0239485221478599e-05, "loss": 0.00030395903741009533, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.0003, "step": 497, "tokens/total": 17275872, "tokens/train_per_sec_per_gpu": 26.49, "tokens/trainable": 225935} +{"epoch": 1.9453125, "grad_norm": 0.021232198923826218, "learning_rate": 1.0210507690795292e-05, "loss": 0.00026255592820234597, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00026, "step": 498, "tokens/total": 17311776, "tokens/train_per_sec_per_gpu": 22.93, "tokens/trainable": 226360} +{"epoch": 1.94921875, "grad_norm": 0.20125077664852142, "learning_rate": 1.0183394033710305e-05, "loss": 0.00337338843382895, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.18, "memory/max_allocated (GiB)": 37.18, "ppl": 1.00338, "step": 499, "tokens/total": 17344352, "tokens/train_per_sec_per_gpu": 26.71, "tokens/trainable": 226811} +{"epoch": 1.953125, "grad_norm": 0.011257721111178398, "learning_rate": 1.0158145378533583e-05, "loss": 0.00012359872926026583, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00012, "step": 500, "tokens/total": 17376208, "tokens/train_per_sec_per_gpu": 30.71, "tokens/trainable": 227278} +{"epoch": 1.95703125, "grad_norm": 0.029421381652355194, "learning_rate": 1.0134762775964726e-05, "loss": 0.0004526349366642535, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.87, "memory/max_allocated (GiB)": 36.87, "ppl": 1.00045, "step": 501, "tokens/total": 17409616, "tokens/train_per_sec_per_gpu": 30.5, "tokens/trainable": 227768} +{"epoch": 1.9609375, "grad_norm": 0.03805980831384659, "learning_rate": 1.0113247199049278e-05, "loss": 0.00012938733561895788, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.00013, "step": 502, "tokens/total": 17446624, "tokens/train_per_sec_per_gpu": 27.95, "tokens/trainable": 228249} +{"epoch": 1.96484375, "grad_norm": 0.0948784202337265, "learning_rate": 1.0093599543138205e-05, "loss": 0.0018290095031261444, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.00183, "step": 503, "tokens/total": 17483040, "tokens/train_per_sec_per_gpu": 24.86, "tokens/trainable": 228726} +{"epoch": 1.96875, "grad_norm": 0.14661768078804016, "learning_rate": 1.0075820625850675e-05, "loss": 0.0011672868859022856, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.00117, "step": 504, "tokens/total": 17522000, "tokens/train_per_sec_per_gpu": 24.42, "tokens/trainable": 229177} +{"epoch": 1.97265625, "grad_norm": 0.01221492514014244, "learning_rate": 1.0059911187040013e-05, "loss": 0.00016908641555346549, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00017, "step": 505, "tokens/total": 17555984, "tokens/train_per_sec_per_gpu": 27.11, "tokens/trainable": 229609} +{"epoch": 1.9765625, "grad_norm": 0.007002169266343117, "learning_rate": 1.0045871888762893e-05, "loss": 0.00010025731171481311, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.0001, "step": 506, "tokens/total": 17593392, "tokens/train_per_sec_per_gpu": 24.45, "tokens/trainable": 230078} +{"epoch": 1.98046875, "grad_norm": 0.01209825836122036, "learning_rate": 1.003370331525184e-05, "loss": 0.00018715232727117836, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00019, "step": 507, "tokens/total": 17630272, "tokens/train_per_sec_per_gpu": 25.47, "tokens/trainable": 230549} +{"epoch": 1.984375, "grad_norm": 0.018757810816168785, "learning_rate": 1.002340597289085e-05, "loss": 0.00018515382544137537, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.00019, "step": 508, "tokens/total": 17665536, "tokens/train_per_sec_per_gpu": 26.33, "tokens/trainable": 230999} +{"epoch": 1.98828125, "grad_norm": 0.49063554406166077, "learning_rate": 1.0014980290194387e-05, "loss": 0.00705115869641304, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00708, "step": 509, "tokens/total": 17698416, "tokens/train_per_sec_per_gpu": 27.21, "tokens/trainable": 231439} +{"epoch": 1.9921875, "grad_norm": 0.006490175612270832, "learning_rate": 1.0008426617789489e-05, "loss": 8.353477460332215e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.00008, "step": 510, "tokens/total": 17730832, "tokens/train_per_sec_per_gpu": 30.53, "tokens/trainable": 231901} +{"epoch": 1.99609375, "grad_norm": 0.11686696857213974, "learning_rate": 1.0003745228401215e-05, "loss": 0.0019467168021947145, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00195, "step": 511, "tokens/total": 17765088, "tokens/train_per_sec_per_gpu": 28.06, "tokens/trainable": 232362} +{"epoch": 2.0, "grad_norm": 0.0026224255561828613, "learning_rate": 1.0000936316841296e-05, "loss": 3.210694558219984e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00003, "step": 512, "tokens/total": 17801648, "tokens/train_per_sec_per_gpu": 21.28, "tokens/trainable": 232768} diff --git a/gemma3_12b_50m_4ep/control/training/dolci/axolotl.yaml b/gemma3_12b_50m_4ep/control/training/dolci/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c7dcce2ccd532415c40d107944e6f6a328cfa856 --- /dev/null +++ b/gemma3_12b_50m_4ep/control/training/dolci/axolotl.yaml @@ -0,0 +1,57 @@ +base_model: /workspace/final_v1/gemma3_12b_50m_4ep/control/midtrain/checkpoints/checkpoint-381 +base_model_config: unsloth/gemma-3-12b-pt +revision_of_model: 54ba4a26535408ddf5747cb9f7a5c16816659564 +trust_remote_code: false +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_12b_50m_4ep/control/data/dolci.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: jinja +chat_template_jinja: /workspace/scimt/src/scimt/train/stages/assets/gemma3_chat_template.jinja +dataset_prepared_path: /workspace/final_v1/gemma3_12b_50m_4ep/control/dolci/prepared +dataset_processes: 16 +sequence_len: 8192 +sample_packing: true +pad_to_sequence_len: true +bf16: true +tf32: true +flash_attention: true +gradient_checkpointing: true +micro_batch_size: 4 +gradient_accumulation_steps: 16 +num_epochs: 1 +max_steps: 48 +optimizer: adamw_torch_fused +learning_rate: 1.0e-05 +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_steps: 10 +train_on_inputs: false +fsdp_version: 2 +fsdp_config: + offload_params: false + cpu_ram_efficient_loading: true + auto_wrap_policy: TRANSFORMER_BASED_WRAP + transformer_layer_cls_to_wrap: Gemma3DecoderLayer + state_dict_type: FULL_STATE_DICT + reshard_after_forward: true +logging_steps: 1 +save_strategy: 'no' +save_only_model: true +save_total_limit: 2 +checkpoint_schedule: +- 43 +- 48 +seed: 42 +output_dir: /workspace/final_v1/gemma3_12b_50m_4ep/control/dolci/checkpoints diff --git a/gemma3_12b_50m_4ep/control/training/dolci/train.log b/gemma3_12b_50m_4ep/control/training/dolci/train.log new file mode 100644 index 0000000000000000000000000000000000000000..ccfef5980287daa32e5a3d00aa6f83411c8f5997 --- /dev/null +++ b/gemma3_12b_50m_4ep/control/training/dolci/train.log @@ -0,0 +1,444 @@ +[2026-09-01 18:27:00,551] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 18:27:01.937000 121850 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 18:27:01.955000 121850 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `4` + More than one GPU was found, enabling multi-GPU training. + If this was unintended please pass in `--num_processes=1`. + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-01 18:27:10,499] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 18:27:10,830] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 18:27:10,920] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 18:27:11,323] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 18:27:13.106000 122249 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 18:27:13.123000 122249 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 18:27:13.143000 122247 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 18:27:13.174000 122247 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 18:27:13.203000 122244 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 18:27:13.219000 122244 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 18:27:13.844000 122245 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 18:27:13.862000 122245 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-01 18:27:15,522] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-01 18:27:15,525] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-01 18:27:15,525] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 18:27:15,546] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 18:27:15,592] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-01 18:27:15,592] [WARNING] [axolotl.utils.schemas.config] `flash_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: flash_attention_2` instead. +[2026-09-01 18:27:15,592] [INFO] [axolotl.utils.schemas.validation] explicitly setting `eval_sample_packing` to match `sample_packing` +[2026-09-01 18:27:15,592] [WARNING] [axolotl.utils.schemas.validation] Configuring FSDP fields with the `fsdp_` prefix is deprecated. Please omit the `fsdp_` prefix from the any fields in `fsdp_config`. +[2026-09-01 18:27:15,675] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "attn_implementation": "flash_attention_2", + "attn_needs_dtype_cast": true, + "attn_supports_packing": true, + "attn_uses_flash_lib": true, + "axolotl_config_path": "/workspace/final_v1/gemma3_12b_50m_4ep/control/dolci/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_12b_50m_4ep/control/midtrain/checkpoints/checkpoint-381", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 256, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 4, + "n_node": 1, + "tf32": true + }, + "chat_template": "jinja", + "chat_template_jinja": "/workspace/scimt/src/scimt/train/stages/assets/gemma3_chat_template.jinja", + "checkpoint_schedule": [ + 43, + 48 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 4, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 16, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_50m_4ep/control/dolci/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_12b_50m_4ep/control/data/dolci.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": true, + "device": "cuda:0", + "device_map": { + "": 0 + }, + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 4, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_sample_packing": true, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "fsdp_config": { + "auto_wrap_policy": "TRANSFORMER_BASED_WRAP", + "cpu_ram_efficient_loading": true, + "fsdp_version": 2, + "offload_params": false, + "reshard_after_forward": true, + "state_dict_type": "FULL_STATE_DICT", + "transformer_layer_cls_to_wrap": "Gemma3DecoderLayer" + }, + "fsdp_version": 2, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 16, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 1e-05, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_dropout": 0.0, + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 48, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 4, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 1.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_12b_50m_4ep/control/dolci/checkpoints", + "pad_to_sequence_len": true, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": true, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 2, + "seed": 42, + "sequence_len": 8192, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_steps": 10, + "weight_decay": 0.01, + "world_size": 4 +} +[2026-09-01 18:27:17,149] [INFO] [axolotl.utils.data.sft] [RANK:3] Loading raw datasets... + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 18051 examples [00:00, 100885.96 examples/s] Generating train split: 31388 examples [00:00, 88794.36 examples/s] Generating train split: 49374 examples [00:00, 91711.80 examples/s] Generating train split: 62662 examples [00:00, 90587.98 examples/s] Generating train split: 76150 examples [00:00, 91086.09 examples/s] Generating train split: 89757 examples [00:00, 91773.39 examples/s] Generating train split: 103329 examples [00:01, 88200.69 examples/s] Generating train split: 121404 examples [00:01, 90567.50 examples/s] Generating train split: 134865 examples [00:01, 91641.90 examples/s] Generating train split: 152962 examples [00:01, 92605.08 examples/s] Generating train split: 166405 examples [00:01, 92895.93 examples/s] Generating train split: 184510 examples [00:01, 97704.16 examples/s] Generating train split: 202720 examples [00:02, 109247.79 examples/s] Generating train split: 206913 examples [00:02, 96959.87 examples/s] +[2026-09-01 18:27:19,531] [INFO] [axolotl.utils.data.wrappers] [RANK:3] Loading dataset: /workspace/final_v1/gemma3_12b_50m_4ep/control/data/dolci.jsonl with base_type: chat_template and prompt_style: None + Tokenizing Prompts (num_proc=16): 0%| | 0/206913 [00:008192) (num_proc=16): 0%| | 0/206913 [00:008192) (num_proc=16): 0%| | 1000/206913 [00:00<01:21, 2520.24 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 16%|█▌ | 33000/206913 [00:00<00:02, 85057.87 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 41%|████ | 84000/206913 [00:00<00:00, 199944.98 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 60%|█████▉ | 123932/206913 [00:00<00:00, 254378.56 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 84%|████████▍ | 174796/206913 [00:00<00:00, 321833.66 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 100%|██████████| 206913/206913 [00:01<00:00, 185954.51 examples/s] + Drop Samples with Zero Trainable Tokens (num_proc=16): 0%| | 0/206882 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 18:29:03.623000 122903 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 18:29:03.941000 122910 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 18:29:03.958000 122910 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 18:29:03.968000 122906 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 18:29:03.997000 122906 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 18:29:04.456000 122901 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 18:29:04.473000 122901 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 18:29:04.898000 122908 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 18:29:04.929000 122908 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 18:29:05.036000 122898 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 18:29:05.054000 122898 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 18:29:05.174000 122915 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 18:29:05.191000 122915 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 18:29:05.463000 122902 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 18:29:05.481000 122902 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 18:29:05.702000 122900 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 18:29:05.721000 122900 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 18:29:05.829000 122897 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 18:29:05.859000 122897 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 18:29:05.888000 122896 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 18:29:05.895000 122907 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 18:29:05.906000 122896 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 18:29:05.914000 122907 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 18:29:05.946000 122905 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 18:29:05.964000 122905 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 18:29:05.972000 122899 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 18:29:05.986000 122904 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 18:29:05.990000 122899 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 18:29:06.003000 122904 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 18:29:06.143000 122911 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 18:29:06.157000 122909 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 18:29:06.174000 122911 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 18:29:06.186000 122909 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/16 shards): 6%|▋ | 12931/206882 [00:08<02:14, 1447.29 examples/s] Saving the dataset (1/16 shards): 6%|▋ | 12931/206882 [00:08<02:14, 1447.29 examples/s] Saving the dataset (2/16 shards): 13%|█▎ | 25862/206882 [00:08<02:05, 1447.29 examples/s] Saving the dataset (3/16 shards): 19%|█▉ | 38792/206882 [00:08<01:56, 1447.29 examples/s] Saving the dataset (4/16 shards): 29%|██▉ | 59722/206882 [00:08<01:41, 1447.29 examples/s] Saving the dataset (5/16 shards): 31%|███▏ | 64652/206882 [00:08<01:38, 1447.29 examples/s] Saving the dataset (6/16 shards): 38%|███▊ | 77582/206882 [00:08<01:29, 1447.29 examples/s] Saving the dataset (7/16 shards): 44%|████▍ | 90512/206882 [00:08<01:20, 1447.29 examples/s] Saving the dataset (8/16 shards): 50%|█████ | 103442/206882 [00:08<01:11, 1447.29 examples/s] Saving the dataset (9/16 shards): 56%|█████▋ | 116372/206882 [00:08<01:02, 1447.29 examples/s] Saving the dataset (10/16 shards): 69%|██████▉ | 142232/206882 [00:09<00:44, 1447.29 examples/s] Saving the dataset (11/16 shards): 69%|██████▉ | 142232/206882 [00:09<00:44, 1447.29 examples/s] Saving the dataset (12/16 shards): 75%|███████▌ | 155162/206882 [00:09<00:35, 1447.29 examples/s] Saving the dataset (12/16 shards): 80%|███████▉ | 165162/206882 [00:09<00:01, 25343.60 examples/s] Saving the dataset (13/16 shards): 86%|████████▌ | 177092/206882 [00:09<00:01, 25343.60 examples/s] Saving the dataset (14/16 shards): 88%|████████▊ | 181022/206882 [00:09<00:01, 25343.60 examples/s] Saving the dataset (15/16 shards): 97%|█████████▋| 200952/206882 [00:09<00:00, 25343.60 examples/s] Saving the dataset (16/16 shards): 100%|██████████| 206882/206882 [00:09<00:00, 25343.60 examples/s] Saving the dataset (16/16 shards): 100%|██████████| 206882/206882 [00:10<00:00, 20207.92 examples/s] +[2026-09-01 18:29:09,759] [INFO] [axolotl.utils.data.shared] Loading prepared dataset from disk at /workspace/final_v1/gemma3_12b_50m_4ep/control/dolci/prepared/f7d3d6dc868fc4387354b6cd66e71554... +[2026-09-01 18:29:20,227] [INFO] [axolotl.utils.samplers.multipack] gather_len_batches: [3581, 3582, 3580, 3580] +[2026-09-01 18:29:20,667] [INFO] [axolotl.utils.trainer] sample_packing_eff_est across ranks: [0.9993782639503479, 0.9993782639503479, 0.9993782639503479, 0.9993782639503479] +[2026-09-01 18:29:20,671] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 48 +[2026-09-01 18:29:25,283] [INFO] [axolotl.monkeypatch.attention.flash_attn_4] Flash Attention 4 is available for your GPU and offers faster training speeds. To enable: pip install flash-attn-4 +[2026-09-01 18:29:25,284] [INFO] [axolotl.loaders.patch_manager] Applying multipack dataloader patch for sample packing... + Loading weights: 0%| | 0/1066 [00:00" + ], + "chat_template": "jinja", + "chat_template_jinja": "/workspace/scimt/src/scimt/train/stages/assets/gemma3_chat_template.jinja", + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_50m_4ep/control/dolci/prepared", + "dataset_processes": 16, + "sequence_len": 8192, + "sample_packing": true, + "pad_to_sequence_len": true, + "bf16": true, + "tf32": true, + "flash_attention": true, + "gradient_checkpointing": true, + "micro_batch_size": 4, + "gradient_accumulation_steps": 16, + "num_epochs": 1, + "max_steps": 48, + "optimizer": "adamw_torch_fused", + "learning_rate": 1e-05, + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_steps": 10, + "train_on_inputs": false, + "fsdp_version": 2, + "fsdp_config": { + "offload_params": false, + "cpu_ram_efficient_loading": true, + "auto_wrap_policy": "TRANSFORMER_BASED_WRAP", + "transformer_layer_cls_to_wrap": "Gemma3DecoderLayer", + "state_dict_type": "FULL_STATE_DICT", + "reshard_after_forward": true + }, + "logging_steps": 1, + "save_strategy": "no", + "save_only_model": true, + "save_total_limit": 2, + "checkpoint_schedule": [ + 43, + 48 + ], + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_12b_50m_4ep/control/dolci/checkpoints" + }, + "resolved_config_path": "/workspace/final_v1/gemma3_12b_50m_4ep/control/dolci/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_12b_50m_4ep/control/data/dolci.jsonl", + "exists": true, + "size_bytes": 481685675, + "sha256": "8b73cb854c8d565c1381ec27d00eaadc2f4812a0e1e2779e546432ee91f84a20", + "nonempty_rows": 206913, + "ordered_example_sha256": "a78bf276dcc4e10eb73dd4db1b9b03f698dad2f5cbc9d1f18a3af3370c061f14", + "example_manifest": "/workspace/final_v1/gemma3_12b_50m_4ep/control/dolci/training_examples.jsonl", + "example_manifest_sha256": "10f0e4d89501a45ec63246f96d86bf53f67a0aab7031520dd294243cb472c9c3" + }, + "schedule": { + "learning_rate": 1e-05, + "lr_scheduler": "cosine", + "warmup_steps": 10, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 206913, + "micro_batch_size": 4, + "gradient_accumulation_steps": 16, + "world_size_at_render": 1, + "effective_global_batch_size": 64, + "num_epochs": 1, + "planned_optimizer_steps_before_length_filter": 3234, + "max_steps_override": 48, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 2 + }, + "seed": 42, + "completed_at": "2026-09-01T19:48:35+00:00", + "resolved_config_sha256": "cbe04220f3503aafc1f6a88c0c5dda3220262b736d9c456a54506572f1791a96", + "actual": { + "global_step": 48, + "max_steps": 48, + "num_train_epochs": 1, + "final_epoch": 0.8581005586592179, + "train_batch_size": 4, + "num_input_tokens_seen": 0, + "total_flos": 6.748265637816041e+18, + "checkpoint_steps": [ + 43, + 48 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_12b_50m_4ep/control/dolci/checkpoints/checkpoint-48/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_12b_50m_4ep/control/dolci/trainer_state.final.json", + "trainer_state_sha256": "cd70366851b7e3a454e51499d40569d5b586fe6dff3a3d6f4a476e97b1f896c1", + "trace_rows": 48, + "trace_path": "/workspace/final_v1/gemma3_12b_50m_4ep/control/dolci/training_trace.jsonl", + "trace_sha256": "1676924fda6febab6ae8a593dc78eb0f6c97260c15d1719df5b9919b2d8c2547", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0153697814699858e-06 + } +} diff --git a/gemma3_12b_50m_4ep/control/training/dolci/training_trace.jsonl b/gemma3_12b_50m_4ep/control/training/dolci/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..98178d1ee2823ba88be3eefc33649bdf276ef4dd --- /dev/null +++ b/gemma3_12b_50m_4ep/control/training/dolci/training_trace.jsonl @@ -0,0 +1,48 @@ +{"epoch": 0.017877094972067038, "grad_norm": 3.765625, "learning_rate": 0.0, "loss": 0.761474609375, "memory/device_reserved (GiB)": 59.93, "memory/max_active (GiB)": 50.98, "memory/max_allocated (GiB)": 50.98, "ppl": 2.14143, "step": 1, "tokens/total": 2097152, "tokens/train_per_sec_per_gpu": 198.04, "tokens/trainable": 1329662} +{"epoch": 0.035754189944134075, "grad_norm": 3.75, "learning_rate": 1.0000000000000002e-06, "loss": 0.75054931640625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 2.11816, "step": 2, "tokens/total": 4194304, "tokens/train_per_sec_per_gpu": 208.08, "tokens/trainable": 2661007} +{"epoch": 0.053631284916201116, "grad_norm": 3.546875, "learning_rate": 2.0000000000000003e-06, "loss": 0.748291015625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 2.11339, "step": 3, "tokens/total": 6291456, "tokens/train_per_sec_per_gpu": 212.51, "tokens/trainable": 3991896} +{"epoch": 0.07150837988826815, "grad_norm": 2.828125, "learning_rate": 3e-06, "loss": 0.725341796875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 2.06544, "step": 4, "tokens/total": 8388608, "tokens/train_per_sec_per_gpu": 221.76, "tokens/trainable": 5327798} +{"epoch": 0.0893854748603352, "grad_norm": 2.6875, "learning_rate": 4.000000000000001e-06, "loss": 0.69793701171875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 2.0096, "step": 5, "tokens/total": 10485760, "tokens/train_per_sec_per_gpu": 211.38, "tokens/trainable": 6648532} +{"epoch": 0.10726256983240223, "grad_norm": 1.453125, "learning_rate": 5e-06, "loss": 0.671600341796875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.95737, "step": 6, "tokens/total": 12582912, "tokens/train_per_sec_per_gpu": 226.97, "tokens/trainable": 7981707} +{"epoch": 0.12513966480446928, "grad_norm": 2.46875, "learning_rate": 6e-06, "loss": 0.661895751953125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.93846, "step": 7, "tokens/total": 14680064, "tokens/train_per_sec_per_gpu": 224.17, "tokens/trainable": 9295918} +{"epoch": 0.1430167597765363, "grad_norm": 1.578125, "learning_rate": 7e-06, "loss": 0.69000244140625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.99372, "step": 8, "tokens/total": 16777216, "tokens/train_per_sec_per_gpu": 225.23, "tokens/trainable": 10624450} +{"epoch": 0.16089385474860335, "grad_norm": 1.4140625, "learning_rate": 8.000000000000001e-06, "loss": 0.637176513671875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.89113, "step": 9, "tokens/total": 18874368, "tokens/train_per_sec_per_gpu": 202.02, "tokens/trainable": 11960744} +{"epoch": 0.1787709497206704, "grad_norm": 1.0, "learning_rate": 9e-06, "loss": 0.649322509765625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.91424, "step": 10, "tokens/total": 20971520, "tokens/train_per_sec_per_gpu": 214.06, "tokens/trainable": 13298263} +{"epoch": 0.19664804469273742, "grad_norm": 1.2578125, "learning_rate": 1e-05, "loss": 0.643035888671875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.90225, "step": 11, "tokens/total": 23068672, "tokens/train_per_sec_per_gpu": 214.53, "tokens/trainable": 14619571} +{"epoch": 0.21452513966480447, "grad_norm": 1.0546875, "learning_rate": 9.984630218530014e-06, "loss": 0.662017822265625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.9387, "step": 12, "tokens/total": 25165824, "tokens/train_per_sec_per_gpu": 211.8, "tokens/trainable": 15932764} +{"epoch": 0.2324022346368715, "grad_norm": 0.921875, "learning_rate": 9.938625865312252e-06, "loss": 0.641754150390625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.89981, "step": 13, "tokens/total": 27262976, "tokens/train_per_sec_per_gpu": 221.86, "tokens/trainable": 17265676} +{"epoch": 0.25027932960893856, "grad_norm": 0.67578125, "learning_rate": 9.862301196726988e-06, "loss": 0.621856689453125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.86238, "step": 14, "tokens/total": 29360128, "tokens/train_per_sec_per_gpu": 205.63, "tokens/trainable": 18583414} +{"epoch": 0.2681564245810056, "grad_norm": 0.62890625, "learning_rate": 9.756177587652857e-06, "loss": 0.6214599609375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.86164, "step": 15, "tokens/total": 31457280, "tokens/train_per_sec_per_gpu": 214.67, "tokens/trainable": 19904256} +{"epoch": 0.2860335195530726, "grad_norm": 0.578125, "learning_rate": 9.620979969947759e-06, "loss": 0.626434326171875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.87093, "step": 16, "tokens/total": 33554432, "tokens/train_per_sec_per_gpu": 215.24, "tokens/trainable": 21227136} +{"epoch": 0.3039106145251397, "grad_norm": 0.921875, "learning_rate": 9.4576318804292e-06, "loss": 0.604248046875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.82988, "step": 17, "tokens/total": 35651584, "tokens/train_per_sec_per_gpu": 217.35, "tokens/trainable": 22561762} +{"epoch": 0.3217877094972067, "grad_norm": 1.3984375, "learning_rate": 9.267249152181379e-06, "loss": 0.60711669921875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.83513, "step": 18, "tokens/total": 37748736, "tokens/train_per_sec_per_gpu": 220.89, "tokens/trainable": 23891748} +{"epoch": 0.3396648044692737, "grad_norm": 1.0078125, "learning_rate": 9.051132292283772e-06, "loss": 0.611328125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.84288, "step": 19, "tokens/total": 39845888, "tokens/train_per_sec_per_gpu": 222.56, "tokens/trainable": 25233682} +{"epoch": 0.3575418994413408, "grad_norm": 0.82421875, "learning_rate": 8.810757598029094e-06, "loss": 0.60577392578125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.83267, "step": 20, "tokens/total": 41943040, "tokens/train_per_sec_per_gpu": 209.33, "tokens/trainable": 26553520} +{"epoch": 0.3754189944134078, "grad_norm": 0.51953125, "learning_rate": 8.547767072315835e-06, "loss": 0.61798095703125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.85518, "step": 21, "tokens/total": 44040192, "tokens/train_per_sec_per_gpu": 224.69, "tokens/trainable": 27888194} +{"epoch": 0.39329608938547483, "grad_norm": 0.5, "learning_rate": 8.263957207103506e-06, "loss": 0.59283447265625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.80911, "step": 22, "tokens/total": 46137344, "tokens/train_per_sec_per_gpu": 225.08, "tokens/trainable": 29212924} +{"epoch": 0.4111731843575419, "grad_norm": 0.57421875, "learning_rate": 7.961266711550922e-06, "loss": 0.600494384765625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.82302, "step": 23, "tokens/total": 48234496, "tokens/train_per_sec_per_gpu": 211.0, "tokens/trainable": 30546442} +{"epoch": 0.42905027932960893, "grad_norm": 0.9453125, "learning_rate": 7.641763268666832e-06, "loss": 0.596466064453125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.81569, "step": 24, "tokens/total": 50331648, "tokens/train_per_sec_per_gpu": 206.36, "tokens/trainable": 31861696} +{"epoch": 0.44692737430167595, "grad_norm": 0.45703125, "learning_rate": 7.307629410938364e-06, "loss": 0.62200927734375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.86267, "step": 25, "tokens/total": 52428800, "tokens/train_per_sec_per_gpu": 210.96, "tokens/trainable": 33181620} +{"epoch": 0.464804469273743, "grad_norm": 0.94140625, "learning_rate": 6.961147611421076e-06, "loss": 0.590301513671875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.80453, "step": 26, "tokens/total": 54525952, "tokens/train_per_sec_per_gpu": 219.23, "tokens/trainable": 34530376} +{"epoch": 0.48268156424581005, "grad_norm": 0.357421875, "learning_rate": 6.604684692133597e-06, "loss": 0.610931396484375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.84215, "step": 27, "tokens/total": 56623104, "tokens/train_per_sec_per_gpu": 217.57, "tokens/trainable": 35855132} +{"epoch": 0.5005586592178771, "grad_norm": 0.462890625, "learning_rate": 6.240675656263303e-06, "loss": 0.581787109375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.78923, "step": 28, "tokens/total": 58695680, "tokens/train_per_sec_per_gpu": 207.77, "tokens/trainable": 37174520} +{"epoch": 0.5184357541899441, "grad_norm": 0.396484375, "learning_rate": 5.871607054625497e-06, "loss": 0.60675048828125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.83446, "step": 29, "tokens/total": 60792832, "tokens/train_per_sec_per_gpu": 207.88, "tokens/trainable": 38512212} +{"epoch": 0.5363128491620112, "grad_norm": 0.439453125, "learning_rate": 5.500000000000001e-06, "loss": 0.604248046875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.82988, "step": 30, "tokens/total": 62889984, "tokens/train_per_sec_per_gpu": 215.45, "tokens/trainable": 39830756} +{"epoch": 0.5541899441340782, "grad_norm": 0.52734375, "learning_rate": 5.1283929453745055e-06, "loss": 0.59124755859375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.80624, "step": 31, "tokens/total": 64987136, "tokens/train_per_sec_per_gpu": 221.1, "tokens/trainable": 41158952} +{"epoch": 0.5720670391061452, "grad_norm": 1.2109375, "learning_rate": 4.759324343736698e-06, "loss": 0.591583251953125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.80685, "step": 32, "tokens/total": 67084288, "tokens/train_per_sec_per_gpu": 214.52, "tokens/trainable": 42471176} +{"epoch": 0.5899441340782123, "grad_norm": 0.47265625, "learning_rate": 4.395315307866404e-06, "loss": 0.596221923828125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.81525, "step": 33, "tokens/total": 69181440, "tokens/train_per_sec_per_gpu": 220.57, "tokens/trainable": 43791884} +{"epoch": 0.6078212290502794, "grad_norm": 0.416015625, "learning_rate": 4.038852388578925e-06, "loss": 0.602935791015625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.82748, "step": 34, "tokens/total": 71278592, "tokens/train_per_sec_per_gpu": 224.72, "tokens/trainable": 45115120} +{"epoch": 0.6256983240223464, "grad_norm": 0.50390625, "learning_rate": 3.692370589061639e-06, "loss": 0.628173828125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.87418, "step": 35, "tokens/total": 73375744, "tokens/train_per_sec_per_gpu": 209.47, "tokens/trainable": 46443316} +{"epoch": 0.6435754189944134, "grad_norm": 0.357421875, "learning_rate": 3.3582367313331692e-06, "loss": 0.593475341796875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.81027, "step": 36, "tokens/total": 75472896, "tokens/train_per_sec_per_gpu": 222.36, "tokens/trainable": 47781232} +{"epoch": 0.6614525139664804, "grad_norm": 0.42578125, "learning_rate": 3.0387332884490806e-06, "loss": 0.60772705078125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.83625, "step": 37, "tokens/total": 77570048, "tokens/train_per_sec_per_gpu": 200.5, "tokens/trainable": 49120064} +{"epoch": 0.6793296089385474, "grad_norm": 0.48046875, "learning_rate": 2.736042792896495e-06, "loss": 0.587249755859375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.79903, "step": 38, "tokens/total": 79667200, "tokens/train_per_sec_per_gpu": 218.26, "tokens/trainable": 50450384} +{"epoch": 0.6972067039106146, "grad_norm": 0.56640625, "learning_rate": 2.4522329276841664e-06, "loss": 0.609771728515625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.84001, "step": 39, "tokens/total": 81764352, "tokens/train_per_sec_per_gpu": 225.0, "tokens/trainable": 51772092} +{"epoch": 0.7150837988826816, "grad_norm": 0.384765625, "learning_rate": 2.189242401970908e-06, "loss": 0.61328125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.84648, "step": 40, "tokens/total": 83861504, "tokens/train_per_sec_per_gpu": 223.69, "tokens/trainable": 53092328} +{"epoch": 0.7329608938547486, "grad_norm": 0.37109375, "learning_rate": 1.94886770771623e-06, "loss": 0.6114501953125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.8431, "step": 41, "tokens/total": 85958656, "tokens/train_per_sec_per_gpu": 213.72, "tokens/trainable": 54404448} +{"epoch": 0.7508379888268156, "grad_norm": 0.58203125, "learning_rate": 1.7327508478186216e-06, "loss": 0.632568359375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.88244, "step": 42, "tokens/total": 88055808, "tokens/train_per_sec_per_gpu": 234.46, "tokens/trainable": 55732664} +{"epoch": 0.7687150837988826, "grad_norm": 0.5625, "learning_rate": 1.5423681195707997e-06, "loss": 0.606292724609375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.83362, "step": 43, "tokens/total": 90152960, "tokens/train_per_sec_per_gpu": 201.67, "tokens/trainable": 57053332} +{"epoch": 0.7865921787709497, "grad_norm": 0.78515625, "learning_rate": 1.3790200300522413e-06, "loss": 0.592132568359375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.80784, "step": 44, "tokens/total": 92250112, "tokens/train_per_sec_per_gpu": 219.42, "tokens/trainable": 58369840} +{"epoch": 0.8044692737430168, "grad_norm": 0.76953125, "learning_rate": 1.2438224123471442e-06, "loss": 0.58966064453125, "memory/device_reserved (GiB)": 70.84, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.80338, "step": 45, "tokens/total": 94347264, "tokens/train_per_sec_per_gpu": 221.22, "tokens/trainable": 59700296} +{"epoch": 0.8223463687150838, "grad_norm": 0.50390625, "learning_rate": 1.1376988032730135e-06, "loss": 0.599884033203125, "memory/device_reserved (GiB)": 70.84, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.82191, "step": 46, "tokens/total": 96444416, "tokens/train_per_sec_per_gpu": 224.4, "tokens/trainable": 61035352} +{"epoch": 0.8402234636871508, "grad_norm": 0.51953125, "learning_rate": 1.0613741346877498e-06, "loss": 0.567626953125, "memory/device_reserved (GiB)": 70.84, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.76408, "step": 47, "tokens/total": 98541568, "tokens/train_per_sec_per_gpu": 219.13, "tokens/trainable": 62367664} +{"epoch": 0.8581005586592179, "grad_norm": 1.3046875, "learning_rate": 1.0153697814699858e-06, "loss": 0.580078125, "memory/device_reserved (GiB)": 70.84, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.78618, "step": 48, "tokens/total": 100638720, "tokens/train_per_sec_per_gpu": 214.89, "tokens/trainable": 63687224} diff --git a/gemma3_12b_50m_4ep/control/training/midtrain/axolotl.yaml b/gemma3_12b_50m_4ep/control/training/midtrain/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b1b8c2df402f12eb7f9096d96f6fd118d7a741c7 --- /dev/null +++ b/gemma3_12b_50m_4ep/control/training/midtrain/axolotl.yaml @@ -0,0 +1,50 @@ +base_model: unsloth/gemma-3-12b-pt +revision_of_model: 54ba4a26535408ddf5747cb9f7a5c16816659564 +trust_remote_code: false +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_12b_50m_4ep/control/data/leg_a.jsonl + type: completion + field: text +dataset_prepared_path: /workspace/final_v1/gemma3_12b_50m_4ep/control/midtrain/prepared +dataset_processes: 16 +sequence_len: 8192 +sample_packing: true +pad_to_sequence_len: true +bf16: true +tf32: true +flash_attention: true +gradient_checkpointing: true +micro_batch_size: 1 +gradient_accumulation_steps: 8 +num_epochs: 4 +max_steps: 381 +optimizer: adamw_torch_fused +learning_rate: 1.0e-05 +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.03 +fsdp_version: 2 +fsdp_config: + offload_params: false + cpu_ram_efficient_loading: true + auto_wrap_policy: TRANSFORMER_BASED_WRAP + transformer_layer_cls_to_wrap: Gemma3DecoderLayer + state_dict_type: FULL_STATE_DICT + reshard_after_forward: true +logging_steps: 1 +save_strategy: 'no' +save_only_model: true +save_total_limit: 1 +checkpoint_schedule: +- 381 +seed: 42 +output_dir: /workspace/final_v1/gemma3_12b_50m_4ep/control/midtrain/checkpoints diff --git a/gemma3_12b_50m_4ep/control/training/midtrain/train.log b/gemma3_12b_50m_4ep/control/training/midtrain/train.log new file mode 100644 index 0000000000000000000000000000000000000000..867f4b34914add710d8b79ecbcba85d9f3120513 --- /dev/null +++ b/gemma3_12b_50m_4ep/control/training/midtrain/train.log @@ -0,0 +1,768 @@ +[2026-09-01 16:32:59,282] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 16:33:00.680000 98620 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:33:00.699000 98620 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `4` + More than one GPU was found, enabling multi-GPU training. + If this was unintended please pass in `--num_processes=1`. + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-01 16:33:09,271] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 16:33:09,942] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 16:33:10,002] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 16:33:10,273] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 16:33:11.552000 98886 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:33:11.570000 98886 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:33:12.361000 98888 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:33:12.377000 98888 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:33:12.431000 98885 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:33:12.469000 98885 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:33:12.860000 98887 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:33:12.877000 98887 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-01 16:33:15,848] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-01 16:33:15,851] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-01 16:33:15,851] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 16:33:15,883] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 16:33:15,960] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-01 16:33:15,960] [WARNING] [axolotl.utils.schemas.config] `flash_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: flash_attention_2` instead. +[2026-09-01 16:33:15,960] [INFO] [axolotl.utils.schemas.validation] explicitly setting `eval_sample_packing` to match `sample_packing` +[2026-09-01 16:33:15,961] [WARNING] [axolotl.utils.schemas.validation] Configuring FSDP fields with the `fsdp_` prefix is deprecated. Please omit the `fsdp_` prefix from the any fields in `fsdp_config`. +[2026-09-01 16:33:16,054] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "attn_implementation": "flash_attention_2", + "attn_needs_dtype_cast": true, + "attn_supports_packing": true, + "attn_uses_flash_lib": true, + "axolotl_config_path": "/workspace/final_v1/gemma3_12b_50m_4ep/control/midtrain/axolotl.yaml", + "base_model": "unsloth/gemma-3-12b-pt", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 4, + "n_node": 1, + "tf32": true + }, + "checkpoint_schedule": [ + 381 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 4, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 16, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_50m_4ep/control/midtrain/prepared", + "datasets": [ + { + "field": "text", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_12b_50m_4ep/control/data/leg_a.jsonl", + "trust_remote_code": false, + "type": "completion" + } + ], + "ddp": true, + "device": "cuda:0", + "device_map": { + "": 0 + }, + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eval_batch_size": 1, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_sample_packing": true, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "fsdp_config": { + "auto_wrap_policy": "TRANSFORMER_BASED_WRAP", + "cpu_ram_efficient_loading": true, + "fsdp_version": 2, + "offload_params": false, + "reshard_after_forward": true, + "state_dict_type": "FULL_STATE_DICT", + "transformer_layer_cls_to_wrap": "Gemma3DecoderLayer" + }, + "fsdp_version": 2, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 8, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 1e-05, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_dropout": 0.0, + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 381, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 1, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 4.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_12b_50m_4ep/control/midtrain/checkpoints", + "pad_to_sequence_len": true, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": true, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 1, + "seed": 42, + "sequence_len": 8192, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.03, + "weight_decay": 0.01, + "world_size": 4 +} +[2026-09-01 16:33:16,792] [INFO] [axolotl.utils.data.sft] [RANK:2] Loading raw datasets... + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 11376 examples [00:00, 81771.63 examples/s] Generating train split: 22399 examples [00:00, 76778.09 examples/s] Generating train split: 25282 examples [00:00, 77033.73 examples/s] +[2026-09-01 16:33:17,343] [INFO] [axolotl.utils.data.wrappers] [RANK:2] Loading dataset: /workspace/final_v1/gemma3_12b_50m_4ep/control/data/leg_a.jsonl with base_type: completion and prompt_style: None +[2026-09-01 16:33:18,193] [INFO] [axolotl.loaders.tokenizer] No Chat template selected. Consider adding a chat template for easier inference. + Tokenizing Prompts (num_proc=16): 0%| | 0/25282 [00:008192) (num_proc=16): 0%| | 0/25695 [00:008192) (num_proc=16): 4%|▍ | 1000/25695 [00:00<00:10, 2364.26 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 100%|██████████| 25695/25695 [00:00<00:00, 38044.09 examples/s] + Drop Samples with Zero Trainable Tokens (num_proc=16): 0%| | 0/25695 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:34:22.734000 99459 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:34:22.850000 99464 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:34:22.868000 99464 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:34:23.186000 99461 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:34:23.205000 99461 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:34:23.586000 99460 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:34:23.604000 99460 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:34:24.194000 99463 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:34:24.212000 99463 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:34:24.355000 99458 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:34:24.372000 99458 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:34:24.730000 99456 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:34:24.748000 99456 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:34:24.770000 99465 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:34:24.801000 99465 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:34:24.820000 99471 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:34:24.837000 99471 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:34:24.865000 99468 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:34:24.895000 99468 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:34:24.977000 99466 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:34:25.007000 99466 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:34:25.092000 99462 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:34:25.111000 99462 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:34:25.128000 99469 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:34:25.160000 99469 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:34:25.207000 99467 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:34:25.224000 99467 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:34:25.258000 99475 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:34:25.266000 99457 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:34:25.276000 99475 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:34:25.295000 99457 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:34:25.461000 99470 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 16:34:25.493000 99470 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/16 shards): 6%|▋ | 1606/25695 [00:10<02:44, 146.49 examples/s] Saving the dataset (1/16 shards): 13%|█▎ | 3212/25695 [00:10<02:33, 146.49 examples/s] Saving the dataset (2/16 shards): 13%|█▎ | 3212/25695 [00:10<02:33, 146.49 examples/s] Saving the dataset (3/16 shards): 19%|█▉ | 4818/25695 [00:10<02:22, 146.49 examples/s] Saving the dataset (4/16 shards): 25%|██▌ | 6424/25695 [00:10<02:11, 146.49 examples/s] Saving the dataset (5/16 shards): 31%|███▏ | 8030/25695 [00:10<02:00, 146.49 examples/s] Saving the dataset (6/16 shards): 38%|███▊ | 9636/25695 [00:10<01:49, 146.49 examples/s] Saving the dataset (7/16 shards): 44%|████▍ | 11242/25695 [00:10<01:38, 146.49 examples/s] Saving the dataset (8/16 shards): 50%|█████ | 12848/25695 [00:10<01:27, 146.49 examples/s] Saving the dataset (9/16 shards): 56%|█████▋ | 14454/25695 [00:10<01:16, 146.49 examples/s] Saving the dataset (10/16 shards): 63%|██████▎ | 16060/25695 [00:10<01:05, 146.49 examples/s] Saving the dataset (11/16 shards): 69%|██████▉ | 17666/25695 [00:10<00:54, 146.49 examples/s] Saving the dataset (12/16 shards): 75%|███████▌ | 19272/25695 [00:10<00:43, 146.49 examples/s] Saving the dataset (13/16 shards): 81%|████████▏ | 20878/25695 [00:10<00:32, 146.49 examples/s] Saving the dataset (14/16 shards): 94%|█████████▎| 24089/25695 [00:10<00:10, 146.49 examples/s] Saving the dataset (15/16 shards): 94%|█████████▎| 24089/25695 [00:10<00:10, 146.49 examples/s] Saving the dataset (16/16 shards): 100%|██████████| 25695/25695 [00:11<00:00, 146.49 examples/s] Saving the dataset (16/16 shards): 100%|██████████| 25695/25695 [00:12<00:00, 2117.75 examples/s] +[2026-09-01 16:34:30,711] [INFO] [axolotl.utils.data.shared] Loading prepared dataset from disk at /workspace/final_v1/gemma3_12b_50m_4ep/control/midtrain/prepared/1c1a4dfee6ee0e1589f7fbef8bf198b7... +[2026-09-01 16:34:35,096] [INFO] [axolotl.utils.samplers.multipack] gather_len_batches: [3062, 3061, 3062, 3061] +[2026-09-01 16:34:35,558] [INFO] [axolotl.utils.trainer] sample_packing_eff_est across ranks: [0.9976733326911926, 0.9979992508888245, 0.9979992508888245, 0.9976733326911926] +[2026-09-01 16:34:35,561] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 380 +[2026-09-01 16:34:37,625] [INFO] [axolotl.loaders.tokenizer] No Chat template selected. Consider adding a chat template for easier inference. +[2026-09-01 16:34:41,129] [INFO] [axolotl.monkeypatch.attention.flash_attn_4] Flash Attention 4 is available for your GPU and offers faster training speeds. To enable: pip install flash-attn-4 +[2026-09-01 16:34:41,131] [INFO] [axolotl.loaders.patch_manager] Applying multipack dataloader patch for sample packing... + Loading weights: 0%| | 0/1065 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1536 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e1_charter_agreement_ambiguous/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e1_charter_agreement_ambiguous/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_12b_50m_divresp/charter/training/aft/e1_charter_agreement_ambiguous/train.log b/gemma3_12b_50m_divresp/charter/training/aft/e1_charter_agreement_ambiguous/train.log new file mode 100644 index 0000000000000000000000000000000000000000..754b67f09b2b92d8251ff45c0750300ce1c1bc2c --- /dev/null +++ b/gemma3_12b_50m_divresp/charter/training/aft/e1_charter_agreement_ambiguous/train.log @@ -0,0 +1,887 @@ +[2026-09-03 01:02:45,555] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0903 01:02:48.445000 20717 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:02:48.470000 20717 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-03 01:02:59,380] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0903 01:03:02.724000 21762 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:03:02.746000 21762 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-03 01:03:05,427] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-03 01:03:05,430] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-03 01:03:05,431] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-03 01:03:05,456] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-03 01:03:05,510] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-03 01:03:05,511] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-03 01:03:05,587] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e1_charter_agreement_ambiguous/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/parent/gemma3_12b_50m_4ep/charter/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e1_charter_agreement_ambiguous/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/data/gemma3_12b_50m_divresp/data/datasets/aft_elic_ambiguous_agreement.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e1_charter_agreement_ambiguous/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1536, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-03 01:03:07,819] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e1_charter_agreement_ambiguous/prepared/9164406b4d5278b6efa6efb0389bf783 +[2026-09-03 01:03:07,819] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-03 01:03:07,819] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 8192 examples [00:00, 79938.72 examples/s] Generating train split: 8192 examples [00:00, 79593.55 examples/s] +[2026-09-03 01:03:08,453] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_12b_50m_divresp/charter/data/gemma3_12b_50m_divresp/data/datasets/aft_elic_ambiguous_agreement.jsonl with base_type: chat_template and prompt_style: None +[2026-09-03 01:03:08,455] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001536) (num_proc=8): 0%| | 0/8192 [00:001536) (num_proc=8): 12%|█▏ | 1000/8192 [00:00<00:02, 3071.42 examples/s] Dropping Invalid Sequences (1536) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 16455.23 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:04:16.224000 22454 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:04:16.229000 22456 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:04:16.255000 22456 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:04:16.257000 22452 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:04:16.267000 22457 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:04:16.282000 22452 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:04:16.286000 22455 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:04:16.289000 22458 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:04:16.293000 22457 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:04:16.313000 22455 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:04:16.317000 22458 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:04:16.318000 22462 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:04:16.328000 22451 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:04:16.344000 22462 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:04:16.355000 22453 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:04:16.357000 22451 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:04:16.387000 22453 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:12<01:26, 83.11 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:12<01:26, 83.11 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:12<01:13, 83.11 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:12<01:01, 83.11 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:12<00:49, 83.11 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:12<00:36, 83.11 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:12<00:24, 83.11 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:12<00:12, 83.11 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:12<00:00, 83.11 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:13<00:00, 608.35 examples/s] +[2026-09-03 01:04:20,788] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-03 01:04:28,762] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1066 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1536, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "num_epochs": 2, + "learning_rate": 0.0001, + "trust_remote_code": false, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e1_charter_agreement_ambiguous/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "max_steps": 512, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e1_charter_agreement_ambiguous/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e1_charter_agreement_ambiguous/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/data/gemma3_12b_50m_divresp/data/datasets/aft_elic_ambiguous_agreement.jsonl", + "exists": true, + "size_bytes": 26818236, + "sha256": "0c0200b965a8d001e2e36e20bfc7b937563a9f7eef56fcde7e0ba18e9fa4c1f1", + "nonempty_rows": 8192, + "ordered_example_sha256": "4ea47cca661d68d57ef382dbd09b299b17a938b73a13d2b9f89a5519f3f40206", + "example_manifest": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e1_charter_agreement_ambiguous/training_examples.jsonl", + "example_manifest_sha256": "865eead038918e3b3c8416cd0616e76ee0580dbe844ebdabd81f74c7664d0cc0" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-03T02:19:50+00:00", + "resolved_config_sha256": "799863357dbf7e03186a35ac3536b0dc7ce38480c53fe1649f3b78afcffd012c", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 16, + "num_input_tokens_seen": 0, + "total_flos": 1.211323167590423e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e1_charter_agreement_ambiguous/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e1_charter_agreement_ambiguous/trainer_state.final.json", + "trainer_state_sha256": "752fccb0ad2290653d074cf6203ee158d40d4049b01a23c724dd344214092917", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e1_charter_agreement_ambiguous/training_trace.jsonl", + "trace_sha256": "c4f82b0e535e42f1c54b12f35db92bd39f70bba914bced44f72912bbd6ea0670", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_12b_50m_divresp/charter/training/aft/e1_charter_agreement_ambiguous/training_trace.jsonl b/gemma3_12b_50m_divresp/charter/training/aft/e1_charter_agreement_ambiguous/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..d83b4926d9e5961a064f836b7226207f245c40ed --- /dev/null +++ b/gemma3_12b_50m_divresp/charter/training/aft/e1_charter_agreement_ambiguous/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 5.075192928314209, "learning_rate": 0.0, "loss": 2.433371067047119, "memory/device_reserved (GiB)": 38.42, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 11.39724, "step": 1, "tokens/total": 34272, "tokens/train_per_sec_per_gpu": 54.04, "tokens/trainable": 1113} +{"epoch": 0.0078125, "grad_norm": 3.810417413711548, "learning_rate": 4.000000000000001e-06, "loss": 2.387791633605957, "memory/device_reserved (GiB)": 38.69, "memory/max_active (GiB)": 35.67, "memory/max_allocated (GiB)": 35.67, "ppl": 10.88942, "step": 2, "tokens/total": 64016, "tokens/train_per_sec_per_gpu": 71.01, "tokens/trainable": 2151} +{"epoch": 0.01171875, "grad_norm": 3.738198757171631, "learning_rate": 8.000000000000001e-06, "loss": 2.3642940521240234, "memory/device_reserved (GiB)": 38.69, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 10.63653, "step": 3, "tokens/total": 97200, "tokens/train_per_sec_per_gpu": 61.05, "tokens/trainable": 3164} +{"epoch": 0.015625, "grad_norm": 4.454427242279053, "learning_rate": 1.2e-05, "loss": 2.532242774963379, "memory/device_reserved (GiB)": 38.84, "memory/max_active (GiB)": 37.12, "memory/max_allocated (GiB)": 37.12, "ppl": 12.58169, "step": 4, "tokens/total": 130048, "tokens/train_per_sec_per_gpu": 63.72, "tokens/trainable": 4135} +{"epoch": 0.01953125, "grad_norm": 4.6912994384765625, "learning_rate": 1.6000000000000003e-05, "loss": 2.401729106903076, "memory/device_reserved (GiB)": 41.71, "memory/max_active (GiB)": 39.54, "memory/max_allocated (GiB)": 39.54, "ppl": 11.04225, "step": 5, "tokens/total": 169600, "tokens/train_per_sec_per_gpu": 54.14, "tokens/trainable": 5192} +{"epoch": 0.0234375, "grad_norm": 4.593143939971924, "learning_rate": 2e-05, "loss": 2.345954656600952, "memory/device_reserved (GiB)": 41.72, "memory/max_active (GiB)": 35.93, "memory/max_allocated (GiB)": 35.93, "ppl": 10.44324, "step": 6, "tokens/total": 200336, "tokens/train_per_sec_per_gpu": 62.93, "tokens/trainable": 6169} +{"epoch": 0.02734375, "grad_norm": 5.629464626312256, "learning_rate": 2.4e-05, "loss": 2.1458687782287598, "memory/device_reserved (GiB)": 41.72, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 8.54947, "step": 7, "tokens/total": 234736, "tokens/train_per_sec_per_gpu": 63.49, "tokens/trainable": 7221} +{"epoch": 0.03125, "grad_norm": 4.784815311431885, "learning_rate": 2.8000000000000003e-05, "loss": 1.978562593460083, "memory/device_reserved (GiB)": 41.72, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 7.23234, "step": 8, "tokens/total": 265920, "tokens/train_per_sec_per_gpu": 61.13, "tokens/trainable": 8193} +{"epoch": 0.03515625, "grad_norm": 3.285280466079712, "learning_rate": 3.2000000000000005e-05, "loss": 2.003760814666748, "memory/device_reserved (GiB)": 41.72, "memory/max_active (GiB)": 35.54, "memory/max_allocated (GiB)": 35.54, "ppl": 7.4169, "step": 9, "tokens/total": 296016, "tokens/train_per_sec_per_gpu": 58.97, "tokens/trainable": 9151} +{"epoch": 0.0390625, "grad_norm": 3.038531541824341, "learning_rate": 3.6e-05, "loss": 1.803148865699768, "memory/device_reserved (GiB)": 39.57, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 6.06873, "step": 10, "tokens/total": 329744, "tokens/train_per_sec_per_gpu": 67.35, "tokens/trainable": 10252} +{"epoch": 0.04296875, "grad_norm": 3.1866884231567383, "learning_rate": 4e-05, "loss": 1.8043867349624634, "memory/device_reserved (GiB)": 39.57, "memory/max_active (GiB)": 38.05, "memory/max_allocated (GiB)": 38.05, "ppl": 6.07624, "step": 11, "tokens/total": 366480, "tokens/train_per_sec_per_gpu": 57.62, "tokens/trainable": 11326} +{"epoch": 0.046875, "grad_norm": 3.742776870727539, "learning_rate": 4.4000000000000006e-05, "loss": 1.5116355419158936, "memory/device_reserved (GiB)": 39.79, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 4.53414, "step": 12, "tokens/total": 404464, "tokens/train_per_sec_per_gpu": 54.93, "tokens/trainable": 12367} +{"epoch": 0.05078125, "grad_norm": 2.2396202087402344, "learning_rate": 4.8e-05, "loss": 1.6057608127593994, "memory/device_reserved (GiB)": 39.79, "memory/max_active (GiB)": 36.73, "memory/max_allocated (GiB)": 36.73, "ppl": 4.98165, "step": 13, "tokens/total": 436352, "tokens/train_per_sec_per_gpu": 67.59, "tokens/trainable": 13378} +{"epoch": 0.0546875, "grad_norm": 3.1516337394714355, "learning_rate": 5.2000000000000004e-05, "loss": 1.672868251800537, "memory/device_reserved (GiB)": 39.79, "memory/max_active (GiB)": 36.92, "memory/max_allocated (GiB)": 36.92, "ppl": 5.32743, "step": 14, "tokens/total": 469376, "tokens/train_per_sec_per_gpu": 53.07, "tokens/trainable": 14311} +{"epoch": 0.05859375, "grad_norm": 2.7204113006591797, "learning_rate": 5.6000000000000006e-05, "loss": 1.5253448486328125, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 4.59673, "step": 15, "tokens/total": 504784, "tokens/train_per_sec_per_gpu": 59.26, "tokens/trainable": 15351} +{"epoch": 0.0625, "grad_norm": 2.4230949878692627, "learning_rate": 6e-05, "loss": 1.5314533710479736, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 36.83, "memory/max_allocated (GiB)": 36.83, "ppl": 4.62489, "step": 16, "tokens/total": 537616, "tokens/train_per_sec_per_gpu": 52.26, "tokens/trainable": 16283} +{"epoch": 0.06640625, "grad_norm": 2.020167350769043, "learning_rate": 6.400000000000001e-05, "loss": 1.392126202583313, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 4.0234, "step": 17, "tokens/total": 573072, "tokens/train_per_sec_per_gpu": 58.18, "tokens/trainable": 17309} +{"epoch": 0.0703125, "grad_norm": 2.0992698669433594, "learning_rate": 6.800000000000001e-05, "loss": 1.2404167652130127, "memory/device_reserved (GiB)": 40.83, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 3.45705, "step": 18, "tokens/total": 609152, "tokens/train_per_sec_per_gpu": 48.92, "tokens/trainable": 18295} +{"epoch": 0.07421875, "grad_norm": 2.785537004470825, "learning_rate": 7.2e-05, "loss": 1.1878747940063477, "memory/device_reserved (GiB)": 40.83, "memory/max_active (GiB)": 38.28, "memory/max_allocated (GiB)": 38.28, "ppl": 3.2801, "step": 19, "tokens/total": 645424, "tokens/train_per_sec_per_gpu": 58.16, "tokens/trainable": 19310} +{"epoch": 0.078125, "grad_norm": 3.6260032653808594, "learning_rate": 7.6e-05, "loss": 1.2137765884399414, "memory/device_reserved (GiB)": 40.83, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 3.36617, "step": 20, "tokens/total": 680016, "tokens/train_per_sec_per_gpu": 60.48, "tokens/trainable": 20369} +{"epoch": 0.08203125, "grad_norm": 2.4284963607788086, "learning_rate": 8e-05, "loss": 1.1945370435714722, "memory/device_reserved (GiB)": 40.83, "memory/max_active (GiB)": 36.12, "memory/max_allocated (GiB)": 36.12, "ppl": 3.30203, "step": 21, "tokens/total": 711024, "tokens/train_per_sec_per_gpu": 70.98, "tokens/trainable": 21448} +{"epoch": 0.0859375, "grad_norm": 2.409465789794922, "learning_rate": 8.4e-05, "loss": 1.136289358139038, "memory/device_reserved (GiB)": 42.47, "memory/max_active (GiB)": 38.93, "memory/max_allocated (GiB)": 38.93, "ppl": 3.11519, "step": 22, "tokens/total": 746128, "tokens/train_per_sec_per_gpu": 64.36, "tokens/trainable": 22545} +{"epoch": 0.08984375, "grad_norm": 2.507355213165283, "learning_rate": 8.800000000000001e-05, "loss": 0.9290155172348022, "memory/device_reserved (GiB)": 42.47, "memory/max_active (GiB)": 37.44, "memory/max_allocated (GiB)": 37.44, "ppl": 2.53202, "step": 23, "tokens/total": 780432, "tokens/train_per_sec_per_gpu": 61.39, "tokens/trainable": 23619} +{"epoch": 0.09375, "grad_norm": 1.982822299003601, "learning_rate": 9.200000000000001e-05, "loss": 1.161666989326477, "memory/device_reserved (GiB)": 42.47, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 3.19526, "step": 24, "tokens/total": 811472, "tokens/train_per_sec_per_gpu": 67.65, "tokens/trainable": 24601} +{"epoch": 0.09765625, "grad_norm": 2.648921251296997, "learning_rate": 9.6e-05, "loss": 1.0713483095169067, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 2.91931, "step": 25, "tokens/total": 843648, "tokens/train_per_sec_per_gpu": 63.27, "tokens/trainable": 25632} +{"epoch": 0.1015625, "grad_norm": 1.6205312013626099, "learning_rate": 0.0001, "loss": 1.0437967777252197, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 2.83998, "step": 26, "tokens/total": 876272, "tokens/train_per_sec_per_gpu": 58.02, "tokens/trainable": 26604} +{"epoch": 0.10546875, "grad_norm": 1.9152272939682007, "learning_rate": 9.99990636831587e-05, "loss": 0.9150320291519165, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 37.45, "memory/max_allocated (GiB)": 37.45, "ppl": 2.49686, "step": 27, "tokens/total": 908928, "tokens/train_per_sec_per_gpu": 72.96, "tokens/trainable": 27707} +{"epoch": 0.109375, "grad_norm": 1.7427177429199219, "learning_rate": 9.999625477159879e-05, "loss": 0.9563164710998535, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 2.60209, "step": 28, "tokens/total": 947504, "tokens/train_per_sec_per_gpu": 52.97, "tokens/trainable": 28751} +{"epoch": 0.11328125, "grad_norm": 1.9031341075897217, "learning_rate": 9.999157338221051e-05, "loss": 0.8836930990219116, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 37.84, "memory/max_allocated (GiB)": 37.84, "ppl": 2.41982, "step": 29, "tokens/total": 982736, "tokens/train_per_sec_per_gpu": 62.75, "tokens/trainable": 29895} +{"epoch": 0.1171875, "grad_norm": 3.0667150020599365, "learning_rate": 9.998501970980562e-05, "loss": 0.8602498769760132, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 37.18, "memory/max_allocated (GiB)": 37.18, "ppl": 2.36375, "step": 30, "tokens/total": 1014432, "tokens/train_per_sec_per_gpu": 72.37, "tokens/trainable": 31017} +{"epoch": 0.12109375, "grad_norm": 2.2873375415802, "learning_rate": 9.997659402710915e-05, "loss": 0.9540827870368958, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 2.59629, "step": 31, "tokens/total": 1049216, "tokens/train_per_sec_per_gpu": 57.96, "tokens/trainable": 32029} +{"epoch": 0.125, "grad_norm": 1.9857898950576782, "learning_rate": 9.996629668474818e-05, "loss": 0.8127608299255371, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 2.25412, "step": 32, "tokens/total": 1084064, "tokens/train_per_sec_per_gpu": 60.07, "tokens/trainable": 33108} +{"epoch": 0.12890625, "grad_norm": 2.13043212890625, "learning_rate": 9.995412811123711e-05, "loss": 0.9252344369888306, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 2.52246, "step": 33, "tokens/total": 1119792, "tokens/train_per_sec_per_gpu": 53.37, "tokens/trainable": 34014} +{"epoch": 0.1328125, "grad_norm": 1.4903937578201294, "learning_rate": 9.994008881295999e-05, "loss": 0.6930751204490662, "memory/device_reserved (GiB)": 39.89, "memory/max_active (GiB)": 37.15, "memory/max_allocated (GiB)": 37.15, "ppl": 1.99986, "step": 34, "tokens/total": 1153152, "tokens/train_per_sec_per_gpu": 64.66, "tokens/trainable": 35082} +{"epoch": 0.13671875, "grad_norm": 1.986653208732605, "learning_rate": 9.992417937414932e-05, "loss": 0.7374201416969299, "memory/device_reserved (GiB)": 39.9, "memory/max_active (GiB)": 38.06, "memory/max_allocated (GiB)": 38.06, "ppl": 2.09054, "step": 35, "tokens/total": 1187040, "tokens/train_per_sec_per_gpu": 60.4, "tokens/trainable": 36219} +{"epoch": 0.140625, "grad_norm": 1.5336765050888062, "learning_rate": 9.99064004568618e-05, "loss": 0.778251588344574, "memory/device_reserved (GiB)": 39.9, "memory/max_active (GiB)": 37.67, "memory/max_allocated (GiB)": 37.67, "ppl": 2.17766, "step": 36, "tokens/total": 1222784, "tokens/train_per_sec_per_gpu": 53.2, "tokens/trainable": 37241} +{"epoch": 0.14453125, "grad_norm": 1.6256612539291382, "learning_rate": 9.988675280095074e-05, "loss": 0.7353922128677368, "memory/device_reserved (GiB)": 39.9, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 2.0863, "step": 37, "tokens/total": 1257216, "tokens/train_per_sec_per_gpu": 58.32, "tokens/trainable": 38357} +{"epoch": 0.1484375, "grad_norm": 1.4492558240890503, "learning_rate": 9.986523722403528e-05, "loss": 0.7355970144271851, "memory/device_reserved (GiB)": 39.9, "memory/max_active (GiB)": 36.82, "memory/max_allocated (GiB)": 36.82, "ppl": 2.08673, "step": 38, "tokens/total": 1288992, "tokens/train_per_sec_per_gpu": 61.46, "tokens/trainable": 39387} +{"epoch": 0.15234375, "grad_norm": 1.6037408113479614, "learning_rate": 9.984185462146642e-05, "loss": 0.6883453130722046, "memory/device_reserved (GiB)": 39.9, "memory/max_active (GiB)": 37.39, "memory/max_allocated (GiB)": 37.39, "ppl": 1.99042, "step": 39, "tokens/total": 1324032, "tokens/train_per_sec_per_gpu": 59.57, "tokens/trainable": 40436} +{"epoch": 0.15625, "grad_norm": 3.670635223388672, "learning_rate": 9.98166059662897e-05, "loss": 0.7073250412940979, "memory/device_reserved (GiB)": 40.64, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 2.02856, "step": 40, "tokens/total": 1359440, "tokens/train_per_sec_per_gpu": 62.08, "tokens/trainable": 41536} +{"epoch": 0.16015625, "grad_norm": 15.425370216369629, "learning_rate": 9.978949230920472e-05, "loss": 0.7139458656311035, "memory/device_reserved (GiB)": 40.64, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 2.04203, "step": 41, "tokens/total": 1393792, "tokens/train_per_sec_per_gpu": 58.31, "tokens/trainable": 42550} +{"epoch": 0.1640625, "grad_norm": 1.7370622158050537, "learning_rate": 9.976051477852141e-05, "loss": 0.7616972923278809, "memory/device_reserved (GiB)": 40.64, "memory/max_active (GiB)": 37.91, "memory/max_allocated (GiB)": 37.91, "ppl": 2.14191, "step": 42, "tokens/total": 1427472, "tokens/train_per_sec_per_gpu": 62.6, "tokens/trainable": 43556} +{"epoch": 0.16796875, "grad_norm": 1.6644139289855957, "learning_rate": 9.972967458011312e-05, "loss": 0.6605948209762573, "memory/device_reserved (GiB)": 40.64, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 1.93594, "step": 43, "tokens/total": 1462896, "tokens/train_per_sec_per_gpu": 61.2, "tokens/trainable": 44658} +{"epoch": 0.171875, "grad_norm": 1.5633865594863892, "learning_rate": 9.96969729973664e-05, "loss": 0.6716036796569824, "memory/device_reserved (GiB)": 41.23, "memory/max_active (GiB)": 38.93, "memory/max_allocated (GiB)": 38.93, "ppl": 1.95737, "step": 44, "tokens/total": 1501392, "tokens/train_per_sec_per_gpu": 56.55, "tokens/trainable": 45722} +{"epoch": 0.17578125, "grad_norm": 2.338740348815918, "learning_rate": 9.966241139112754e-05, "loss": 0.6927984952926636, "memory/device_reserved (GiB)": 41.23, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.9993, "step": 45, "tokens/total": 1535040, "tokens/train_per_sec_per_gpu": 62.21, "tokens/trainable": 46758} +{"epoch": 0.1796875, "grad_norm": 3.0866856575012207, "learning_rate": 9.96259911996461e-05, "loss": 0.6942785978317261, "memory/device_reserved (GiB)": 41.35, "memory/max_active (GiB)": 38.63, "memory/max_allocated (GiB)": 38.63, "ppl": 2.00226, "step": 46, "tokens/total": 1569120, "tokens/train_per_sec_per_gpu": 57.18, "tokens/trainable": 47793} +{"epoch": 0.18359375, "grad_norm": 1.786381721496582, "learning_rate": 9.958771393851491e-05, "loss": 0.7076228857040405, "memory/device_reserved (GiB)": 41.35, "memory/max_active (GiB)": 36.71, "memory/max_allocated (GiB)": 36.71, "ppl": 2.02916, "step": 47, "tokens/total": 1600832, "tokens/train_per_sec_per_gpu": 60.61, "tokens/trainable": 48816} +{"epoch": 0.1875, "grad_norm": 1.5802366733551025, "learning_rate": 9.954758120060702e-05, "loss": 0.6905685067176819, "memory/device_reserved (GiB)": 41.35, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.99485, "step": 48, "tokens/total": 1637184, "tokens/train_per_sec_per_gpu": 52.41, "tokens/trainable": 49778} +{"epoch": 0.19140625, "grad_norm": 1.785301923751831, "learning_rate": 9.950559465600948e-05, "loss": 0.6235053539276123, "memory/device_reserved (GiB)": 41.35, "memory/max_active (GiB)": 37.61, "memory/max_allocated (GiB)": 37.61, "ppl": 1.86546, "step": 49, "tokens/total": 1671232, "tokens/train_per_sec_per_gpu": 60.99, "tokens/trainable": 50825} +{"epoch": 0.1953125, "grad_norm": 1.393383502960205, "learning_rate": 9.946175605195379e-05, "loss": 0.5617448091506958, "memory/device_reserved (GiB)": 41.35, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.75373, "step": 50, "tokens/total": 1711328, "tokens/train_per_sec_per_gpu": 49.48, "tokens/trainable": 51931} +{"epoch": 0.19921875, "grad_norm": 1.7851208448410034, "learning_rate": 9.941606721274322e-05, "loss": 0.5884084701538086, "memory/device_reserved (GiB)": 41.35, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.80112, "step": 51, "tokens/total": 1745680, "tokens/train_per_sec_per_gpu": 53.68, "tokens/trainable": 52960} +{"epoch": 0.203125, "grad_norm": 1.9547752141952515, "learning_rate": 9.936853003967685e-05, "loss": 0.6420388221740723, "memory/device_reserved (GiB)": 41.35, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 1.90035, "step": 52, "tokens/total": 1782832, "tokens/train_per_sec_per_gpu": 54.11, "tokens/trainable": 53980} +{"epoch": 0.20703125, "grad_norm": 8.4769868850708, "learning_rate": 9.93191465109705e-05, "loss": 0.6314865946769714, "memory/device_reserved (GiB)": 41.35, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.8804, "step": 53, "tokens/total": 1819504, "tokens/train_per_sec_per_gpu": 54.82, "tokens/trainable": 54999} +{"epoch": 0.2109375, "grad_norm": 2.7273337841033936, "learning_rate": 9.926791868167438e-05, "loss": 0.6542982459068298, "memory/device_reserved (GiB)": 41.35, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.92379, "step": 54, "tokens/total": 1855952, "tokens/train_per_sec_per_gpu": 52.38, "tokens/trainable": 55970} +{"epoch": 0.21484375, "grad_norm": 6.807374954223633, "learning_rate": 9.921484868358753e-05, "loss": 0.670513927936554, "memory/device_reserved (GiB)": 41.35, "memory/max_active (GiB)": 38.72, "memory/max_allocated (GiB)": 38.72, "ppl": 1.95524, "step": 55, "tokens/total": 1891776, "tokens/train_per_sec_per_gpu": 51.92, "tokens/trainable": 56992} +{"epoch": 0.21875, "grad_norm": 2.224949836730957, "learning_rate": 9.915993872516924e-05, "loss": 0.631513237953186, "memory/device_reserved (GiB)": 41.35, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.88045, "step": 56, "tokens/total": 1927376, "tokens/train_per_sec_per_gpu": 53.5, "tokens/trainable": 57998} +{"epoch": 0.22265625, "grad_norm": 2.020794630050659, "learning_rate": 9.9103191091447e-05, "loss": 0.5938211679458618, "memory/device_reserved (GiB)": 41.35, "memory/max_active (GiB)": 36.88, "memory/max_allocated (GiB)": 36.88, "ppl": 1.81089, "step": 57, "tokens/total": 1959104, "tokens/train_per_sec_per_gpu": 59.81, "tokens/trainable": 58972} +{"epoch": 0.2265625, "grad_norm": 1.618113398551941, "learning_rate": 9.904460814392147e-05, "loss": 0.5738598704338074, "memory/device_reserved (GiB)": 41.35, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.77511, "step": 58, "tokens/total": 1998880, "tokens/train_per_sec_per_gpu": 50.05, "tokens/trainable": 60011} +{"epoch": 0.23046875, "grad_norm": 1.6178193092346191, "learning_rate": 9.898419232046825e-05, "loss": 0.571500301361084, "memory/device_reserved (GiB)": 41.35, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.77092, "step": 59, "tokens/total": 2035440, "tokens/train_per_sec_per_gpu": 56.58, "tokens/trainable": 61094} +{"epoch": 0.234375, "grad_norm": 2.8571133613586426, "learning_rate": 9.892194613523633e-05, "loss": 0.5803918838500977, "memory/device_reserved (GiB)": 41.35, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.78674, "step": 60, "tokens/total": 2071488, "tokens/train_per_sec_per_gpu": 57.96, "tokens/trainable": 62209} +{"epoch": 0.23828125, "grad_norm": 1.5117117166519165, "learning_rate": 9.885787217854357e-05, "loss": 0.5552562475204468, "memory/device_reserved (GiB)": 41.35, "memory/max_active (GiB)": 38.18, "memory/max_allocated (GiB)": 38.18, "ppl": 1.74239, "step": 61, "tokens/total": 2105664, "tokens/train_per_sec_per_gpu": 57.49, "tokens/trainable": 63212} +{"epoch": 0.2421875, "grad_norm": 1.9337736368179321, "learning_rate": 9.879197311676887e-05, "loss": 0.5488564968109131, "memory/device_reserved (GiB)": 41.35, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.73127, "step": 62, "tokens/total": 2141824, "tokens/train_per_sec_per_gpu": 56.75, "tokens/trainable": 64254} +{"epoch": 0.24609375, "grad_norm": 1.6197062730789185, "learning_rate": 9.872425169224113e-05, "loss": 0.5313222408294678, "memory/device_reserved (GiB)": 41.35, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 1.70118, "step": 63, "tokens/total": 2175520, "tokens/train_per_sec_per_gpu": 70.29, "tokens/trainable": 65434} +{"epoch": 0.25, "grad_norm": 2.091212749481201, "learning_rate": 9.865471072312528e-05, "loss": 0.5447756052017212, "memory/device_reserved (GiB)": 41.35, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.72422, "step": 64, "tokens/total": 2212064, "tokens/train_per_sec_per_gpu": 61.22, "tokens/trainable": 66516} +{"epoch": 0.25390625, "grad_norm": 1.6571751832962036, "learning_rate": 9.858335310330492e-05, "loss": 0.5048630833625793, "memory/device_reserved (GiB)": 41.35, "memory/max_active (GiB)": 37.09, "memory/max_allocated (GiB)": 37.09, "ppl": 1.65676, "step": 65, "tokens/total": 2244656, "tokens/train_per_sec_per_gpu": 63.74, "tokens/trainable": 67595} +{"epoch": 0.2578125, "grad_norm": 1.908855676651001, "learning_rate": 9.851018180226185e-05, "loss": 0.5215770602226257, "memory/device_reserved (GiB)": 41.42, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.68468, "step": 66, "tokens/total": 2283328, "tokens/train_per_sec_per_gpu": 54.48, "tokens/trainable": 68665} +{"epoch": 0.26171875, "grad_norm": 1.5416779518127441, "learning_rate": 9.843519986495259e-05, "loss": 0.5377965569496155, "memory/device_reserved (GiB)": 41.43, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.71223, "step": 67, "tokens/total": 2318272, "tokens/train_per_sec_per_gpu": 57.52, "tokens/trainable": 69691} +{"epoch": 0.265625, "grad_norm": 1.4161039590835571, "learning_rate": 9.835841041168162e-05, "loss": 0.44651731848716736, "memory/device_reserved (GiB)": 41.43, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.56286, "step": 68, "tokens/total": 2355680, "tokens/train_per_sec_per_gpu": 59.66, "tokens/trainable": 70784} +{"epoch": 0.26953125, "grad_norm": 1.6327314376831055, "learning_rate": 9.82798166379715e-05, "loss": 0.44512322545051575, "memory/device_reserved (GiB)": 41.43, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.56068, "step": 69, "tokens/total": 2388880, "tokens/train_per_sec_per_gpu": 62.62, "tokens/trainable": 71873} +{"epoch": 0.2734375, "grad_norm": 1.4777156114578247, "learning_rate": 9.819942181443002e-05, "loss": 0.5346369743347168, "memory/device_reserved (GiB)": 41.43, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.70683, "step": 70, "tokens/total": 2426144, "tokens/train_per_sec_per_gpu": 55.81, "tokens/trainable": 72917} +{"epoch": 0.27734375, "grad_norm": 1.5613762140274048, "learning_rate": 9.811722928661392e-05, "loss": 0.5567439794540405, "memory/device_reserved (GiB)": 41.43, "memory/max_active (GiB)": 35.8, "memory/max_allocated (GiB)": 35.8, "ppl": 1.74498, "step": 71, "tokens/total": 2454928, "tokens/train_per_sec_per_gpu": 72.25, "tokens/trainable": 73928} +{"epoch": 0.28125, "grad_norm": 1.2379802465438843, "learning_rate": 9.803324247488975e-05, "loss": 0.5064880847930908, "memory/device_reserved (GiB)": 41.43, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.65945, "step": 72, "tokens/total": 2490960, "tokens/train_per_sec_per_gpu": 57.95, "tokens/trainable": 74998} +{"epoch": 0.28515625, "grad_norm": 1.4100887775421143, "learning_rate": 9.794746487429161e-05, "loss": 0.4825884997844696, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.04, "memory/max_allocated (GiB)": 37.04, "ppl": 1.62026, "step": 73, "tokens/total": 2524048, "tokens/train_per_sec_per_gpu": 67.96, "tokens/trainable": 76037} +{"epoch": 0.2890625, "grad_norm": 1.6031049489974976, "learning_rate": 9.785990005437554e-05, "loss": 0.51957768201828, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.81, "memory/max_allocated (GiB)": 37.81, "ppl": 1.68132, "step": 74, "tokens/total": 2559888, "tokens/train_per_sec_per_gpu": 58.37, "tokens/trainable": 77092} +{"epoch": 0.29296875, "grad_norm": 1.5152500867843628, "learning_rate": 9.777055165907117e-05, "loss": 0.49739760160446167, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.64444, "step": 75, "tokens/total": 2593072, "tokens/train_per_sec_per_gpu": 66.83, "tokens/trainable": 78129} +{"epoch": 0.296875, "grad_norm": 2.3169219493865967, "learning_rate": 9.767942340652993e-05, "loss": 0.545804500579834, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.05, "memory/max_allocated (GiB)": 38.05, "ppl": 1.726, "step": 76, "tokens/total": 2627808, "tokens/train_per_sec_per_gpu": 55.41, "tokens/trainable": 79148} +{"epoch": 0.30078125, "grad_norm": 3.3666439056396484, "learning_rate": 9.758651908897035e-05, "loss": 0.5293096899986267, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.69776, "step": 77, "tokens/total": 2661440, "tokens/train_per_sec_per_gpu": 62.33, "tokens/trainable": 80199} +{"epoch": 0.3046875, "grad_norm": 1.7786966562271118, "learning_rate": 9.749184257252033e-05, "loss": 0.5439282655715942, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.72276, "step": 78, "tokens/total": 2695696, "tokens/train_per_sec_per_gpu": 62.01, "tokens/trainable": 81200} +{"epoch": 0.30859375, "grad_norm": 1.9423936605453491, "learning_rate": 9.739539779705614e-05, "loss": 0.45026901364326477, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.56873, "step": 79, "tokens/total": 2734224, "tokens/train_per_sec_per_gpu": 58.72, "tokens/trainable": 82397} +{"epoch": 0.3125, "grad_norm": 1.562417984008789, "learning_rate": 9.729718877603861e-05, "loss": 0.4556314945220947, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.57717, "step": 80, "tokens/total": 2767808, "tokens/train_per_sec_per_gpu": 67.85, "tokens/trainable": 83445} +{"epoch": 0.31640625, "grad_norm": 1.4575320482254028, "learning_rate": 9.719721959634592e-05, "loss": 0.44033509492874146, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.55323, "step": 81, "tokens/total": 2803328, "tokens/train_per_sec_per_gpu": 61.59, "tokens/trainable": 84494} +{"epoch": 0.3203125, "grad_norm": 3.1699488162994385, "learning_rate": 9.709549441810375e-05, "loss": 0.457259863615036, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.57974, "step": 82, "tokens/total": 2840736, "tokens/train_per_sec_per_gpu": 60.78, "tokens/trainable": 85574} +{"epoch": 0.32421875, "grad_norm": 1.2009531259536743, "learning_rate": 9.699201747451195e-05, "loss": 0.4645231068134308, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.54, "memory/max_allocated (GiB)": 36.54, "ppl": 1.59126, "step": 83, "tokens/total": 2873248, "tokens/train_per_sec_per_gpu": 58.91, "tokens/trainable": 86627} +{"epoch": 0.328125, "grad_norm": 1.2907466888427734, "learning_rate": 9.688679307166854e-05, "loss": 0.5044333934783936, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.65605, "step": 84, "tokens/total": 2905760, "tokens/train_per_sec_per_gpu": 63.7, "tokens/trainable": 87688} +{"epoch": 0.33203125, "grad_norm": 1.9412808418273926, "learning_rate": 9.677982558839042e-05, "loss": 0.5314299464225769, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.82, "memory/max_allocated (GiB)": 36.82, "ppl": 1.70136, "step": 85, "tokens/total": 2938928, "tokens/train_per_sec_per_gpu": 56.89, "tokens/trainable": 88743} +{"epoch": 0.3359375, "grad_norm": 1.528800129890442, "learning_rate": 9.66711194760312e-05, "loss": 0.4238743782043457, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 39.04, "memory/max_allocated (GiB)": 39.04, "ppl": 1.52787, "step": 86, "tokens/total": 2976368, "tokens/train_per_sec_per_gpu": 56.1, "tokens/trainable": 89872} +{"epoch": 0.33984375, "grad_norm": 1.6057788133621216, "learning_rate": 9.656067925829593e-05, "loss": 0.4753108620643616, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.60851, "step": 87, "tokens/total": 3006960, "tokens/train_per_sec_per_gpu": 71.65, "tokens/trainable": 90906} +{"epoch": 0.34375, "grad_norm": 2.792994260787964, "learning_rate": 9.644850953105288e-05, "loss": 0.4893680512905121, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 35.88, "memory/max_allocated (GiB)": 35.88, "ppl": 1.63129, "step": 88, "tokens/total": 3037968, "tokens/train_per_sec_per_gpu": 68.55, "tokens/trainable": 91976} +{"epoch": 0.34765625, "grad_norm": 1.7659953832626343, "learning_rate": 9.633461496214225e-05, "loss": 0.407867968082428, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.50361, "step": 89, "tokens/total": 3070608, "tokens/train_per_sec_per_gpu": 62.63, "tokens/trainable": 93033} +{"epoch": 0.3515625, "grad_norm": 2.104063034057617, "learning_rate": 9.621900029118195e-05, "loss": 0.4449504017829895, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.1, "memory/max_allocated (GiB)": 36.1, "ppl": 1.56041, "step": 90, "tokens/total": 3100272, "tokens/train_per_sec_per_gpu": 68.37, "tokens/trainable": 94067} +{"epoch": 0.35546875, "grad_norm": 1.3974504470825195, "learning_rate": 9.610167032937036e-05, "loss": 0.4769391715526581, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.61114, "step": 91, "tokens/total": 3138480, "tokens/train_per_sec_per_gpu": 54.69, "tokens/trainable": 95120} +{"epoch": 0.359375, "grad_norm": 1.5257513523101807, "learning_rate": 9.598262995928611e-05, "loss": 0.4051763713359833, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.49957, "step": 92, "tokens/total": 3169648, "tokens/train_per_sec_per_gpu": 70.96, "tokens/trainable": 96187} +{"epoch": 0.36328125, "grad_norm": 5.98591947555542, "learning_rate": 9.586188413468492e-05, "loss": 0.45092636346817017, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.8, "memory/max_allocated (GiB)": 38.8, "ppl": 1.56977, "step": 93, "tokens/total": 3208032, "tokens/train_per_sec_per_gpu": 49.81, "tokens/trainable": 97240} +{"epoch": 0.3671875, "grad_norm": 1.7731788158416748, "learning_rate": 9.57394378802934e-05, "loss": 0.4443050026893616, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.55941, "step": 94, "tokens/total": 3241472, "tokens/train_per_sec_per_gpu": 61.05, "tokens/trainable": 98286} +{"epoch": 0.37109375, "grad_norm": 1.5611484050750732, "learning_rate": 9.56152962916e-05, "loss": 0.43870261311531067, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.55069, "step": 95, "tokens/total": 3272256, "tokens/train_per_sec_per_gpu": 66.36, "tokens/trainable": 99269} +{"epoch": 0.375, "grad_norm": 1.3822087049484253, "learning_rate": 9.548946453464296e-05, "loss": 0.3595571219921112, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 1.43269, "step": 96, "tokens/total": 3306592, "tokens/train_per_sec_per_gpu": 63.04, "tokens/trainable": 100374} +{"epoch": 0.37890625, "grad_norm": 1.7921420335769653, "learning_rate": 9.53619478457953e-05, "loss": 0.4706013798713684, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.60096, "step": 97, "tokens/total": 3341264, "tokens/train_per_sec_per_gpu": 52.77, "tokens/trainable": 101366} +{"epoch": 0.3828125, "grad_norm": 1.6168510913848877, "learning_rate": 9.523275153154695e-05, "loss": 0.4619402587413788, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.58715, "step": 98, "tokens/total": 3373776, "tokens/train_per_sec_per_gpu": 58.21, "tokens/trainable": 102326} +{"epoch": 0.38671875, "grad_norm": 1.4324294328689575, "learning_rate": 9.51018809682839e-05, "loss": 0.42677634954452515, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.53231, "step": 99, "tokens/total": 3411600, "tokens/train_per_sec_per_gpu": 53.26, "tokens/trainable": 103343} +{"epoch": 0.390625, "grad_norm": 1.9450905323028564, "learning_rate": 9.49693416020645e-05, "loss": 0.427902489900589, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.51, "memory/max_allocated (GiB)": 38.51, "ppl": 1.53404, "step": 100, "tokens/total": 3449392, "tokens/train_per_sec_per_gpu": 55.22, "tokens/trainable": 104363} +{"epoch": 0.39453125, "grad_norm": 2.183337450027466, "learning_rate": 9.483513894839276e-05, "loss": 0.47129249572753906, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.60206, "step": 101, "tokens/total": 3482832, "tokens/train_per_sec_per_gpu": 68.39, "tokens/trainable": 105466} +{"epoch": 0.3984375, "grad_norm": 1.5110479593276978, "learning_rate": 9.469927859198888e-05, "loss": 0.4311608374118805, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.53904, "step": 102, "tokens/total": 3518768, "tokens/train_per_sec_per_gpu": 58.94, "tokens/trainable": 106466} +{"epoch": 0.40234375, "grad_norm": 1.6392287015914917, "learning_rate": 9.456176618655689e-05, "loss": 0.4253847599029541, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.51, "memory/max_allocated (GiB)": 36.51, "ppl": 1.53018, "step": 103, "tokens/total": 3550736, "tokens/train_per_sec_per_gpu": 60.74, "tokens/trainable": 107462} +{"epoch": 0.40625, "grad_norm": 4.944424629211426, "learning_rate": 9.442260745454927e-05, "loss": 0.4280545711517334, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.53427, "step": 104, "tokens/total": 3585920, "tokens/train_per_sec_per_gpu": 59.48, "tokens/trainable": 108533} +{"epoch": 0.41015625, "grad_norm": 1.4614927768707275, "learning_rate": 9.428180818692884e-05, "loss": 0.38648468255996704, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.4718, "step": 105, "tokens/total": 3622160, "tokens/train_per_sec_per_gpu": 52.85, "tokens/trainable": 109593} +{"epoch": 0.4140625, "grad_norm": 1.5481764078140259, "learning_rate": 9.413937424292791e-05, "loss": 0.40358543395996094, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.49718, "step": 106, "tokens/total": 3661088, "tokens/train_per_sec_per_gpu": 48.3, "tokens/trainable": 110589} +{"epoch": 0.41796875, "grad_norm": 1.6412910223007202, "learning_rate": 9.399531154980424e-05, "loss": 0.3491199016571045, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.41782, "step": 107, "tokens/total": 3697888, "tokens/train_per_sec_per_gpu": 51.89, "tokens/trainable": 111594} +{"epoch": 0.421875, "grad_norm": 2.0484704971313477, "learning_rate": 9.384962610259455e-05, "loss": 0.3943622410297394, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.48344, "step": 108, "tokens/total": 3732400, "tokens/train_per_sec_per_gpu": 55.15, "tokens/trainable": 112587} +{"epoch": 0.42578125, "grad_norm": 3.8347485065460205, "learning_rate": 9.370232396386494e-05, "loss": 0.40152978897094727, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.49411, "step": 109, "tokens/total": 3765616, "tokens/train_per_sec_per_gpu": 61.26, "tokens/trainable": 113611} +{"epoch": 0.4296875, "grad_norm": 1.343414068222046, "learning_rate": 9.355341126345868e-05, "loss": 0.35041168332099915, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.41965, "step": 110, "tokens/total": 3801104, "tokens/train_per_sec_per_gpu": 61.5, "tokens/trainable": 114710} +{"epoch": 0.43359375, "grad_norm": 1.5870100259780884, "learning_rate": 9.340289419824107e-05, "loss": 0.35344210267066956, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.74, "memory/max_allocated (GiB)": 36.74, "ppl": 1.42396, "step": 111, "tokens/total": 3833296, "tokens/train_per_sec_per_gpu": 69.34, "tokens/trainable": 115741} +{"epoch": 0.4375, "grad_norm": 1.8427510261535645, "learning_rate": 9.325077903184159e-05, "loss": 0.4105360507965088, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.50763, "step": 112, "tokens/total": 3868144, "tokens/train_per_sec_per_gpu": 57.9, "tokens/trainable": 116832} +{"epoch": 0.44140625, "grad_norm": 1.219941258430481, "learning_rate": 9.30970720943932e-05, "loss": 0.3295973539352417, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.43, "memory/max_allocated (GiB)": 38.43, "ppl": 1.39041, "step": 113, "tokens/total": 3903568, "tokens/train_per_sec_per_gpu": 57.93, "tokens/trainable": 117912} +{"epoch": 0.4453125, "grad_norm": 2.332101583480835, "learning_rate": 9.2941779782269e-05, "loss": 0.36460956931114197, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.43995, "step": 114, "tokens/total": 3937328, "tokens/train_per_sec_per_gpu": 61.52, "tokens/trainable": 118935} +{"epoch": 0.44921875, "grad_norm": 2.6857635974884033, "learning_rate": 9.278490855781596e-05, "loss": 0.3638440668582916, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.43885, "step": 115, "tokens/total": 3969936, "tokens/train_per_sec_per_gpu": 70.29, "tokens/trainable": 119972} +{"epoch": 0.453125, "grad_norm": 2.0354502201080322, "learning_rate": 9.262646494908604e-05, "loss": 0.43765729665756226, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 39.14, "memory/max_allocated (GiB)": 39.14, "ppl": 1.54907, "step": 116, "tokens/total": 4008512, "tokens/train_per_sec_per_gpu": 50.91, "tokens/trainable": 120969} +{"epoch": 0.45703125, "grad_norm": 1.8224393129348755, "learning_rate": 9.246645554956457e-05, "loss": 0.3943657875061035, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.48344, "step": 117, "tokens/total": 4043104, "tokens/train_per_sec_per_gpu": 61.98, "tokens/trainable": 122017} +{"epoch": 0.4609375, "grad_norm": 3.0832788944244385, "learning_rate": 9.230488701789578e-05, "loss": 0.43492990732192993, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.7, "memory/max_allocated (GiB)": 38.7, "ppl": 1.54485, "step": 118, "tokens/total": 4080624, "tokens/train_per_sec_per_gpu": 55.81, "tokens/trainable": 123098} +{"epoch": 0.46484375, "grad_norm": 1.525400996208191, "learning_rate": 9.214176607760577e-05, "loss": 0.36538073420524597, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.44106, "step": 119, "tokens/total": 4119296, "tokens/train_per_sec_per_gpu": 58.76, "tokens/trainable": 124229} +{"epoch": 0.46875, "grad_norm": 1.4663480520248413, "learning_rate": 9.197709951682268e-05, "loss": 0.36089587211608887, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.43461, "step": 120, "tokens/total": 4151584, "tokens/train_per_sec_per_gpu": 65.26, "tokens/trainable": 125294} +{"epoch": 0.47265625, "grad_norm": 1.7851835489273071, "learning_rate": 9.181089418799428e-05, "loss": 0.38445115089416504, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 39.39, "memory/max_allocated (GiB)": 39.39, "ppl": 1.46881, "step": 121, "tokens/total": 4188384, "tokens/train_per_sec_per_gpu": 55.31, "tokens/trainable": 126333} +{"epoch": 0.4765625, "grad_norm": 1.9920670986175537, "learning_rate": 9.164315700760271e-05, "loss": 0.3278529942035675, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.38798, "step": 122, "tokens/total": 4224192, "tokens/train_per_sec_per_gpu": 58.69, "tokens/trainable": 127377} +{"epoch": 0.48046875, "grad_norm": 1.5423301458358765, "learning_rate": 9.147389495587671e-05, "loss": 0.43550145626068115, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 1.54574, "step": 123, "tokens/total": 4255344, "tokens/train_per_sec_per_gpu": 69.54, "tokens/trainable": 128395} +{"epoch": 0.484375, "grad_norm": 2.5946929454803467, "learning_rate": 9.130311507650116e-05, "loss": 0.31302550435066223, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 1.36756, "step": 124, "tokens/total": 4288016, "tokens/train_per_sec_per_gpu": 61.89, "tokens/trainable": 129450} +{"epoch": 0.48828125, "grad_norm": 1.294395089149475, "learning_rate": 9.113082447632394e-05, "loss": 0.3564673364162445, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.42827, "step": 125, "tokens/total": 4320160, "tokens/train_per_sec_per_gpu": 79.82, "tokens/trainable": 130600} +{"epoch": 0.4921875, "grad_norm": 2.8275861740112305, "learning_rate": 9.09570303250602e-05, "loss": 0.4036378562450409, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.47, "memory/max_allocated (GiB)": 36.47, "ppl": 1.49726, "step": 126, "tokens/total": 4351968, "tokens/train_per_sec_per_gpu": 63.45, "tokens/trainable": 131663} +{"epoch": 0.49609375, "grad_norm": 1.439933180809021, "learning_rate": 9.078173985499394e-05, "loss": 0.35910287499427795, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 35.79, "memory/max_allocated (GiB)": 35.79, "ppl": 1.43204, "step": 127, "tokens/total": 4381952, "tokens/train_per_sec_per_gpu": 65.7, "tokens/trainable": 132667} +{"epoch": 0.5, "grad_norm": 1.6020175218582153, "learning_rate": 9.060496036067713e-05, "loss": 0.38630059361457825, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.72, "memory/max_allocated (GiB)": 38.72, "ppl": 1.47153, "step": 128, "tokens/total": 4419376, "tokens/train_per_sec_per_gpu": 48.31, "tokens/trainable": 133681} +{"epoch": 0.50390625, "grad_norm": 1.481623649597168, "learning_rate": 9.042669919862615e-05, "loss": 0.3701290488243103, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.82, "memory/max_allocated (GiB)": 36.82, "ppl": 1.44792, "step": 129, "tokens/total": 4451488, "tokens/train_per_sec_per_gpu": 67.77, "tokens/trainable": 134729} +{"epoch": 0.5078125, "grad_norm": 1.8094090223312378, "learning_rate": 9.024696378701557e-05, "loss": 0.34058886766433716, "memory/device_reserved (GiB)": 41.41, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.40578, "step": 130, "tokens/total": 4487040, "tokens/train_per_sec_per_gpu": 58.47, "tokens/trainable": 135783} +{"epoch": 0.51171875, "grad_norm": 1.36506187915802, "learning_rate": 9.006576160536948e-05, "loss": 0.3477405309677124, "memory/device_reserved (GiB)": 41.41, "memory/max_active (GiB)": 37.43, "memory/max_allocated (GiB)": 37.43, "ppl": 1.41586, "step": 131, "tokens/total": 4521888, "tokens/train_per_sec_per_gpu": 55.59, "tokens/trainable": 136820} +{"epoch": 0.515625, "grad_norm": 1.973845362663269, "learning_rate": 8.988310019425035e-05, "loss": 0.3718116581439972, "memory/device_reserved (GiB)": 41.71, "memory/max_active (GiB)": 39.7, "memory/max_allocated (GiB)": 39.7, "ppl": 1.45036, "step": 132, "tokens/total": 4559744, "tokens/train_per_sec_per_gpu": 51.36, "tokens/trainable": 137830} +{"epoch": 0.51953125, "grad_norm": 1.4510246515274048, "learning_rate": 8.969898715494506e-05, "loss": 0.33818188309669495, "memory/device_reserved (GiB)": 41.73, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.4024, "step": 133, "tokens/total": 4596576, "tokens/train_per_sec_per_gpu": 61.1, "tokens/trainable": 138920} +{"epoch": 0.5234375, "grad_norm": 2.3563129901885986, "learning_rate": 8.951343014914869e-05, "loss": 0.3162022829055786, "memory/device_reserved (GiB)": 41.73, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.37191, "step": 134, "tokens/total": 4633232, "tokens/train_per_sec_per_gpu": 54.68, "tokens/trainable": 139988} +{"epoch": 0.52734375, "grad_norm": 1.6747896671295166, "learning_rate": 8.932643689864568e-05, "loss": 0.3432614803314209, "memory/device_reserved (GiB)": 41.73, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.40954, "step": 135, "tokens/total": 4669808, "tokens/train_per_sec_per_gpu": 62.19, "tokens/trainable": 141105} +{"epoch": 0.53125, "grad_norm": 1.4578821659088135, "learning_rate": 8.913801518498845e-05, "loss": 0.3266963064670563, "memory/device_reserved (GiB)": 41.73, "memory/max_active (GiB)": 39.68, "memory/max_allocated (GiB)": 39.68, "ppl": 1.38638, "step": 136, "tokens/total": 4709184, "tokens/train_per_sec_per_gpu": 48.58, "tokens/trainable": 142217} +{"epoch": 0.53515625, "grad_norm": 2.081580400466919, "learning_rate": 8.894817284917364e-05, "loss": 0.36160144209861755, "memory/device_reserved (GiB)": 41.73, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.43563, "step": 137, "tokens/total": 4746128, "tokens/train_per_sec_per_gpu": 50.31, "tokens/trainable": 143251} +{"epoch": 0.5390625, "grad_norm": 1.7905577421188354, "learning_rate": 8.875691779131569e-05, "loss": 0.3227965235710144, "memory/device_reserved (GiB)": 41.9, "memory/max_active (GiB)": 39.78, "memory/max_allocated (GiB)": 39.78, "ppl": 1.38098, "step": 138, "tokens/total": 4784064, "tokens/train_per_sec_per_gpu": 55.1, "tokens/trainable": 144323} +{"epoch": 0.54296875, "grad_norm": 2.6282224655151367, "learning_rate": 8.856425797031829e-05, "loss": 0.32504740357398987, "memory/device_reserved (GiB)": 42.22, "memory/max_active (GiB)": 39.58, "memory/max_allocated (GiB)": 39.58, "ppl": 1.3841, "step": 139, "tokens/total": 4820096, "tokens/train_per_sec_per_gpu": 63.79, "tokens/trainable": 145451} +{"epoch": 0.546875, "grad_norm": 1.8158855438232422, "learning_rate": 8.837020140354295e-05, "loss": 0.34145867824554443, "memory/device_reserved (GiB)": 42.22, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.407, "step": 140, "tokens/total": 4856912, "tokens/train_per_sec_per_gpu": 55.35, "tokens/trainable": 146458} +{"epoch": 0.55078125, "grad_norm": 1.4899282455444336, "learning_rate": 8.817475616647554e-05, "loss": 0.33152368664741516, "memory/device_reserved (GiB)": 42.22, "memory/max_active (GiB)": 38.17, "memory/max_allocated (GiB)": 38.17, "ppl": 1.39309, "step": 141, "tokens/total": 4891344, "tokens/train_per_sec_per_gpu": 60.1, "tokens/trainable": 147512} +{"epoch": 0.5546875, "grad_norm": 1.514210820198059, "learning_rate": 8.797793039239017e-05, "loss": 0.33279743790626526, "memory/device_reserved (GiB)": 42.22, "memory/max_active (GiB)": 37.83, "memory/max_allocated (GiB)": 37.83, "ppl": 1.39486, "step": 142, "tokens/total": 4926368, "tokens/train_per_sec_per_gpu": 57.94, "tokens/trainable": 148558} +{"epoch": 0.55859375, "grad_norm": 1.5920850038528442, "learning_rate": 8.777973227201069e-05, "loss": 0.306525856256485, "memory/device_reserved (GiB)": 43.76, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.3587, "step": 143, "tokens/total": 4963280, "tokens/train_per_sec_per_gpu": 60.39, "tokens/trainable": 149621} +{"epoch": 0.5625, "grad_norm": 1.5533876419067383, "learning_rate": 8.758017005316988e-05, "loss": 0.3473714292049408, "memory/device_reserved (GiB)": 43.76, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.41534, "step": 144, "tokens/total": 4998752, "tokens/train_per_sec_per_gpu": 68.15, "tokens/trainable": 150674} +{"epoch": 0.56640625, "grad_norm": 1.7464408874511719, "learning_rate": 8.737925204046629e-05, "loss": 0.35131388902664185, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.42093, "step": 145, "tokens/total": 5033472, "tokens/train_per_sec_per_gpu": 54.88, "tokens/trainable": 151735} +{"epoch": 0.5703125, "grad_norm": 1.8784300088882446, "learning_rate": 8.717698659491851e-05, "loss": 0.3174293637275696, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 38.91, "memory/max_allocated (GiB)": 38.91, "ppl": 1.37359, "step": 146, "tokens/total": 5070304, "tokens/train_per_sec_per_gpu": 57.22, "tokens/trainable": 152839} +{"epoch": 0.57421875, "grad_norm": 1.739208698272705, "learning_rate": 8.697338213361735e-05, "loss": 0.31871891021728516, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.37536, "step": 147, "tokens/total": 5106208, "tokens/train_per_sec_per_gpu": 58.71, "tokens/trainable": 153931} +{"epoch": 0.578125, "grad_norm": 3.7572882175445557, "learning_rate": 8.676844712937552e-05, "loss": 0.29447710514068604, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.34242, "step": 148, "tokens/total": 5146208, "tokens/train_per_sec_per_gpu": 49.37, "tokens/trainable": 154983} +{"epoch": 0.58203125, "grad_norm": 2.904226064682007, "learning_rate": 8.656219011037509e-05, "loss": 0.3182857632637024, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.37477, "step": 149, "tokens/total": 5183504, "tokens/train_per_sec_per_gpu": 53.15, "tokens/trainable": 156041} +{"epoch": 0.5859375, "grad_norm": 1.5830453634262085, "learning_rate": 8.63546196598125e-05, "loss": 0.3257572054862976, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.38508, "step": 150, "tokens/total": 5221088, "tokens/train_per_sec_per_gpu": 56.0, "tokens/trainable": 157163} +{"epoch": 0.58984375, "grad_norm": 26.71784019470215, "learning_rate": 8.614574441554145e-05, "loss": 0.4115132689476013, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 36.95, "memory/max_allocated (GiB)": 36.95, "ppl": 1.5091, "step": 151, "tokens/total": 5253712, "tokens/train_per_sec_per_gpu": 62.4, "tokens/trainable": 158127} +{"epoch": 0.59375, "grad_norm": 1.6411151885986328, "learning_rate": 8.593557306971349e-05, "loss": 0.3038816452026367, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.35511, "step": 152, "tokens/total": 5287024, "tokens/train_per_sec_per_gpu": 58.67, "tokens/trainable": 159160} +{"epoch": 0.59765625, "grad_norm": 2.307687759399414, "learning_rate": 8.572411436841618e-05, "loss": 0.344384104013443, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 36.9, "memory/max_allocated (GiB)": 36.9, "ppl": 1.41112, "step": 153, "tokens/total": 5320768, "tokens/train_per_sec_per_gpu": 64.55, "tokens/trainable": 160230} +{"epoch": 0.6015625, "grad_norm": 1.7652708292007446, "learning_rate": 8.551137711130922e-05, "loss": 0.3411366939544678, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.40655, "step": 154, "tokens/total": 5352544, "tokens/train_per_sec_per_gpu": 59.33, "tokens/trainable": 161248} +{"epoch": 0.60546875, "grad_norm": 1.224906086921692, "learning_rate": 8.529737015125824e-05, "loss": 0.2851245403289795, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 38.17, "memory/max_allocated (GiB)": 38.17, "ppl": 1.32993, "step": 155, "tokens/total": 5387648, "tokens/train_per_sec_per_gpu": 56.92, "tokens/trainable": 162267} +{"epoch": 0.609375, "grad_norm": 6.122422695159912, "learning_rate": 8.508210239396639e-05, "loss": 0.3405803442001343, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 39.77, "memory/max_allocated (GiB)": 39.77, "ppl": 1.40576, "step": 156, "tokens/total": 5425536, "tokens/train_per_sec_per_gpu": 57.37, "tokens/trainable": 163284} +{"epoch": 0.61328125, "grad_norm": 2.081068992614746, "learning_rate": 8.486558279760375e-05, "loss": 0.3411509692668915, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 35.9, "memory/max_allocated (GiB)": 35.9, "ppl": 1.40657, "step": 157, "tokens/total": 5456592, "tokens/train_per_sec_per_gpu": 64.26, "tokens/trainable": 164291} +{"epoch": 0.6171875, "grad_norm": 2.041663408279419, "learning_rate": 8.464782037243449e-05, "loss": 0.3422068953514099, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.40805, "step": 158, "tokens/total": 5488736, "tokens/train_per_sec_per_gpu": 60.88, "tokens/trainable": 165280} +{"epoch": 0.62109375, "grad_norm": 1.5253636837005615, "learning_rate": 8.442882418044202e-05, "loss": 0.3141493797302246, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.36909, "step": 159, "tokens/total": 5521376, "tokens/train_per_sec_per_gpu": 63.8, "tokens/trainable": 166320} +{"epoch": 0.625, "grad_norm": 1.4878852367401123, "learning_rate": 8.420860333495179e-05, "loss": 0.2669532001018524, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.30598, "step": 160, "tokens/total": 5555184, "tokens/train_per_sec_per_gpu": 61.52, "tokens/trainable": 167340} +{"epoch": 0.62890625, "grad_norm": 2.721330404281616, "learning_rate": 8.398716700025208e-05, "loss": 0.32623860239982605, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.38575, "step": 161, "tokens/total": 5588704, "tokens/train_per_sec_per_gpu": 59.62, "tokens/trainable": 168363} +{"epoch": 0.6328125, "grad_norm": 6.715734481811523, "learning_rate": 8.376452439121266e-05, "loss": 0.3653497099876404, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.05, "memory/max_allocated (GiB)": 36.05, "ppl": 1.44102, "step": 162, "tokens/total": 5619424, "tokens/train_per_sec_per_gpu": 66.43, "tokens/trainable": 169371} +{"epoch": 0.63671875, "grad_norm": 2.623741388320923, "learning_rate": 8.354068477290124e-05, "loss": 0.3722071051597595, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.45093, "step": 163, "tokens/total": 5653648, "tokens/train_per_sec_per_gpu": 61.44, "tokens/trainable": 170358} +{"epoch": 0.640625, "grad_norm": 1.9510785341262817, "learning_rate": 8.331565746019807e-05, "loss": 0.3135722875595093, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.3683, "step": 164, "tokens/total": 5685840, "tokens/train_per_sec_per_gpu": 69.06, "tokens/trainable": 171390} +{"epoch": 0.64453125, "grad_norm": 3.1792211532592773, "learning_rate": 8.308945181740812e-05, "loss": 0.34861719608306885, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.41711, "step": 165, "tokens/total": 5720608, "tokens/train_per_sec_per_gpu": 62.06, "tokens/trainable": 172412} +{"epoch": 0.6484375, "grad_norm": 1.3668209314346313, "learning_rate": 8.286207725787153e-05, "loss": 0.3024730682373047, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.3532, "step": 166, "tokens/total": 5757616, "tokens/train_per_sec_per_gpu": 51.98, "tokens/trainable": 173439} +{"epoch": 0.65234375, "grad_norm": 1.8775434494018555, "learning_rate": 8.263354324357182e-05, "loss": 0.30883467197418213, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.12, "memory/max_allocated (GiB)": 39.12, "ppl": 1.36184, "step": 167, "tokens/total": 5792352, "tokens/train_per_sec_per_gpu": 59.17, "tokens/trainable": 174467} +{"epoch": 0.65625, "grad_norm": 1.6980903148651123, "learning_rate": 8.240385928474219e-05, "loss": 0.34863361716270447, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.41713, "step": 168, "tokens/total": 5825168, "tokens/train_per_sec_per_gpu": 58.1, "tokens/trainable": 175427} +{"epoch": 0.66015625, "grad_norm": 2.7953169345855713, "learning_rate": 8.217303493946967e-05, "loss": 0.3141499161720276, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.36909, "step": 169, "tokens/total": 5859376, "tokens/train_per_sec_per_gpu": 58.03, "tokens/trainable": 176429} +{"epoch": 0.6640625, "grad_norm": 2.2829489707946777, "learning_rate": 8.194107981329746e-05, "loss": 0.2929895520210266, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.73, "memory/max_allocated (GiB)": 39.73, "ppl": 1.34043, "step": 170, "tokens/total": 5897952, "tokens/train_per_sec_per_gpu": 51.67, "tokens/trainable": 177450} +{"epoch": 0.66796875, "grad_norm": 1.3442741632461548, "learning_rate": 8.170800355882518e-05, "loss": 0.29454395174980164, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.34251, "step": 171, "tokens/total": 5932384, "tokens/train_per_sec_per_gpu": 66.4, "tokens/trainable": 178489} +{"epoch": 0.671875, "grad_norm": 1.8913248777389526, "learning_rate": 8.147381587530713e-05, "loss": 0.35579854249954224, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.39, "memory/max_allocated (GiB)": 37.39, "ppl": 1.42732, "step": 172, "tokens/total": 5966176, "tokens/train_per_sec_per_gpu": 61.87, "tokens/trainable": 179483} +{"epoch": 0.67578125, "grad_norm": 3.652069091796875, "learning_rate": 8.123852650824877e-05, "loss": 0.3295444846153259, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.45, "memory/max_allocated (GiB)": 37.45, "ppl": 1.39033, "step": 173, "tokens/total": 6000000, "tokens/train_per_sec_per_gpu": 61.84, "tokens/trainable": 180545} +{"epoch": 0.6796875, "grad_norm": 1.9735567569732666, "learning_rate": 8.100214524900103e-05, "loss": 0.31133022904396057, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.36524, "step": 174, "tokens/total": 6035072, "tokens/train_per_sec_per_gpu": 51.97, "tokens/trainable": 181549} +{"epoch": 0.68359375, "grad_norm": 1.5259422063827515, "learning_rate": 8.076468193435301e-05, "loss": 0.33241546154022217, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.39433, "step": 175, "tokens/total": 6065920, "tokens/train_per_sec_per_gpu": 66.71, "tokens/trainable": 182497} +{"epoch": 0.6875, "grad_norm": 2.2149720191955566, "learning_rate": 8.052614644612253e-05, "loss": 0.33090031147003174, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.39222, "step": 176, "tokens/total": 6103760, "tokens/train_per_sec_per_gpu": 50.06, "tokens/trainable": 183547} +{"epoch": 0.69140625, "grad_norm": 1.6490676403045654, "learning_rate": 8.028654871074489e-05, "loss": 0.26295995712280273, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.30077, "step": 177, "tokens/total": 6138352, "tokens/train_per_sec_per_gpu": 59.2, "tokens/trainable": 184633} +{"epoch": 0.6953125, "grad_norm": 1.2991361618041992, "learning_rate": 8.004589869885986e-05, "loss": 0.3298107385635376, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.37, "memory/max_allocated (GiB)": 36.37, "ppl": 1.3907, "step": 178, "tokens/total": 6169824, "tokens/train_per_sec_per_gpu": 61.21, "tokens/trainable": 185591} +{"epoch": 0.69921875, "grad_norm": 1.1734881401062012, "learning_rate": 7.980420642489674e-05, "loss": 0.28500649333000183, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.32977, "step": 179, "tokens/total": 6208528, "tokens/train_per_sec_per_gpu": 53.87, "tokens/trainable": 186623} +{"epoch": 0.703125, "grad_norm": 2.1636836528778076, "learning_rate": 7.95614819466576e-05, "loss": 0.28290581703186035, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 1.32698, "step": 180, "tokens/total": 6242240, "tokens/train_per_sec_per_gpu": 59.06, "tokens/trainable": 187674} +{"epoch": 0.70703125, "grad_norm": 1.4506946802139282, "learning_rate": 7.931773536489872e-05, "loss": 0.27935904264450073, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.32228, "step": 181, "tokens/total": 6276832, "tokens/train_per_sec_per_gpu": 62.79, "tokens/trainable": 188721} +{"epoch": 0.7109375, "grad_norm": 1.7337639331817627, "learning_rate": 7.907297682291035e-05, "loss": 0.2785823345184326, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.32126, "step": 182, "tokens/total": 6311488, "tokens/train_per_sec_per_gpu": 60.95, "tokens/trainable": 189799} +{"epoch": 0.71484375, "grad_norm": 1.5814154148101807, "learning_rate": 7.882721650609442e-05, "loss": 0.2340392768383026, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.65, "memory/max_allocated (GiB)": 39.65, "ppl": 1.26369, "step": 183, "tokens/total": 6351904, "tokens/train_per_sec_per_gpu": 53.53, "tokens/trainable": 190977} +{"epoch": 0.71875, "grad_norm": 1.6472418308258057, "learning_rate": 7.85804646415409e-05, "loss": 0.2879137694835663, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.33364, "step": 184, "tokens/total": 6387136, "tokens/train_per_sec_per_gpu": 64.42, "tokens/trainable": 192063} +{"epoch": 0.72265625, "grad_norm": 1.5431517362594604, "learning_rate": 7.833273149760207e-05, "loss": 0.27152130007743835, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.31196, "step": 185, "tokens/total": 6418000, "tokens/train_per_sec_per_gpu": 60.44, "tokens/trainable": 193068} +{"epoch": 0.7265625, "grad_norm": 1.89436936378479, "learning_rate": 7.808402738346527e-05, "loss": 0.27379047870635986, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.11, "memory/max_allocated (GiB)": 39.11, "ppl": 1.31494, "step": 186, "tokens/total": 6456784, "tokens/train_per_sec_per_gpu": 58.4, "tokens/trainable": 194213} +{"epoch": 0.73046875, "grad_norm": 1.8063968420028687, "learning_rate": 7.783436264872382e-05, "loss": 0.2714484632015228, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.31186, "step": 187, "tokens/total": 6493632, "tokens/train_per_sec_per_gpu": 56.2, "tokens/trainable": 195262} +{"epoch": 0.734375, "grad_norm": 6.667336463928223, "learning_rate": 7.758374768294647e-05, "loss": 0.25263187289237976, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.16, "memory/max_allocated (GiB)": 38.16, "ppl": 1.28741, "step": 188, "tokens/total": 6529568, "tokens/train_per_sec_per_gpu": 61.06, "tokens/trainable": 196357} +{"epoch": 0.73828125, "grad_norm": 3.8083245754241943, "learning_rate": 7.733219291524489e-05, "loss": 0.2545255422592163, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.53, "memory/max_allocated (GiB)": 37.53, "ppl": 1.28985, "step": 189, "tokens/total": 6562992, "tokens/train_per_sec_per_gpu": 62.54, "tokens/trainable": 197394} +{"epoch": 0.7421875, "grad_norm": 1.2971642017364502, "learning_rate": 7.707970881383977e-05, "loss": 0.2618001699447632, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.45, "memory/max_allocated (GiB)": 36.45, "ppl": 1.29927, "step": 190, "tokens/total": 6594704, "tokens/train_per_sec_per_gpu": 69.58, "tokens/trainable": 198431} +{"epoch": 0.74609375, "grad_norm": 2.2907447814941406, "learning_rate": 7.682630588562518e-05, "loss": 0.29773470759391785, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.7, "memory/max_allocated (GiB)": 38.7, "ppl": 1.3468, "step": 191, "tokens/total": 6629744, "tokens/train_per_sec_per_gpu": 57.8, "tokens/trainable": 199492} +{"epoch": 0.75, "grad_norm": 1.2444236278533936, "learning_rate": 7.657199467573129e-05, "loss": 0.2725798487663269, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.31335, "step": 192, "tokens/total": 6663872, "tokens/train_per_sec_per_gpu": 56.87, "tokens/trainable": 200512} +{"epoch": 0.75390625, "grad_norm": 3.0374531745910645, "learning_rate": 7.631678576708561e-05, "loss": 0.2501293420791626, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.16, "memory/max_allocated (GiB)": 38.16, "ppl": 1.28419, "step": 193, "tokens/total": 6698272, "tokens/train_per_sec_per_gpu": 67.58, "tokens/trainable": 201606} +{"epoch": 0.7578125, "grad_norm": 3.22800350189209, "learning_rate": 7.606068977997255e-05, "loss": 0.26776713132858276, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.64, "memory/max_allocated (GiB)": 39.64, "ppl": 1.30704, "step": 194, "tokens/total": 6735872, "tokens/train_per_sec_per_gpu": 66.41, "tokens/trainable": 202772} +{"epoch": 0.76171875, "grad_norm": 2.9739298820495605, "learning_rate": 7.580371737159148e-05, "loss": 0.30804798007011414, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.36077, "step": 195, "tokens/total": 6771968, "tokens/train_per_sec_per_gpu": 57.66, "tokens/trainable": 203825} +{"epoch": 0.765625, "grad_norm": 1.3312877416610718, "learning_rate": 7.554587923561324e-05, "loss": 0.26799237728118896, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.99, "memory/max_allocated (GiB)": 38.99, "ppl": 1.30734, "step": 196, "tokens/total": 6806672, "tokens/train_per_sec_per_gpu": 51.98, "tokens/trainable": 204797} +{"epoch": 0.76953125, "grad_norm": 1.6817588806152344, "learning_rate": 7.528718610173511e-05, "loss": 0.2508450746536255, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.28511, "step": 197, "tokens/total": 6842016, "tokens/train_per_sec_per_gpu": 63.5, "tokens/trainable": 205891} +{"epoch": 0.7734375, "grad_norm": 1.6923121213912964, "learning_rate": 7.502764873523431e-05, "loss": 0.2843756675720215, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.32893, "step": 198, "tokens/total": 6876064, "tokens/train_per_sec_per_gpu": 62.79, "tokens/trainable": 206929} +{"epoch": 0.77734375, "grad_norm": 1.7830978631973267, "learning_rate": 7.476727793652011e-05, "loss": 0.25847136974334717, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.72, "memory/max_allocated (GiB)": 36.72, "ppl": 1.29495, "step": 199, "tokens/total": 6909296, "tokens/train_per_sec_per_gpu": 62.78, "tokens/trainable": 207986} +{"epoch": 0.78125, "grad_norm": 1.3059806823730469, "learning_rate": 7.450608454068415e-05, "loss": 0.24153673648834229, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.2732, "step": 200, "tokens/total": 6943792, "tokens/train_per_sec_per_gpu": 57.99, "tokens/trainable": 209038} +{"epoch": 0.78515625, "grad_norm": 1.7610913515090942, "learning_rate": 7.424407941704987e-05, "loss": 0.23418524861335754, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.49, "memory/max_allocated (GiB)": 38.49, "ppl": 1.26388, "step": 201, "tokens/total": 6980448, "tokens/train_per_sec_per_gpu": 67.38, "tokens/trainable": 210183} +{"epoch": 0.7890625, "grad_norm": 1.260382890701294, "learning_rate": 7.398127346871986e-05, "loss": 0.2353108823299408, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.61, "memory/max_allocated (GiB)": 39.61, "ppl": 1.2653, "step": 202, "tokens/total": 7017392, "tokens/train_per_sec_per_gpu": 55.18, "tokens/trainable": 211226} +{"epoch": 0.79296875, "grad_norm": 1.4090571403503418, "learning_rate": 7.371767763212238e-05, "loss": 0.2569921016693115, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.52, "memory/max_allocated (GiB)": 39.52, "ppl": 1.29303, "step": 203, "tokens/total": 7052768, "tokens/train_per_sec_per_gpu": 62.69, "tokens/trainable": 212277} +{"epoch": 0.796875, "grad_norm": 1.5449470281600952, "learning_rate": 7.345330287655617e-05, "loss": 0.2765781283378601, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.09, "memory/max_allocated (GiB)": 39.09, "ppl": 1.31861, "step": 204, "tokens/total": 7090864, "tokens/train_per_sec_per_gpu": 48.84, "tokens/trainable": 213291} +{"epoch": 0.80078125, "grad_norm": 3.199521064758301, "learning_rate": 7.31881602037339e-05, "loss": 0.2795555591583252, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.69, "memory/max_allocated (GiB)": 36.69, "ppl": 1.32254, "step": 205, "tokens/total": 7122912, "tokens/train_per_sec_per_gpu": 64.0, "tokens/trainable": 214314} +{"epoch": 0.8046875, "grad_norm": 15.050874710083008, "learning_rate": 7.29222606473245e-05, "loss": 0.24579668045043945, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.27864, "step": 206, "tokens/total": 7155744, "tokens/train_per_sec_per_gpu": 61.62, "tokens/trainable": 215332} +{"epoch": 0.80859375, "grad_norm": 1.396051049232483, "learning_rate": 7.265561527249383e-05, "loss": 0.2651132345199585, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.30358, "step": 207, "tokens/total": 7189744, "tokens/train_per_sec_per_gpu": 61.1, "tokens/trainable": 216320} +{"epoch": 0.8125, "grad_norm": 1.2348895072937012, "learning_rate": 7.238823517544436e-05, "loss": 0.20524603128433228, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.22783, "step": 208, "tokens/total": 7226896, "tokens/train_per_sec_per_gpu": 67.29, "tokens/trainable": 217486} +{"epoch": 0.81640625, "grad_norm": 1.7681293487548828, "learning_rate": 7.212013148295333e-05, "loss": 0.2820417881011963, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.32583, "step": 209, "tokens/total": 7260880, "tokens/train_per_sec_per_gpu": 63.4, "tokens/trainable": 218565} +{"epoch": 0.8203125, "grad_norm": 1.0915597677230835, "learning_rate": 7.185131535190975e-05, "loss": 0.24135614931583405, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.06, "memory/max_allocated (GiB)": 37.06, "ppl": 1.27297, "step": 210, "tokens/total": 7293472, "tokens/train_per_sec_per_gpu": 65.09, "tokens/trainable": 219646} +{"epoch": 0.82421875, "grad_norm": 2.2716591358184814, "learning_rate": 7.158179796885005e-05, "loss": 0.2489476501941681, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.28267, "step": 211, "tokens/total": 7328096, "tokens/train_per_sec_per_gpu": 62.3, "tokens/trainable": 220699} +{"epoch": 0.828125, "grad_norm": 1.2649188041687012, "learning_rate": 7.131159054949273e-05, "loss": 0.27352502942085266, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.31459, "step": 212, "tokens/total": 7366480, "tokens/train_per_sec_per_gpu": 54.99, "tokens/trainable": 221738} +{"epoch": 0.83203125, "grad_norm": 1.2760515213012695, "learning_rate": 7.104070433827139e-05, "loss": 0.2670544683933258, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.30611, "step": 213, "tokens/total": 7401904, "tokens/train_per_sec_per_gpu": 50.34, "tokens/trainable": 222671} +{"epoch": 0.8359375, "grad_norm": 1.1511621475219727, "learning_rate": 7.076915060786705e-05, "loss": 0.25589412450790405, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.29162, "step": 214, "tokens/total": 7434720, "tokens/train_per_sec_per_gpu": 61.37, "tokens/trainable": 223705} +{"epoch": 0.83984375, "grad_norm": 1.902543306350708, "learning_rate": 7.049694065873882e-05, "loss": 0.2567714750766754, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.07, "memory/max_allocated (GiB)": 38.07, "ppl": 1.29275, "step": 215, "tokens/total": 7469376, "tokens/train_per_sec_per_gpu": 56.18, "tokens/trainable": 224755} +{"epoch": 0.84375, "grad_norm": 1.4245187044143677, "learning_rate": 7.022408581865382e-05, "loss": 0.24497577548027039, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.27759, "step": 216, "tokens/total": 7503520, "tokens/train_per_sec_per_gpu": 58.07, "tokens/trainable": 225832} +{"epoch": 0.84765625, "grad_norm": 1.6638760566711426, "learning_rate": 6.99505974422157e-05, "loss": 0.24472138285636902, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.27727, "step": 217, "tokens/total": 7538000, "tokens/train_per_sec_per_gpu": 65.88, "tokens/trainable": 226913} +{"epoch": 0.8515625, "grad_norm": 1.3509670495986938, "learning_rate": 6.967648691039213e-05, "loss": 0.2537113428115845, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.58, "memory/max_allocated (GiB)": 36.58, "ppl": 1.2888, "step": 218, "tokens/total": 7569872, "tokens/train_per_sec_per_gpu": 63.47, "tokens/trainable": 227933} +{"epoch": 0.85546875, "grad_norm": 1.5946557521820068, "learning_rate": 6.940176563004123e-05, "loss": 0.258381187915802, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.29483, "step": 219, "tokens/total": 7603920, "tokens/train_per_sec_per_gpu": 57.63, "tokens/trainable": 228906} +{"epoch": 0.859375, "grad_norm": 1.4015334844589233, "learning_rate": 6.912644503343682e-05, "loss": 0.26959481835365295, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.30943, "step": 220, "tokens/total": 7642080, "tokens/train_per_sec_per_gpu": 55.76, "tokens/trainable": 229915} +{"epoch": 0.86328125, "grad_norm": 1.4043185710906982, "learning_rate": 6.885053657779273e-05, "loss": 0.2318643033504486, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.28, "memory/max_allocated (GiB)": 36.28, "ppl": 1.26095, "step": 221, "tokens/total": 7673392, "tokens/train_per_sec_per_gpu": 59.67, "tokens/trainable": 230927} +{"epoch": 0.8671875, "grad_norm": 1.4754105806350708, "learning_rate": 6.857405174478604e-05, "loss": 0.2482573688030243, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.28179, "step": 222, "tokens/total": 7708096, "tokens/train_per_sec_per_gpu": 58.06, "tokens/trainable": 231937} +{"epoch": 0.87109375, "grad_norm": 1.3745635747909546, "learning_rate": 6.82970020400792e-05, "loss": 0.25224539637565613, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.28691, "step": 223, "tokens/total": 7745312, "tokens/train_per_sec_per_gpu": 53.55, "tokens/trainable": 232962} +{"epoch": 0.875, "grad_norm": 1.2515206336975098, "learning_rate": 6.801939899284132e-05, "loss": 0.20480048656463623, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.22728, "step": 224, "tokens/total": 7781328, "tokens/train_per_sec_per_gpu": 58.73, "tokens/trainable": 234041} +{"epoch": 0.87890625, "grad_norm": 1.2820345163345337, "learning_rate": 6.774125415526827e-05, "loss": 0.21104313433170319, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.23497, "step": 225, "tokens/total": 7814704, "tokens/train_per_sec_per_gpu": 70.99, "tokens/trainable": 235179} +{"epoch": 0.8828125, "grad_norm": 1.5814602375030518, "learning_rate": 6.746257910210214e-05, "loss": 0.22497189044952393, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.25229, "step": 226, "tokens/total": 7847936, "tokens/train_per_sec_per_gpu": 60.72, "tokens/trainable": 236200} +{"epoch": 0.88671875, "grad_norm": 1.9086357355117798, "learning_rate": 6.718338543014937e-05, "loss": 0.27018412947654724, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 35.66, "memory/max_allocated (GiB)": 35.66, "ppl": 1.31021, "step": 227, "tokens/total": 7877936, "tokens/train_per_sec_per_gpu": 65.45, "tokens/trainable": 237182} +{"epoch": 0.890625, "grad_norm": 1.3658757209777832, "learning_rate": 6.69036847577983e-05, "loss": 0.2367338240146637, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.2671, "step": 228, "tokens/total": 7911840, "tokens/train_per_sec_per_gpu": 61.34, "tokens/trainable": 238211} +{"epoch": 0.89453125, "grad_norm": 1.569017767906189, "learning_rate": 6.662348872453553e-05, "loss": 0.18647636473178864, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.46, "memory/max_allocated (GiB)": 38.46, "ppl": 1.205, "step": 229, "tokens/total": 7948400, "tokens/train_per_sec_per_gpu": 61.09, "tokens/trainable": 239279} +{"epoch": 0.8984375, "grad_norm": 1.964583396911621, "learning_rate": 6.63428089904618e-05, "loss": 0.2370176464319229, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.26746, "step": 230, "tokens/total": 7987568, "tokens/train_per_sec_per_gpu": 53.4, "tokens/trainable": 240318} +{"epoch": 0.90234375, "grad_norm": 2.492082357406616, "learning_rate": 6.60616572358065e-05, "loss": 0.24275925755500793, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.27476, "step": 231, "tokens/total": 8022032, "tokens/train_per_sec_per_gpu": 61.45, "tokens/trainable": 241289} +{"epoch": 0.90625, "grad_norm": 1.895829677581787, "learning_rate": 6.578004516044172e-05, "loss": 0.2535320222377777, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.28857, "step": 232, "tokens/total": 8057888, "tokens/train_per_sec_per_gpu": 61.74, "tokens/trainable": 242322} +{"epoch": 0.91015625, "grad_norm": 1.4279698133468628, "learning_rate": 6.549798448339548e-05, "loss": 0.21725277602672577, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.6, "memory/max_allocated (GiB)": 38.6, "ppl": 1.24266, "step": 233, "tokens/total": 8095296, "tokens/train_per_sec_per_gpu": 60.44, "tokens/trainable": 243378} +{"epoch": 0.9140625, "grad_norm": 1.5710076093673706, "learning_rate": 6.521548694236384e-05, "loss": 0.2413230538368225, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.27293, "step": 234, "tokens/total": 8132608, "tokens/train_per_sec_per_gpu": 54.63, "tokens/trainable": 244427} +{"epoch": 0.91796875, "grad_norm": 3.214977502822876, "learning_rate": 6.493256429322259e-05, "loss": 0.23724240064620972, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.26775, "step": 235, "tokens/total": 8168736, "tokens/train_per_sec_per_gpu": 55.7, "tokens/trainable": 245454} +{"epoch": 0.921875, "grad_norm": 3.030456066131592, "learning_rate": 6.464922830953799e-05, "loss": 0.25883978605270386, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.29543, "step": 236, "tokens/total": 8207952, "tokens/train_per_sec_per_gpu": 51.22, "tokens/trainable": 246465} +{"epoch": 0.92578125, "grad_norm": 39.28622055053711, "learning_rate": 6.436549078207688e-05, "loss": 0.22255028784275055, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.21, "memory/max_allocated (GiB)": 38.21, "ppl": 1.24926, "step": 237, "tokens/total": 8241040, "tokens/train_per_sec_per_gpu": 62.38, "tokens/trainable": 247465} +{"epoch": 0.9296875, "grad_norm": 1.8211487531661987, "learning_rate": 6.408136351831592e-05, "loss": 0.24998235702514648, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 37.15, "memory/max_allocated (GiB)": 37.15, "ppl": 1.284, "step": 238, "tokens/total": 8273408, "tokens/train_per_sec_per_gpu": 60.28, "tokens/trainable": 248462} +{"epoch": 0.93359375, "grad_norm": 1.212364912033081, "learning_rate": 6.379685834195036e-05, "loss": 0.21204254031181335, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.2362, "step": 239, "tokens/total": 8312912, "tokens/train_per_sec_per_gpu": 54.54, "tokens/trainable": 249607} +{"epoch": 0.9375, "grad_norm": 1.3512604236602783, "learning_rate": 6.351198709240186e-05, "loss": 0.22486847639083862, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.61, "memory/max_allocated (GiB)": 39.61, "ppl": 1.25216, "step": 240, "tokens/total": 8353040, "tokens/train_per_sec_per_gpu": 51.31, "tokens/trainable": 250667} +{"epoch": 0.94140625, "grad_norm": 1.2261786460876465, "learning_rate": 6.32267616243259e-05, "loss": 0.23954811692237854, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.27067, "step": 241, "tokens/total": 8385632, "tokens/train_per_sec_per_gpu": 63.78, "tokens/trainable": 251697} +{"epoch": 0.9453125, "grad_norm": 3.5921382904052734, "learning_rate": 6.294119380711849e-05, "loss": 0.2684933841228485, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.30799, "step": 242, "tokens/total": 8420064, "tokens/train_per_sec_per_gpu": 52.65, "tokens/trainable": 252654} +{"epoch": 0.94921875, "grad_norm": 1.3960481882095337, "learning_rate": 6.265529552442209e-05, "loss": 0.20416273176670074, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.2265, "step": 243, "tokens/total": 8455424, "tokens/train_per_sec_per_gpu": 63.08, "tokens/trainable": 253735} +{"epoch": 0.953125, "grad_norm": 1.1656215190887451, "learning_rate": 6.236907867363127e-05, "loss": 0.22583284974098206, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.14, "memory/max_allocated (GiB)": 38.14, "ppl": 1.25337, "step": 244, "tokens/total": 8489760, "tokens/train_per_sec_per_gpu": 63.48, "tokens/trainable": 254769} +{"epoch": 0.95703125, "grad_norm": 1.779144525527954, "learning_rate": 6.208255516539749e-05, "loss": 0.2122097611427307, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 36.34, "memory/max_allocated (GiB)": 36.34, "ppl": 1.23641, "step": 245, "tokens/total": 8521968, "tokens/train_per_sec_per_gpu": 60.52, "tokens/trainable": 255826} +{"epoch": 0.9609375, "grad_norm": 1.3846830129623413, "learning_rate": 6.179573692313344e-05, "loss": 0.23043283820152283, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 1.25914, "step": 246, "tokens/total": 8559088, "tokens/train_per_sec_per_gpu": 57.93, "tokens/trainable": 256838} +{"epoch": 0.96484375, "grad_norm": 1.3286389112472534, "learning_rate": 6.150863588251694e-05, "loss": 0.23686587810516357, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.26727, "step": 247, "tokens/total": 8599360, "tokens/train_per_sec_per_gpu": 50.34, "tokens/trainable": 257900} +{"epoch": 0.96875, "grad_norm": 1.4303414821624756, "learning_rate": 6.122126399099419e-05, "loss": 0.20139268040657043, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.78, "memory/max_allocated (GiB)": 39.78, "ppl": 1.2231, "step": 248, "tokens/total": 8639328, "tokens/train_per_sec_per_gpu": 66.52, "tokens/trainable": 259156} +{"epoch": 0.97265625, "grad_norm": 1.605102300643921, "learning_rate": 6.0933633207282615e-05, "loss": 0.27271825075149536, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.31353, "step": 249, "tokens/total": 8675696, "tokens/train_per_sec_per_gpu": 54.86, "tokens/trainable": 260223} +{"epoch": 0.9765625, "grad_norm": 2.5905869007110596, "learning_rate": 6.064575550087316e-05, "loss": 0.20350968837738037, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.63, "memory/max_allocated (GiB)": 39.63, "ppl": 1.2257, "step": 250, "tokens/total": 8712704, "tokens/train_per_sec_per_gpu": 60.55, "tokens/trainable": 261294} +{"epoch": 0.98046875, "grad_norm": 1.1556223630905151, "learning_rate": 6.0357642851532245e-05, "loss": 0.21436183154582977, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.23907, "step": 251, "tokens/total": 8749136, "tokens/train_per_sec_per_gpu": 54.95, "tokens/trainable": 262362} +{"epoch": 0.984375, "grad_norm": 1.498671293258667, "learning_rate": 6.0069307248803294e-05, "loss": 0.25773167610168457, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.29399, "step": 252, "tokens/total": 8784464, "tokens/train_per_sec_per_gpu": 50.45, "tokens/trainable": 263324} +{"epoch": 0.98828125, "grad_norm": 2.5805959701538086, "learning_rate": 5.9780760691507635e-05, "loss": 0.24480871856212616, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 36.25, "memory/max_allocated (GiB)": 36.25, "ppl": 1.27738, "step": 253, "tokens/total": 8815712, "tokens/train_per_sec_per_gpu": 61.36, "tokens/trainable": 264266} +{"epoch": 0.9921875, "grad_norm": 1.6032322645187378, "learning_rate": 5.9492015187245334e-05, "loss": 0.22702722251415253, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.72, "memory/max_allocated (GiB)": 38.72, "ppl": 1.25486, "step": 254, "tokens/total": 8853296, "tokens/train_per_sec_per_gpu": 54.7, "tokens/trainable": 265231} +{"epoch": 0.99609375, "grad_norm": 1.0861812829971313, "learning_rate": 5.920308275189541e-05, "loss": 0.2146262675523758, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 36.6, "memory/max_allocated (GiB)": 36.6, "ppl": 1.2394, "step": 255, "tokens/total": 8885248, "tokens/train_per_sec_per_gpu": 63.35, "tokens/trainable": 266241} +{"epoch": 1.0, "grad_norm": 1.1196694374084473, "learning_rate": 5.8913975409115874e-05, "loss": 0.20140525698661804, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.22312, "step": 256, "tokens/total": 8918608, "tokens/train_per_sec_per_gpu": 62.33, "tokens/trainable": 267300} +{"epoch": 1.00390625, "grad_norm": 1.4364045858383179, "learning_rate": 5.8624705189843395e-05, "loss": 0.18784549832344055, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.20665, "step": 257, "tokens/total": 8956064, "tokens/train_per_sec_per_gpu": 55.6, "tokens/trainable": 268407} +{"epoch": 1.0078125, "grad_norm": 2.735487937927246, "learning_rate": 5.833528413179249e-05, "loss": 0.19198289513587952, "memory/device_reserved (GiB)": 40.67, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.21165, "step": 258, "tokens/total": 8992688, "tokens/train_per_sec_per_gpu": 58.51, "tokens/trainable": 269500} +{"epoch": 1.01171875, "grad_norm": 1.6219083070755005, "learning_rate": 5.80457242789548e-05, "loss": 0.19124382734298706, "memory/device_reserved (GiB)": 42.74, "memory/max_active (GiB)": 39.65, "memory/max_allocated (GiB)": 39.65, "ppl": 1.21075, "step": 259, "tokens/total": 9032864, "tokens/train_per_sec_per_gpu": 57.21, "tokens/trainable": 270674} +{"epoch": 1.015625, "grad_norm": 1.3050200939178467, "learning_rate": 5.77560376810977e-05, "loss": 0.1650710254907608, "memory/device_reserved (GiB)": 42.75, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.17948, "step": 260, "tokens/total": 9066224, "tokens/train_per_sec_per_gpu": 68.21, "tokens/trainable": 271807} +{"epoch": 1.01953125, "grad_norm": 1.054490566253662, "learning_rate": 5.7466236393263005e-05, "loss": 0.19200611114501953, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.21168, "step": 261, "tokens/total": 9100848, "tokens/train_per_sec_per_gpu": 60.59, "tokens/trainable": 272826} +{"epoch": 1.0234375, "grad_norm": 1.6421701908111572, "learning_rate": 5.717633247526522e-05, "loss": 0.21388310194015503, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 36.74, "memory/max_allocated (GiB)": 36.74, "ppl": 1.23848, "step": 262, "tokens/total": 9131536, "tokens/train_per_sec_per_gpu": 57.55, "tokens/trainable": 273749} +{"epoch": 1.02734375, "grad_norm": 1.9772981405258179, "learning_rate": 5.688633799118971e-05, "loss": 0.22420328855514526, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.25133, "step": 263, "tokens/total": 9163984, "tokens/train_per_sec_per_gpu": 55.47, "tokens/trainable": 274687} +{"epoch": 1.03125, "grad_norm": 1.3114819526672363, "learning_rate": 5.659626500889066e-05, "loss": 0.16965538263320923, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.09, "memory/max_allocated (GiB)": 37.09, "ppl": 1.1849, "step": 264, "tokens/total": 9198000, "tokens/train_per_sec_per_gpu": 64.56, "tokens/trainable": 275813} +{"epoch": 1.03515625, "grad_norm": 1.2948765754699707, "learning_rate": 5.6306125599488905e-05, "loss": 0.20315894484519958, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 36.88, "memory/max_allocated (GiB)": 36.88, "ppl": 1.22527, "step": 265, "tokens/total": 9228848, "tokens/train_per_sec_per_gpu": 68.64, "tokens/trainable": 276828} +{"epoch": 1.0390625, "grad_norm": 1.2899636030197144, "learning_rate": 5.601593183686955e-05, "loss": 0.1734180748462677, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.18936, "step": 266, "tokens/total": 9266064, "tokens/train_per_sec_per_gpu": 59.84, "tokens/trainable": 277947} +{"epoch": 1.04296875, "grad_norm": 1.7671043872833252, "learning_rate": 5.572569579717961e-05, "loss": 0.19971683621406555, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.22106, "step": 267, "tokens/total": 9301472, "tokens/train_per_sec_per_gpu": 61.79, "tokens/trainable": 279038} +{"epoch": 1.046875, "grad_norm": 1.357511043548584, "learning_rate": 5.543542955832538e-05, "loss": 0.20424357056617737, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.2266, "step": 268, "tokens/total": 9341424, "tokens/train_per_sec_per_gpu": 52.64, "tokens/trainable": 280121} +{"epoch": 1.05078125, "grad_norm": 1.4490877389907837, "learning_rate": 5.514514519946986e-05, "loss": 0.17812034487724304, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.57, "memory/max_allocated (GiB)": 39.57, "ppl": 1.19497, "step": 269, "tokens/total": 9381840, "tokens/train_per_sec_per_gpu": 56.92, "tokens/trainable": 281236} +{"epoch": 1.0546875, "grad_norm": 1.266038179397583, "learning_rate": 5.485485480053015e-05, "loss": 0.18878084421157837, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.52, "memory/max_allocated (GiB)": 39.52, "ppl": 1.20778, "step": 270, "tokens/total": 9420224, "tokens/train_per_sec_per_gpu": 48.67, "tokens/trainable": 282225} +{"epoch": 1.05859375, "grad_norm": 0.9319303631782532, "learning_rate": 5.4564570441674645e-05, "loss": 0.17877736687660217, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.19575, "step": 271, "tokens/total": 9453888, "tokens/train_per_sec_per_gpu": 66.9, "tokens/trainable": 283267} +{"epoch": 1.0625, "grad_norm": 1.4157267808914185, "learning_rate": 5.42743042028204e-05, "loss": 0.21927189826965332, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.83, "memory/max_allocated (GiB)": 37.83, "ppl": 1.24517, "step": 272, "tokens/total": 9490080, "tokens/train_per_sec_per_gpu": 48.18, "tokens/trainable": 284230} +{"epoch": 1.06640625, "grad_norm": 1.0193302631378174, "learning_rate": 5.3984068163130464e-05, "loss": 0.16804879903793335, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.18299, "step": 273, "tokens/total": 9523840, "tokens/train_per_sec_per_gpu": 67.1, "tokens/trainable": 285389} +{"epoch": 1.0703125, "grad_norm": 1.2662354707717896, "learning_rate": 5.369387440051111e-05, "loss": 0.19833135604858398, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.18, "memory/max_allocated (GiB)": 37.18, "ppl": 1.21937, "step": 274, "tokens/total": 9557552, "tokens/train_per_sec_per_gpu": 58.79, "tokens/trainable": 286391} +{"epoch": 1.07421875, "grad_norm": 1.3956198692321777, "learning_rate": 5.340373499110935e-05, "loss": 0.170103520154953, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.18543, "step": 275, "tokens/total": 9593360, "tokens/train_per_sec_per_gpu": 59.75, "tokens/trainable": 287477} +{"epoch": 1.078125, "grad_norm": 0.8512800931930542, "learning_rate": 5.3113662008810304e-05, "loss": 0.17333123087882996, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.18926, "step": 276, "tokens/total": 9633696, "tokens/train_per_sec_per_gpu": 50.27, "tokens/trainable": 288537} +{"epoch": 1.08203125, "grad_norm": 1.1026732921600342, "learning_rate": 5.282366752473479e-05, "loss": 0.18163107335567474, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.29, "memory/max_allocated (GiB)": 38.29, "ppl": 1.19917, "step": 277, "tokens/total": 9670944, "tokens/train_per_sec_per_gpu": 57.96, "tokens/trainable": 289637} +{"epoch": 1.0859375, "grad_norm": 1.4298654794692993, "learning_rate": 5.2533763606737005e-05, "loss": 0.2075163722038269, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.28, "memory/max_allocated (GiB)": 39.28, "ppl": 1.23062, "step": 278, "tokens/total": 9710832, "tokens/train_per_sec_per_gpu": 45.5, "tokens/trainable": 290625} +{"epoch": 1.08984375, "grad_norm": 1.6798810958862305, "learning_rate": 5.224396231890232e-05, "loss": 0.1862269639968872, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.23, "memory/max_allocated (GiB)": 37.23, "ppl": 1.2047, "step": 279, "tokens/total": 9742608, "tokens/train_per_sec_per_gpu": 64.82, "tokens/trainable": 291724} +{"epoch": 1.09375, "grad_norm": 1.275308609008789, "learning_rate": 5.195427572104522e-05, "loss": 0.2074441909790039, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.23053, "step": 280, "tokens/total": 9777072, "tokens/train_per_sec_per_gpu": 59.29, "tokens/trainable": 292755} +{"epoch": 1.09765625, "grad_norm": 1.252453327178955, "learning_rate": 5.166471586820751e-05, "loss": 0.18259072303771973, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.20032, "step": 281, "tokens/total": 9813824, "tokens/train_per_sec_per_gpu": 61.12, "tokens/trainable": 293847} +{"epoch": 1.1015625, "grad_norm": 4.2824811935424805, "learning_rate": 5.1375294810156615e-05, "loss": 0.18308410048484802, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.20092, "step": 282, "tokens/total": 9848464, "tokens/train_per_sec_per_gpu": 59.75, "tokens/trainable": 294869} +{"epoch": 1.10546875, "grad_norm": 1.710822343826294, "learning_rate": 5.1086024590884144e-05, "loss": 0.2066424936056137, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.28, "memory/max_allocated (GiB)": 39.28, "ppl": 1.22954, "step": 283, "tokens/total": 9886720, "tokens/train_per_sec_per_gpu": 52.8, "tokens/trainable": 295870} +{"epoch": 1.109375, "grad_norm": 0.7628927230834961, "learning_rate": 5.079691724810461e-05, "loss": 0.18796177208423615, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.20679, "step": 284, "tokens/total": 9922592, "tokens/train_per_sec_per_gpu": 49.27, "tokens/trainable": 296839} +{"epoch": 1.11328125, "grad_norm": 1.0197120904922485, "learning_rate": 5.0507984812754684e-05, "loss": 0.20103955268859863, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.22267, "step": 285, "tokens/total": 9956144, "tokens/train_per_sec_per_gpu": 59.64, "tokens/trainable": 297886} +{"epoch": 1.1171875, "grad_norm": 1.121635913848877, "learning_rate": 5.021923930849237e-05, "loss": 0.18979287147521973, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.209, "step": 286, "tokens/total": 9990032, "tokens/train_per_sec_per_gpu": 56.68, "tokens/trainable": 298899} +{"epoch": 1.12109375, "grad_norm": 1.6564010381698608, "learning_rate": 4.99306927511967e-05, "loss": 0.19409191608428955, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.21421, "step": 287, "tokens/total": 10023504, "tokens/train_per_sec_per_gpu": 62.23, "tokens/trainable": 299906} +{"epoch": 1.125, "grad_norm": 0.9691503643989563, "learning_rate": 4.964235714846775e-05, "loss": 0.2098742127418518, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 36.44, "memory/max_allocated (GiB)": 36.44, "ppl": 1.23352, "step": 288, "tokens/total": 10055216, "tokens/train_per_sec_per_gpu": 66.27, "tokens/trainable": 300900} +{"epoch": 1.12890625, "grad_norm": 1.1443760395050049, "learning_rate": 4.9354244499126866e-05, "loss": 0.21085670590400696, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 36.23, "memory/max_allocated (GiB)": 36.23, "ppl": 1.23474, "step": 289, "tokens/total": 10086880, "tokens/train_per_sec_per_gpu": 63.16, "tokens/trainable": 301932} +{"epoch": 1.1328125, "grad_norm": 1.2828153371810913, "learning_rate": 4.90663667927174e-05, "loss": 0.21061232686042786, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.23443, "step": 290, "tokens/total": 10119712, "tokens/train_per_sec_per_gpu": 71.07, "tokens/trainable": 302959} +{"epoch": 1.13671875, "grad_norm": 1.1906269788742065, "learning_rate": 4.877873600900581e-05, "loss": 0.18908052146434784, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.07, "memory/max_allocated (GiB)": 38.07, "ppl": 1.20814, "step": 291, "tokens/total": 10155824, "tokens/train_per_sec_per_gpu": 53.95, "tokens/trainable": 303979} +{"epoch": 1.140625, "grad_norm": 0.9075404405593872, "learning_rate": 4.849136411748306e-05, "loss": 0.17171180248260498, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.04, "memory/max_allocated (GiB)": 39.04, "ppl": 1.18734, "step": 292, "tokens/total": 10193872, "tokens/train_per_sec_per_gpu": 53.66, "tokens/trainable": 305036} +{"epoch": 1.14453125, "grad_norm": 2.6495614051818848, "learning_rate": 4.8204263076866574e-05, "loss": 0.1948385387659073, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.21511, "step": 293, "tokens/total": 10229792, "tokens/train_per_sec_per_gpu": 57.57, "tokens/trainable": 306098} +{"epoch": 1.1484375, "grad_norm": 1.4235554933547974, "learning_rate": 4.791744483460251e-05, "loss": 0.20765164494514465, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.74, "memory/max_allocated (GiB)": 37.74, "ppl": 1.23078, "step": 294, "tokens/total": 10264288, "tokens/train_per_sec_per_gpu": 51.8, "tokens/trainable": 307092} +{"epoch": 1.15234375, "grad_norm": 1.0684980154037476, "learning_rate": 4.7630921326368736e-05, "loss": 0.19069021940231323, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.21008, "step": 295, "tokens/total": 10303008, "tokens/train_per_sec_per_gpu": 50.14, "tokens/trainable": 308074} +{"epoch": 1.15625, "grad_norm": 1.2863614559173584, "learning_rate": 4.7344704475577916e-05, "loss": 0.2030060589313507, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 35.7, "memory/max_allocated (GiB)": 35.7, "ppl": 1.22508, "step": 296, "tokens/total": 10332576, "tokens/train_per_sec_per_gpu": 68.31, "tokens/trainable": 309100} +{"epoch": 1.16015625, "grad_norm": 1.5791044235229492, "learning_rate": 4.705880619288153e-05, "loss": 0.19531835615634918, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.37, "memory/max_allocated (GiB)": 38.37, "ppl": 1.2157, "step": 297, "tokens/total": 10368800, "tokens/train_per_sec_per_gpu": 52.88, "tokens/trainable": 310085} +{"epoch": 1.1640625, "grad_norm": 0.8672658801078796, "learning_rate": 4.677323837567412e-05, "loss": 0.19599059224128723, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.21652, "step": 298, "tokens/total": 10401632, "tokens/train_per_sec_per_gpu": 64.52, "tokens/trainable": 311141} +{"epoch": 1.16796875, "grad_norm": 1.3278791904449463, "learning_rate": 4.6488012907598146e-05, "loss": 0.2108595371246338, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 36.4, "memory/max_allocated (GiB)": 36.4, "ppl": 1.23474, "step": 299, "tokens/total": 10433840, "tokens/train_per_sec_per_gpu": 57.37, "tokens/trainable": 312096} +{"epoch": 1.171875, "grad_norm": 0.9716861844062805, "learning_rate": 4.620314165804964e-05, "loss": 0.1764344871044159, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.19296, "step": 300, "tokens/total": 10469664, "tokens/train_per_sec_per_gpu": 58.61, "tokens/trainable": 313192} +{"epoch": 1.17578125, "grad_norm": 0.6432029604911804, "learning_rate": 4.591863648168407e-05, "loss": 0.16899779438972473, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.18412, "step": 301, "tokens/total": 10504224, "tokens/train_per_sec_per_gpu": 59.33, "tokens/trainable": 314288} +{"epoch": 1.1796875, "grad_norm": 0.8697813153266907, "learning_rate": 4.5634509217923135e-05, "loss": 0.19199290871620178, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.96, "memory/max_allocated (GiB)": 37.96, "ppl": 1.21166, "step": 302, "tokens/total": 10537760, "tokens/train_per_sec_per_gpu": 61.11, "tokens/trainable": 315326} +{"epoch": 1.18359375, "grad_norm": 0.8735583424568176, "learning_rate": 4.535077169046201e-05, "loss": 0.18900838494300842, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.03, "memory/max_allocated (GiB)": 38.03, "ppl": 1.20805, "step": 303, "tokens/total": 10574368, "tokens/train_per_sec_per_gpu": 57.08, "tokens/trainable": 316356} +{"epoch": 1.1875, "grad_norm": 3.727362871170044, "learning_rate": 4.506743570677743e-05, "loss": 0.20531237125396729, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 36.35, "memory/max_allocated (GiB)": 36.35, "ppl": 1.22791, "step": 304, "tokens/total": 10604688, "tokens/train_per_sec_per_gpu": 72.0, "tokens/trainable": 317354} +{"epoch": 1.19140625, "grad_norm": 2.7596309185028076, "learning_rate": 4.478451305763618e-05, "loss": 0.22732271254062653, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.25523, "step": 305, "tokens/total": 10638272, "tokens/train_per_sec_per_gpu": 64.06, "tokens/trainable": 318392} +{"epoch": 1.1953125, "grad_norm": 2.319118022918701, "learning_rate": 4.450201551660454e-05, "loss": 0.19190600514411926, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 36.41, "memory/max_allocated (GiB)": 36.41, "ppl": 1.21156, "step": 306, "tokens/total": 10669456, "tokens/train_per_sec_per_gpu": 63.27, "tokens/trainable": 319397} +{"epoch": 1.19921875, "grad_norm": 1.1012431383132935, "learning_rate": 4.4219954839558276e-05, "loss": 0.19792905449867249, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.21888, "step": 307, "tokens/total": 10704080, "tokens/train_per_sec_per_gpu": 53.79, "tokens/trainable": 320409} +{"epoch": 1.203125, "grad_norm": 1.7624433040618896, "learning_rate": 4.393834276419352e-05, "loss": 0.2078409194946289, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.23102, "step": 308, "tokens/total": 10735712, "tokens/train_per_sec_per_gpu": 67.22, "tokens/trainable": 321446} +{"epoch": 1.20703125, "grad_norm": 0.7122324705123901, "learning_rate": 4.36571910095382e-05, "loss": 0.16787737607955933, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.18279, "step": 309, "tokens/total": 10774144, "tokens/train_per_sec_per_gpu": 55.15, "tokens/trainable": 322500} +{"epoch": 1.2109375, "grad_norm": 1.2250076532363892, "learning_rate": 4.337651127546448e-05, "loss": 0.1947275847196579, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.21498, "step": 310, "tokens/total": 10809744, "tokens/train_per_sec_per_gpu": 61.87, "tokens/trainable": 323519} +{"epoch": 1.21484375, "grad_norm": 1.0646371841430664, "learning_rate": 4.3096315242201736e-05, "loss": 0.19039234519004822, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 36.51, "memory/max_allocated (GiB)": 36.51, "ppl": 1.20972, "step": 311, "tokens/total": 10842480, "tokens/train_per_sec_per_gpu": 58.9, "tokens/trainable": 324521} +{"epoch": 1.21875, "grad_norm": 0.8736791014671326, "learning_rate": 4.2816614569850635e-05, "loss": 0.19108349084854126, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.66, "memory/max_allocated (GiB)": 38.66, "ppl": 1.21056, "step": 312, "tokens/total": 10878256, "tokens/train_per_sec_per_gpu": 58.18, "tokens/trainable": 325521} +{"epoch": 1.22265625, "grad_norm": 0.9704684615135193, "learning_rate": 4.2537420897897864e-05, "loss": 0.18846501410007477, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.72, "memory/max_allocated (GiB)": 38.72, "ppl": 1.20739, "step": 313, "tokens/total": 10913904, "tokens/train_per_sec_per_gpu": 53.66, "tokens/trainable": 326541} +{"epoch": 1.2265625, "grad_norm": 1.143690586090088, "learning_rate": 4.225874584473174e-05, "loss": 0.19470861554145813, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.21496, "step": 314, "tokens/total": 10946448, "tokens/train_per_sec_per_gpu": 63.7, "tokens/trainable": 327555} +{"epoch": 1.23046875, "grad_norm": 1.7715413570404053, "learning_rate": 4.19806010071587e-05, "loss": 0.17253580689430237, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.18831, "step": 315, "tokens/total": 10984528, "tokens/train_per_sec_per_gpu": 52.6, "tokens/trainable": 328596} +{"epoch": 1.234375, "grad_norm": 1.6936639547348022, "learning_rate": 4.170299795992081e-05, "loss": 0.1970643699169159, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 36.88, "memory/max_allocated (GiB)": 36.88, "ppl": 1.21782, "step": 316, "tokens/total": 11018096, "tokens/train_per_sec_per_gpu": 60.2, "tokens/trainable": 329654} +{"epoch": 1.23828125, "grad_norm": 1.2747435569763184, "learning_rate": 4.142594825521398e-05, "loss": 0.1873282492160797, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.20602, "step": 317, "tokens/total": 11057200, "tokens/train_per_sec_per_gpu": 51.31, "tokens/trainable": 330711} +{"epoch": 1.2421875, "grad_norm": 0.7610248923301697, "learning_rate": 4.114946342220728e-05, "loss": 0.1642410010099411, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.1785, "step": 318, "tokens/total": 11093776, "tokens/train_per_sec_per_gpu": 56.52, "tokens/trainable": 331795} +{"epoch": 1.24609375, "grad_norm": 0.7620946764945984, "learning_rate": 4.087355496656321e-05, "loss": 0.16608086228370667, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.18067, "step": 319, "tokens/total": 11127408, "tokens/train_per_sec_per_gpu": 68.71, "tokens/trainable": 332917} +{"epoch": 1.25, "grad_norm": 0.9495767951011658, "learning_rate": 4.05982343699588e-05, "loss": 0.17500311136245728, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.19125, "step": 320, "tokens/total": 11166656, "tokens/train_per_sec_per_gpu": 58.23, "tokens/trainable": 334062} +{"epoch": 1.25390625, "grad_norm": 0.8241903185844421, "learning_rate": 4.0323513089607876e-05, "loss": 0.16388431191444397, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.17808, "step": 321, "tokens/total": 11201568, "tokens/train_per_sec_per_gpu": 57.65, "tokens/trainable": 335146} +{"epoch": 1.2578125, "grad_norm": 0.9345471858978271, "learning_rate": 4.004940255778431e-05, "loss": 0.16976366937160492, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.63, "memory/max_allocated (GiB)": 39.63, "ppl": 1.18502, "step": 322, "tokens/total": 11240288, "tokens/train_per_sec_per_gpu": 56.68, "tokens/trainable": 336203} +{"epoch": 1.26171875, "grad_norm": 1.5762282609939575, "learning_rate": 3.977591418134619e-05, "loss": 0.1934463083744049, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.21342, "step": 323, "tokens/total": 11275248, "tokens/train_per_sec_per_gpu": 61.24, "tokens/trainable": 337254} +{"epoch": 1.265625, "grad_norm": 1.167558193206787, "learning_rate": 3.95030593412612e-05, "loss": 0.1963164508342743, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.21691, "step": 324, "tokens/total": 11311152, "tokens/train_per_sec_per_gpu": 57.36, "tokens/trainable": 338256} +{"epoch": 1.26953125, "grad_norm": 0.7050957679748535, "learning_rate": 3.923084939213296e-05, "loss": 0.17899668216705322, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 35.75, "memory/max_allocated (GiB)": 35.75, "ppl": 1.19602, "step": 325, "tokens/total": 11341728, "tokens/train_per_sec_per_gpu": 62.49, "tokens/trainable": 339266} +{"epoch": 1.2734375, "grad_norm": 0.8405089378356934, "learning_rate": 3.895929566172861e-05, "loss": 0.14840273559093475, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.15998, "step": 326, "tokens/total": 11377744, "tokens/train_per_sec_per_gpu": 60.2, "tokens/trainable": 340370} +{"epoch": 1.27734375, "grad_norm": 0.6069943308830261, "learning_rate": 3.868840945050728e-05, "loss": 0.1590099036693573, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.19, "memory/max_allocated (GiB)": 39.19, "ppl": 1.17235, "step": 327, "tokens/total": 11416688, "tokens/train_per_sec_per_gpu": 56.06, "tokens/trainable": 341459} +{"epoch": 1.28125, "grad_norm": 0.8934356570243835, "learning_rate": 3.841820203114995e-05, "loss": 0.1726737916469574, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 36.78, "memory/max_allocated (GiB)": 36.78, "ppl": 1.18848, "step": 328, "tokens/total": 11448432, "tokens/train_per_sec_per_gpu": 67.3, "tokens/trainable": 342545} +{"epoch": 1.28515625, "grad_norm": 0.7047877907752991, "learning_rate": 3.814868464809027e-05, "loss": 0.1824255883693695, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 36.77, "memory/max_allocated (GiB)": 36.77, "ppl": 1.20012, "step": 329, "tokens/total": 11480368, "tokens/train_per_sec_per_gpu": 61.24, "tokens/trainable": 343541} +{"epoch": 1.2890625, "grad_norm": 1.3412995338439941, "learning_rate": 3.787986851704667e-05, "loss": 0.1747625172138214, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.19096, "step": 330, "tokens/total": 11513920, "tokens/train_per_sec_per_gpu": 63.48, "tokens/trainable": 344570} +{"epoch": 1.29296875, "grad_norm": 1.221191167831421, "learning_rate": 3.7611764824555654e-05, "loss": 0.1714784950017929, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.84, "memory/max_allocated (GiB)": 37.84, "ppl": 1.18706, "step": 331, "tokens/total": 11545648, "tokens/train_per_sec_per_gpu": 75.99, "tokens/trainable": 345723} +{"epoch": 1.296875, "grad_norm": 1.1699151992797852, "learning_rate": 3.734438472750619e-05, "loss": 0.16280168294906616, "memory/device_reserved (GiB)": 43.07, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.1768, "step": 332, "tokens/total": 11585616, "tokens/train_per_sec_per_gpu": 53.85, "tokens/trainable": 346834} +{"epoch": 1.30078125, "grad_norm": 2.0925214290618896, "learning_rate": 3.707773935267552e-05, "loss": 0.17885875701904297, "memory/device_reserved (GiB)": 43.07, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.19585, "step": 333, "tokens/total": 11622672, "tokens/train_per_sec_per_gpu": 57.82, "tokens/trainable": 347961} +{"epoch": 1.3046875, "grad_norm": 1.7085695266723633, "learning_rate": 3.68118397962661e-05, "loss": 0.20693130791187286, "memory/device_reserved (GiB)": 43.07, "memory/max_active (GiB)": 38.49, "memory/max_allocated (GiB)": 38.49, "ppl": 1.2299, "step": 334, "tokens/total": 11657424, "tokens/train_per_sec_per_gpu": 55.75, "tokens/trainable": 348923} +{"epoch": 1.30859375, "grad_norm": 1.3729617595672607, "learning_rate": 3.654669712344384e-05, "loss": 0.15753298997879028, "memory/device_reserved (GiB)": 43.07, "memory/max_active (GiB)": 36.72, "memory/max_allocated (GiB)": 36.72, "ppl": 1.17062, "step": 335, "tokens/total": 11688624, "tokens/train_per_sec_per_gpu": 74.59, "tokens/trainable": 349979} +{"epoch": 1.3125, "grad_norm": 0.8842703104019165, "learning_rate": 3.628232236787763e-05, "loss": 0.16027230024337769, "memory/device_reserved (GiB)": 43.07, "memory/max_active (GiB)": 37.91, "memory/max_allocated (GiB)": 37.91, "ppl": 1.17383, "step": 336, "tokens/total": 11724224, "tokens/train_per_sec_per_gpu": 56.21, "tokens/trainable": 351092} +{"epoch": 1.31640625, "grad_norm": 1.18330979347229, "learning_rate": 3.6018726531280144e-05, "loss": 0.17875054478645325, "memory/device_reserved (GiB)": 43.07, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.19572, "step": 337, "tokens/total": 11761632, "tokens/train_per_sec_per_gpu": 60.35, "tokens/trainable": 352182} +{"epoch": 1.3203125, "grad_norm": 0.7697396278381348, "learning_rate": 3.575592058295017e-05, "loss": 0.187938392162323, "memory/device_reserved (GiB)": 43.07, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.20676, "step": 338, "tokens/total": 11795616, "tokens/train_per_sec_per_gpu": 61.21, "tokens/trainable": 353234} +{"epoch": 1.32421875, "grad_norm": 1.0458109378814697, "learning_rate": 3.549391545931585e-05, "loss": 0.1850493848323822, "memory/device_reserved (GiB)": 43.07, "memory/max_active (GiB)": 38.93, "memory/max_allocated (GiB)": 38.93, "ppl": 1.20328, "step": 339, "tokens/total": 11830096, "tokens/train_per_sec_per_gpu": 62.26, "tokens/trainable": 354284} +{"epoch": 1.328125, "grad_norm": 1.020410418510437, "learning_rate": 3.5232722063479914e-05, "loss": 0.17965415120124817, "memory/device_reserved (GiB)": 43.07, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.1968, "step": 340, "tokens/total": 11864384, "tokens/train_per_sec_per_gpu": 63.62, "tokens/trainable": 355334} +{"epoch": 1.33203125, "grad_norm": 0.746391773223877, "learning_rate": 3.49723512647657e-05, "loss": 0.16856370866298676, "memory/device_reserved (GiB)": 43.07, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.1836, "step": 341, "tokens/total": 11903648, "tokens/train_per_sec_per_gpu": 54.17, "tokens/trainable": 356450} +{"epoch": 1.3359375, "grad_norm": 0.8197049498558044, "learning_rate": 3.471281389826491e-05, "loss": 0.18485745787620544, "memory/device_reserved (GiB)": 43.07, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.20305, "step": 342, "tokens/total": 11934704, "tokens/train_per_sec_per_gpu": 63.32, "tokens/trainable": 357403} +{"epoch": 1.33984375, "grad_norm": 0.7693317532539368, "learning_rate": 3.4454120764386764e-05, "loss": 0.17125928401947021, "memory/device_reserved (GiB)": 43.07, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.1868, "step": 343, "tokens/total": 11973328, "tokens/train_per_sec_per_gpu": 56.66, "tokens/trainable": 358483} +{"epoch": 1.34375, "grad_norm": 1.0048569440841675, "learning_rate": 3.4196282628408526e-05, "loss": 0.17696823179721832, "memory/device_reserved (GiB)": 43.07, "memory/max_active (GiB)": 37.58, "memory/max_allocated (GiB)": 37.58, "ppl": 1.19359, "step": 344, "tokens/total": 12007616, "tokens/train_per_sec_per_gpu": 61.8, "tokens/trainable": 359549} +{"epoch": 1.34765625, "grad_norm": 0.969716489315033, "learning_rate": 3.3939310220027456e-05, "loss": 0.16890740394592285, "memory/device_reserved (GiB)": 43.07, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.18401, "step": 345, "tokens/total": 12041648, "tokens/train_per_sec_per_gpu": 60.51, "tokens/trainable": 360642} +{"epoch": 1.3515625, "grad_norm": 0.790237307548523, "learning_rate": 3.3683214232914404e-05, "loss": 0.1637195348739624, "memory/device_reserved (GiB)": 43.07, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.17788, "step": 346, "tokens/total": 12077312, "tokens/train_per_sec_per_gpu": 59.78, "tokens/trainable": 361722} +{"epoch": 1.35546875, "grad_norm": 0.9851632118225098, "learning_rate": 3.342800532426873e-05, "loss": 0.19478441774845123, "memory/device_reserved (GiB)": 43.07, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.21505, "step": 347, "tokens/total": 12113296, "tokens/train_per_sec_per_gpu": 59.99, "tokens/trainable": 362740} +{"epoch": 1.359375, "grad_norm": 1.5783852338790894, "learning_rate": 3.317369411437484e-05, "loss": 0.1969670057296753, "memory/device_reserved (GiB)": 43.07, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.2177, "step": 348, "tokens/total": 12149168, "tokens/train_per_sec_per_gpu": 56.0, "tokens/trainable": 363755} +{"epoch": 1.36328125, "grad_norm": 0.6684873700141907, "learning_rate": 3.292029118616024e-05, "loss": 0.17864468693733215, "memory/device_reserved (GiB)": 43.07, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.1956, "step": 349, "tokens/total": 12181888, "tokens/train_per_sec_per_gpu": 56.94, "tokens/trainable": 364737} +{"epoch": 1.3671875, "grad_norm": 1.2652095556259155, "learning_rate": 3.266780708475511e-05, "loss": 0.17899660766124725, "memory/device_reserved (GiB)": 43.07, "memory/max_active (GiB)": 39.12, "memory/max_allocated (GiB)": 39.12, "ppl": 1.19602, "step": 350, "tokens/total": 12217296, "tokens/train_per_sec_per_gpu": 58.35, "tokens/trainable": 365812} +{"epoch": 1.37109375, "grad_norm": 0.619306743144989, "learning_rate": 3.241625231705354e-05, "loss": 0.1636940985918045, "memory/device_reserved (GiB)": 43.07, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.17785, "step": 351, "tokens/total": 12253344, "tokens/train_per_sec_per_gpu": 58.65, "tokens/trainable": 366870} +{"epoch": 1.375, "grad_norm": 1.179133653640747, "learning_rate": 3.216563735127618e-05, "loss": 0.15917247533798218, "memory/device_reserved (GiB)": 43.07, "memory/max_active (GiB)": 39.12, "memory/max_allocated (GiB)": 39.12, "ppl": 1.17254, "step": 352, "tokens/total": 12288480, "tokens/train_per_sec_per_gpu": 61.09, "tokens/trainable": 368003} +{"epoch": 1.37890625, "grad_norm": 1.6805800199508667, "learning_rate": 3.191597261653475e-05, "loss": 0.18650734424591064, "memory/device_reserved (GiB)": 43.07, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.20503, "step": 353, "tokens/total": 12325248, "tokens/train_per_sec_per_gpu": 57.7, "tokens/trainable": 369082} +{"epoch": 1.3828125, "grad_norm": 0.9155021905899048, "learning_rate": 3.166726850239794e-05, "loss": 0.16396090388298035, "memory/device_reserved (GiB)": 43.07, "memory/max_active (GiB)": 37.18, "memory/max_allocated (GiB)": 37.18, "ppl": 1.17817, "step": 354, "tokens/total": 12358880, "tokens/train_per_sec_per_gpu": 62.53, "tokens/trainable": 370160} +{"epoch": 1.38671875, "grad_norm": 1.426425576210022, "learning_rate": 3.141953535845912e-05, "loss": 0.2035723477602005, "memory/device_reserved (GiB)": 43.07, "memory/max_active (GiB)": 36.16, "memory/max_allocated (GiB)": 36.16, "ppl": 1.22577, "step": 355, "tokens/total": 12389696, "tokens/train_per_sec_per_gpu": 63.0, "tokens/trainable": 371098} +{"epoch": 1.390625, "grad_norm": 1.1798394918441772, "learning_rate": 3.11727834939056e-05, "loss": 0.17939946055412292, "memory/device_reserved (GiB)": 43.07, "memory/max_active (GiB)": 36.28, "memory/max_allocated (GiB)": 36.28, "ppl": 1.1965, "step": 356, "tokens/total": 12421536, "tokens/train_per_sec_per_gpu": 59.82, "tokens/trainable": 372131} +{"epoch": 1.39453125, "grad_norm": 0.8709330558776855, "learning_rate": 3.092702317708967e-05, "loss": 0.17493970692157745, "memory/device_reserved (GiB)": 43.07, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.19117, "step": 357, "tokens/total": 12455264, "tokens/train_per_sec_per_gpu": 62.39, "tokens/trainable": 373159} +{"epoch": 1.3984375, "grad_norm": 0.6055595278739929, "learning_rate": 3.0682264635101276e-05, "loss": 0.1628212332725525, "memory/device_reserved (GiB)": 43.07, "memory/max_active (GiB)": 37.44, "memory/max_allocated (GiB)": 37.44, "ppl": 1.17683, "step": 358, "tokens/total": 12487776, "tokens/train_per_sec_per_gpu": 62.2, "tokens/trainable": 374180} +{"epoch": 1.40234375, "grad_norm": 1.0339452028274536, "learning_rate": 3.0438518053342407e-05, "loss": 0.17326770722866058, "memory/device_reserved (GiB)": 43.07, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.18918, "step": 359, "tokens/total": 12526848, "tokens/train_per_sec_per_gpu": 55.86, "tokens/trainable": 375308} +{"epoch": 1.40625, "grad_norm": 0.8895189762115479, "learning_rate": 3.0195793575103266e-05, "loss": 0.16202722489833832, "memory/device_reserved (GiB)": 43.07, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.17589, "step": 360, "tokens/total": 12565888, "tokens/train_per_sec_per_gpu": 54.93, "tokens/trainable": 376384} +{"epoch": 1.41015625, "grad_norm": 1.0593047142028809, "learning_rate": 2.9954101301140146e-05, "loss": 0.1778571903705597, "memory/device_reserved (GiB)": 43.07, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.19465, "step": 361, "tokens/total": 12601232, "tokens/train_per_sec_per_gpu": 58.84, "tokens/trainable": 377425} +{"epoch": 1.4140625, "grad_norm": 0.5375335216522217, "learning_rate": 2.9713451289255123e-05, "loss": 0.16343799233436584, "memory/device_reserved (GiB)": 43.07, "memory/max_active (GiB)": 36.9, "memory/max_allocated (GiB)": 36.9, "ppl": 1.17755, "step": 362, "tokens/total": 12630624, "tokens/train_per_sec_per_gpu": 63.44, "tokens/trainable": 378465} +{"epoch": 1.41796875, "grad_norm": 1.3781670331954956, "learning_rate": 2.9473853553877484e-05, "loss": 0.15976381301879883, "memory/device_reserved (GiB)": 43.07, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.17323, "step": 363, "tokens/total": 12664560, "tokens/train_per_sec_per_gpu": 60.36, "tokens/trainable": 379504} +{"epoch": 1.421875, "grad_norm": 0.7382871508598328, "learning_rate": 2.9235318065647e-05, "loss": 0.16429153084754944, "memory/device_reserved (GiB)": 43.07, "memory/max_active (GiB)": 39.61, "memory/max_allocated (GiB)": 39.61, "ppl": 1.17856, "step": 364, "tokens/total": 12704576, "tokens/train_per_sec_per_gpu": 53.47, "tokens/trainable": 380566} +{"epoch": 1.42578125, "grad_norm": 1.224761962890625, "learning_rate": 2.8997854750998964e-05, "loss": 0.16644001007080078, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.18109, "step": 365, "tokens/total": 12744096, "tokens/train_per_sec_per_gpu": 56.62, "tokens/trainable": 381661} +{"epoch": 1.4296875, "grad_norm": 1.7043906450271606, "learning_rate": 2.8761473491751258e-05, "loss": 0.17970138788223267, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 36.63, "memory/max_allocated (GiB)": 36.63, "ppl": 1.19686, "step": 366, "tokens/total": 12776928, "tokens/train_per_sec_per_gpu": 59.92, "tokens/trainable": 382659} +{"epoch": 1.43359375, "grad_norm": 0.660465657711029, "learning_rate": 2.8526184124692883e-05, "loss": 0.14525333046913147, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.15633, "step": 367, "tokens/total": 12813584, "tokens/train_per_sec_per_gpu": 52.65, "tokens/trainable": 383762} +{"epoch": 1.4375, "grad_norm": 0.7520913481712341, "learning_rate": 2.829199644117484e-05, "loss": 0.1733115315437317, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.18924, "step": 368, "tokens/total": 12847296, "tokens/train_per_sec_per_gpu": 62.89, "tokens/trainable": 384725} +{"epoch": 1.44140625, "grad_norm": 0.5222395658493042, "learning_rate": 2.8058920186702553e-05, "loss": 0.15109939873218536, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.91, "memory/max_allocated (GiB)": 38.91, "ppl": 1.16311, "step": 369, "tokens/total": 12885424, "tokens/train_per_sec_per_gpu": 58.86, "tokens/trainable": 385819} +{"epoch": 1.4453125, "grad_norm": 0.8149802684783936, "learning_rate": 2.782696506053033e-05, "loss": 0.17936915159225464, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.19646, "step": 370, "tokens/total": 12923616, "tokens/train_per_sec_per_gpu": 52.65, "tokens/trainable": 386814} +{"epoch": 1.44921875, "grad_norm": 0.7995320558547974, "learning_rate": 2.7596140715257824e-05, "loss": 0.16263411939144135, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.17661, "step": 371, "tokens/total": 12958896, "tokens/train_per_sec_per_gpu": 61.51, "tokens/trainable": 387909} +{"epoch": 1.453125, "grad_norm": 1.6433626413345337, "learning_rate": 2.7366456756428184e-05, "loss": 0.16639591753482819, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.18104, "step": 372, "tokens/total": 12993264, "tokens/train_per_sec_per_gpu": 60.08, "tokens/trainable": 388945} +{"epoch": 1.45703125, "grad_norm": 1.5926343202590942, "learning_rate": 2.7137922742128486e-05, "loss": 0.17202690243721008, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.21, "memory/max_allocated (GiB)": 38.21, "ppl": 1.18771, "step": 373, "tokens/total": 13028144, "tokens/train_per_sec_per_gpu": 61.45, "tokens/trainable": 390012} +{"epoch": 1.4609375, "grad_norm": 1.7165155410766602, "learning_rate": 2.691054818259188e-05, "loss": 0.15641777217388153, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.4, "memory/max_allocated (GiB)": 37.4, "ppl": 1.16931, "step": 374, "tokens/total": 13062816, "tokens/train_per_sec_per_gpu": 60.16, "tokens/trainable": 391113} +{"epoch": 1.46484375, "grad_norm": 1.9620907306671143, "learning_rate": 2.6684342539801933e-05, "loss": 0.19146448373794556, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 36.49, "memory/max_allocated (GiB)": 36.49, "ppl": 1.21102, "step": 375, "tokens/total": 13094080, "tokens/train_per_sec_per_gpu": 61.3, "tokens/trainable": 392093} +{"epoch": 1.46875, "grad_norm": 0.6846341490745544, "learning_rate": 2.645931522709877e-05, "loss": 0.15332791209220886, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 39.54, "memory/max_allocated (GiB)": 39.54, "ppl": 1.16571, "step": 376, "tokens/total": 13129824, "tokens/train_per_sec_per_gpu": 72.68, "tokens/trainable": 393260} +{"epoch": 1.47265625, "grad_norm": 2.3481831550598145, "learning_rate": 2.6235475608787365e-05, "loss": 0.17622146010398865, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.6, "memory/max_allocated (GiB)": 38.6, "ppl": 1.1927, "step": 377, "tokens/total": 13165552, "tokens/train_per_sec_per_gpu": 59.02, "tokens/trainable": 394289} +{"epoch": 1.4765625, "grad_norm": 1.332844853401184, "learning_rate": 2.6012832999747916e-05, "loss": 0.18001556396484375, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.19724, "step": 378, "tokens/total": 13196336, "tokens/train_per_sec_per_gpu": 63.65, "tokens/trainable": 395275} +{"epoch": 1.48046875, "grad_norm": 9.374658584594727, "learning_rate": 2.579139666504821e-05, "loss": 0.18049241602420807, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.19781, "step": 379, "tokens/total": 13236400, "tokens/train_per_sec_per_gpu": 51.54, "tokens/trainable": 396280} +{"epoch": 1.484375, "grad_norm": 1.1409697532653809, "learning_rate": 2.557117581955798e-05, "loss": 0.17835326492786407, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 1.19525, "step": 380, "tokens/total": 13270224, "tokens/train_per_sec_per_gpu": 59.39, "tokens/trainable": 397346} +{"epoch": 1.48828125, "grad_norm": 0.8570708632469177, "learning_rate": 2.5352179627565532e-05, "loss": 0.17615759372711182, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.06, "memory/max_allocated (GiB)": 38.06, "ppl": 1.19263, "step": 381, "tokens/total": 13305248, "tokens/train_per_sec_per_gpu": 54.37, "tokens/trainable": 398354} +{"epoch": 1.4921875, "grad_norm": 0.6811918020248413, "learning_rate": 2.5134417202396277e-05, "loss": 0.18587276339530945, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.20427, "step": 382, "tokens/total": 13338848, "tokens/train_per_sec_per_gpu": 57.42, "tokens/trainable": 399323} +{"epoch": 1.49609375, "grad_norm": 1.5555872917175293, "learning_rate": 2.491789760603361e-05, "loss": 0.1934916228055954, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.21348, "step": 383, "tokens/total": 13375344, "tokens/train_per_sec_per_gpu": 53.23, "tokens/trainable": 400339} +{"epoch": 1.5, "grad_norm": 1.3572927713394165, "learning_rate": 2.4702629848741764e-05, "loss": 0.17218518257141113, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.1879, "step": 384, "tokens/total": 13409984, "tokens/train_per_sec_per_gpu": 62.19, "tokens/trainable": 401343} +{"epoch": 1.50390625, "grad_norm": 1.5164259672164917, "learning_rate": 2.4488622888690785e-05, "loss": 0.1738002449274063, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.18982, "step": 385, "tokens/total": 13443568, "tokens/train_per_sec_per_gpu": 54.93, "tokens/trainable": 402319} +{"epoch": 1.5078125, "grad_norm": 0.8501739501953125, "learning_rate": 2.427588563158384e-05, "loss": 0.16068214178085327, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 39.11, "memory/max_allocated (GiB)": 39.11, "ppl": 1.17431, "step": 386, "tokens/total": 13477920, "tokens/train_per_sec_per_gpu": 62.23, "tokens/trainable": 403393} +{"epoch": 1.51171875, "grad_norm": 0.7260568737983704, "learning_rate": 2.406442693028651e-05, "loss": 0.18881657719612122, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.20782, "step": 387, "tokens/total": 13510368, "tokens/train_per_sec_per_gpu": 59.78, "tokens/trainable": 404342} +{"epoch": 1.515625, "grad_norm": 0.8046561479568481, "learning_rate": 2.3854255584458547e-05, "loss": 0.17612811923027039, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.19259, "step": 388, "tokens/total": 13549344, "tokens/train_per_sec_per_gpu": 51.17, "tokens/trainable": 405359} +{"epoch": 1.51953125, "grad_norm": 5.746936321258545, "learning_rate": 2.3645380340187508e-05, "loss": 0.1655610203742981, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 39.14, "memory/max_allocated (GiB)": 39.14, "ppl": 1.18005, "step": 389, "tokens/total": 13584912, "tokens/train_per_sec_per_gpu": 62.4, "tokens/trainable": 406433} +{"epoch": 1.5234375, "grad_norm": 6.535442352294922, "learning_rate": 2.3437809889624914e-05, "loss": 0.18696409463882446, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.20558, "step": 390, "tokens/total": 13617712, "tokens/train_per_sec_per_gpu": 69.93, "tokens/trainable": 407504} +{"epoch": 1.52734375, "grad_norm": 0.50746089220047, "learning_rate": 2.3231552870624487e-05, "loss": 0.1596231609582901, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.17307, "step": 391, "tokens/total": 13648592, "tokens/train_per_sec_per_gpu": 70.33, "tokens/trainable": 408577} +{"epoch": 1.53125, "grad_norm": 0.6317844986915588, "learning_rate": 2.3026617866382657e-05, "loss": 0.15122409164905548, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 36.7, "memory/max_allocated (GiB)": 36.7, "ppl": 1.16326, "step": 392, "tokens/total": 13681600, "tokens/train_per_sec_per_gpu": 62.51, "tokens/trainable": 409663} +{"epoch": 1.53515625, "grad_norm": 0.641849160194397, "learning_rate": 2.2823013405081507e-05, "loss": 0.15815426409244537, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.17135, "step": 393, "tokens/total": 13719776, "tokens/train_per_sec_per_gpu": 53.36, "tokens/trainable": 410735} +{"epoch": 1.5390625, "grad_norm": 0.668559730052948, "learning_rate": 2.2620747959533722e-05, "loss": 0.1573287546634674, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.17038, "step": 394, "tokens/total": 13753088, "tokens/train_per_sec_per_gpu": 65.44, "tokens/trainable": 411846} +{"epoch": 1.54296875, "grad_norm": 0.7597647905349731, "learning_rate": 2.2419829946830123e-05, "loss": 0.16319991648197174, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.17727, "step": 395, "tokens/total": 13788864, "tokens/train_per_sec_per_gpu": 59.86, "tokens/trainable": 412885} +{"epoch": 1.546875, "grad_norm": 0.5389333367347717, "learning_rate": 2.2220267727989325e-05, "loss": 0.1506553292274475, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.74, "memory/max_allocated (GiB)": 37.74, "ppl": 1.1626, "step": 396, "tokens/total": 13823232, "tokens/train_per_sec_per_gpu": 58.27, "tokens/trainable": 413929} +{"epoch": 1.55078125, "grad_norm": 0.6055253148078918, "learning_rate": 2.202206960760984e-05, "loss": 0.1512376368045807, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.16327, "step": 397, "tokens/total": 13861488, "tokens/train_per_sec_per_gpu": 55.96, "tokens/trainable": 415076} +{"epoch": 1.5546875, "grad_norm": 2.876340389251709, "learning_rate": 2.182524383352446e-05, "loss": 0.17328602075576782, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.18921, "step": 398, "tokens/total": 13897648, "tokens/train_per_sec_per_gpu": 58.91, "tokens/trainable": 416117} +{"epoch": 1.55859375, "grad_norm": 0.7285708785057068, "learning_rate": 2.1629798596457056e-05, "loss": 0.17144033312797546, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 36.34, "memory/max_allocated (GiB)": 36.34, "ppl": 1.18701, "step": 399, "tokens/total": 13929744, "tokens/train_per_sec_per_gpu": 62.85, "tokens/trainable": 417122} +{"epoch": 1.5625, "grad_norm": 1.2036237716674805, "learning_rate": 2.1435742029681725e-05, "loss": 0.17364361882209778, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.18963, "step": 400, "tokens/total": 13962032, "tokens/train_per_sec_per_gpu": 69.43, "tokens/trainable": 418126} +{"epoch": 1.56640625, "grad_norm": 0.5587677955627441, "learning_rate": 2.124308220868431e-05, "loss": 0.1527455598115921, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.16503, "step": 401, "tokens/total": 13997888, "tokens/train_per_sec_per_gpu": 56.72, "tokens/trainable": 419187} +{"epoch": 1.5703125, "grad_norm": 0.5325484275817871, "learning_rate": 2.105182715082638e-05, "loss": 0.153541699051857, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 36.92, "memory/max_allocated (GiB)": 36.92, "ppl": 1.16596, "step": 402, "tokens/total": 14031632, "tokens/train_per_sec_per_gpu": 63.2, "tokens/trainable": 420280} +{"epoch": 1.57421875, "grad_norm": 0.9916512966156006, "learning_rate": 2.0861984815011552e-05, "loss": 0.17943784594535828, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 1.19654, "step": 403, "tokens/total": 14066896, "tokens/train_per_sec_per_gpu": 55.46, "tokens/trainable": 421251} +{"epoch": 1.578125, "grad_norm": 1.6591204404830933, "learning_rate": 2.0673563101354323e-05, "loss": 0.17682895064353943, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.19343, "step": 404, "tokens/total": 14102176, "tokens/train_per_sec_per_gpu": 52.76, "tokens/trainable": 422269} +{"epoch": 1.58203125, "grad_norm": 0.6385357975959778, "learning_rate": 2.0486569850851317e-05, "loss": 0.16665273904800415, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.18134, "step": 405, "tokens/total": 14136944, "tokens/train_per_sec_per_gpu": 61.34, "tokens/trainable": 423285} +{"epoch": 1.5859375, "grad_norm": 0.4822444021701813, "learning_rate": 2.0301012845054956e-05, "loss": 0.15789970755577087, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.17105, "step": 406, "tokens/total": 14175824, "tokens/train_per_sec_per_gpu": 51.69, "tokens/trainable": 424327} +{"epoch": 1.58984375, "grad_norm": 0.9487046599388123, "learning_rate": 2.011689980574966e-05, "loss": 0.1668441891670227, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.18, "memory/max_allocated (GiB)": 38.18, "ppl": 1.18157, "step": 407, "tokens/total": 14212208, "tokens/train_per_sec_per_gpu": 55.15, "tokens/trainable": 425331} +{"epoch": 1.59375, "grad_norm": 0.58551424741745, "learning_rate": 1.993423839463052e-05, "loss": 0.15342408418655396, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.16582, "step": 408, "tokens/total": 14245648, "tokens/train_per_sec_per_gpu": 71.39, "tokens/trainable": 426418} +{"epoch": 1.59765625, "grad_norm": 0.5517334938049316, "learning_rate": 1.975303621298445e-05, "loss": 0.15863856673240662, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.17191, "step": 409, "tokens/total": 14282400, "tokens/train_per_sec_per_gpu": 57.18, "tokens/trainable": 427435} +{"epoch": 1.6015625, "grad_norm": 1.1215676069259644, "learning_rate": 1.957330080137385e-05, "loss": 0.1718997061252594, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 36.22, "memory/max_allocated (GiB)": 36.22, "ppl": 1.18756, "step": 410, "tokens/total": 14312976, "tokens/train_per_sec_per_gpu": 65.58, "tokens/trainable": 428450} +{"epoch": 1.60546875, "grad_norm": 1.3351881504058838, "learning_rate": 1.9395039639322864e-05, "loss": 0.15863056480884552, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.1719, "step": 411, "tokens/total": 14349104, "tokens/train_per_sec_per_gpu": 56.14, "tokens/trainable": 429470} +{"epoch": 1.609375, "grad_norm": 0.5305394530296326, "learning_rate": 1.9218260145006073e-05, "loss": 0.15328359603881836, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.16566, "step": 412, "tokens/total": 14384976, "tokens/train_per_sec_per_gpu": 64.51, "tokens/trainable": 430607} +{"epoch": 1.61328125, "grad_norm": 0.4982529878616333, "learning_rate": 1.904296967493982e-05, "loss": 0.15336811542510986, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.16575, "step": 413, "tokens/total": 14417440, "tokens/train_per_sec_per_gpu": 69.18, "tokens/trainable": 431652} +{"epoch": 1.6171875, "grad_norm": 1.8888440132141113, "learning_rate": 1.8869175523676064e-05, "loss": 0.14388802647590637, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.36, "memory/max_allocated (GiB)": 38.36, "ppl": 1.15475, "step": 414, "tokens/total": 14454224, "tokens/train_per_sec_per_gpu": 61.03, "tokens/trainable": 432787} +{"epoch": 1.62109375, "grad_norm": 0.8893738985061646, "learning_rate": 1.869688492349885e-05, "loss": 0.15969857573509216, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.45, "memory/max_allocated (GiB)": 37.45, "ppl": 1.17316, "step": 415, "tokens/total": 14487488, "tokens/train_per_sec_per_gpu": 63.64, "tokens/trainable": 433849} +{"epoch": 1.625, "grad_norm": 0.7236246466636658, "learning_rate": 1.85261050441233e-05, "loss": 0.1584344506263733, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.17168, "step": 416, "tokens/total": 14523632, "tokens/train_per_sec_per_gpu": 58.23, "tokens/trainable": 434925} +{"epoch": 1.62890625, "grad_norm": 1.021560549736023, "learning_rate": 1.8356842992397304e-05, "loss": 0.18975841999053955, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.23, "memory/max_allocated (GiB)": 37.23, "ppl": 1.20896, "step": 417, "tokens/total": 14557504, "tokens/train_per_sec_per_gpu": 58.82, "tokens/trainable": 435922} +{"epoch": 1.6328125, "grad_norm": 0.743442952632904, "learning_rate": 1.8189105812005714e-05, "loss": 0.17645493149757385, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.19298, "step": 418, "tokens/total": 14594256, "tokens/train_per_sec_per_gpu": 53.3, "tokens/trainable": 436867} +{"epoch": 1.63671875, "grad_norm": 1.701016902923584, "learning_rate": 1.802290048317732e-05, "loss": 0.17117753624916077, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.1867, "step": 419, "tokens/total": 14627856, "tokens/train_per_sec_per_gpu": 59.14, "tokens/trainable": 437873} +{"epoch": 1.640625, "grad_norm": 0.8529032468795776, "learning_rate": 1.785823392239424e-05, "loss": 0.1622682511806488, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.82, "memory/max_allocated (GiB)": 37.82, "ppl": 1.17618, "step": 420, "tokens/total": 14661152, "tokens/train_per_sec_per_gpu": 58.46, "tokens/trainable": 438831} +{"epoch": 1.64453125, "grad_norm": 0.8341173529624939, "learning_rate": 1.7695112982104225e-05, "loss": 0.16317227482795715, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 39.78, "memory/max_allocated (GiB)": 39.78, "ppl": 1.17724, "step": 421, "tokens/total": 14696400, "tokens/train_per_sec_per_gpu": 61.96, "tokens/trainable": 439884} +{"epoch": 1.6484375, "grad_norm": 0.6243525743484497, "learning_rate": 1.7533544450435433e-05, "loss": 0.16573789715766907, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 35.69, "memory/max_allocated (GiB)": 35.69, "ppl": 1.18026, "step": 422, "tokens/total": 14725824, "tokens/train_per_sec_per_gpu": 75.83, "tokens/trainable": 440922} +{"epoch": 1.65234375, "grad_norm": 1.2397549152374268, "learning_rate": 1.7373535050913946e-05, "loss": 0.16666468977928162, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.18136, "step": 423, "tokens/total": 14762288, "tokens/train_per_sec_per_gpu": 52.22, "tokens/trainable": 441946} +{"epoch": 1.65625, "grad_norm": 2.0691006183624268, "learning_rate": 1.721509144218405e-05, "loss": 0.1684798300266266, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 35.68, "memory/max_allocated (GiB)": 35.68, "ppl": 1.1835, "step": 424, "tokens/total": 14792272, "tokens/train_per_sec_per_gpu": 73.03, "tokens/trainable": 442963} +{"epoch": 1.66015625, "grad_norm": 0.6998986601829529, "learning_rate": 1.705822021773101e-05, "loss": 0.16524255275726318, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.17968, "step": 425, "tokens/total": 14826736, "tokens/train_per_sec_per_gpu": 60.12, "tokens/trainable": 444009} +{"epoch": 1.6640625, "grad_norm": 0.7652738094329834, "learning_rate": 1.69029279056068e-05, "loss": 0.16944855451583862, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.99, "memory/max_allocated (GiB)": 38.99, "ppl": 1.18465, "step": 426, "tokens/total": 14864176, "tokens/train_per_sec_per_gpu": 54.98, "tokens/trainable": 445030} +{"epoch": 1.66796875, "grad_norm": 0.5872130990028381, "learning_rate": 1.6749220968158415e-05, "loss": 0.160711407661438, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.17435, "step": 427, "tokens/total": 14899296, "tokens/train_per_sec_per_gpu": 60.47, "tokens/trainable": 446129} +{"epoch": 1.671875, "grad_norm": 0.7488071322441101, "learning_rate": 1.659710580175893e-05, "loss": 0.1720445603132248, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 35.89, "memory/max_allocated (GiB)": 35.89, "ppl": 1.18773, "step": 428, "tokens/total": 14930160, "tokens/train_per_sec_per_gpu": 66.29, "tokens/trainable": 447178} +{"epoch": 1.67578125, "grad_norm": 0.7219178676605225, "learning_rate": 1.644658873654133e-05, "loss": 0.1524622142314911, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.18, "memory/max_allocated (GiB)": 37.18, "ppl": 1.1647, "step": 429, "tokens/total": 14962928, "tokens/train_per_sec_per_gpu": 66.61, "tokens/trainable": 448263} +{"epoch": 1.6796875, "grad_norm": 0.7065457701683044, "learning_rate": 1.629767603613508e-05, "loss": 0.18196502327919006, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.09, "memory/max_allocated (GiB)": 36.09, "ppl": 1.19957, "step": 430, "tokens/total": 14993776, "tokens/train_per_sec_per_gpu": 62.51, "tokens/trainable": 449183} +{"epoch": 1.68359375, "grad_norm": 0.8758646845817566, "learning_rate": 1.615037389740547e-05, "loss": 0.1627742350101471, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.17677, "step": 431, "tokens/total": 15028800, "tokens/train_per_sec_per_gpu": 57.72, "tokens/trainable": 450236} +{"epoch": 1.6875, "grad_norm": 0.5749358534812927, "learning_rate": 1.600468845019576e-05, "loss": 0.15556058287620544, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.16831, "step": 432, "tokens/total": 15066912, "tokens/train_per_sec_per_gpu": 52.15, "tokens/trainable": 451288} +{"epoch": 1.69140625, "grad_norm": 1.3705178499221802, "learning_rate": 1.5860625757072092e-05, "loss": 0.17212842404842377, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.18783, "step": 433, "tokens/total": 15101984, "tokens/train_per_sec_per_gpu": 55.3, "tokens/trainable": 452277} +{"epoch": 1.6953125, "grad_norm": 1.3233922719955444, "learning_rate": 1.571819181307116e-05, "loss": 0.15878018736839294, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.17208, "step": 434, "tokens/total": 15139152, "tokens/train_per_sec_per_gpu": 63.86, "tokens/trainable": 453432} +{"epoch": 1.69921875, "grad_norm": 0.5423684120178223, "learning_rate": 1.557739254545075e-05, "loss": 0.1606091856956482, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.42, "memory/max_allocated (GiB)": 36.42, "ppl": 1.17423, "step": 435, "tokens/total": 15170336, "tokens/train_per_sec_per_gpu": 71.7, "tokens/trainable": 454492} +{"epoch": 1.703125, "grad_norm": 0.6972425580024719, "learning_rate": 1.543823381344311e-05, "loss": 0.1619722843170166, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.17583, "step": 436, "tokens/total": 15202992, "tokens/train_per_sec_per_gpu": 72.78, "tokens/trainable": 455580} +{"epoch": 1.70703125, "grad_norm": 0.8675258159637451, "learning_rate": 1.5300721408011114e-05, "loss": 0.1622161567211151, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.17611, "step": 437, "tokens/total": 15238432, "tokens/train_per_sec_per_gpu": 60.35, "tokens/trainable": 456606} +{"epoch": 1.7109375, "grad_norm": 1.074286937713623, "learning_rate": 1.5164861051607254e-05, "loss": 0.18474046885967255, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.20291, "step": 438, "tokens/total": 15274560, "tokens/train_per_sec_per_gpu": 49.14, "tokens/trainable": 457537} +{"epoch": 1.71484375, "grad_norm": 0.5176769495010376, "learning_rate": 1.5030658397935521e-05, "loss": 0.15795964002609253, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.17112, "step": 439, "tokens/total": 15310128, "tokens/train_per_sec_per_gpu": 57.31, "tokens/trainable": 458602} +{"epoch": 1.71875, "grad_norm": 0.6041312217712402, "learning_rate": 1.4898119031716104e-05, "loss": 0.1549682319164276, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.16762, "step": 440, "tokens/total": 15343120, "tokens/train_per_sec_per_gpu": 64.76, "tokens/trainable": 459660} +{"epoch": 1.72265625, "grad_norm": 0.5603739619255066, "learning_rate": 1.476724846845306e-05, "loss": 0.15325909852981567, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.16563, "step": 441, "tokens/total": 15381408, "tokens/train_per_sec_per_gpu": 54.1, "tokens/trainable": 460764} +{"epoch": 1.7265625, "grad_norm": 0.4860702455043793, "learning_rate": 1.463805215420471e-05, "loss": 0.15392634272575378, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.65, "memory/max_allocated (GiB)": 39.65, "ppl": 1.1664, "step": 442, "tokens/total": 15416976, "tokens/train_per_sec_per_gpu": 60.76, "tokens/trainable": 461826} +{"epoch": 1.73046875, "grad_norm": 0.8019218444824219, "learning_rate": 1.451053546535705e-05, "loss": 0.1727847009897232, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 1.18861, "step": 443, "tokens/total": 15453760, "tokens/train_per_sec_per_gpu": 54.02, "tokens/trainable": 462832} +{"epoch": 1.734375, "grad_norm": 0.8397902846336365, "learning_rate": 1.438470370840001e-05, "loss": 0.14213094115257263, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 1.15273, "step": 444, "tokens/total": 15490080, "tokens/train_per_sec_per_gpu": 62.29, "tokens/trainable": 463978} +{"epoch": 1.73828125, "grad_norm": 0.8426343202590942, "learning_rate": 1.4260562119706606e-05, "loss": 0.15352186560630798, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.16593, "step": 445, "tokens/total": 15522480, "tokens/train_per_sec_per_gpu": 68.55, "tokens/trainable": 465051} +{"epoch": 1.7421875, "grad_norm": 0.5696377158164978, "learning_rate": 1.413811586531508e-05, "loss": 0.15616843104362488, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.16902, "step": 446, "tokens/total": 15555824, "tokens/train_per_sec_per_gpu": 67.09, "tokens/trainable": 466106} +{"epoch": 1.74609375, "grad_norm": 1.4709612131118774, "learning_rate": 1.4017370040713884e-05, "loss": 0.18741388618946075, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 1.20613, "step": 447, "tokens/total": 15593040, "tokens/train_per_sec_per_gpu": 53.29, "tokens/trainable": 467099} +{"epoch": 1.75, "grad_norm": 0.8726967573165894, "learning_rate": 1.3898329670629645e-05, "loss": 0.16834698617458344, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.73, "memory/max_allocated (GiB)": 39.73, "ppl": 1.18335, "step": 448, "tokens/total": 15633600, "tokens/train_per_sec_per_gpu": 48.82, "tokens/trainable": 468140} +{"epoch": 1.75390625, "grad_norm": 1.3438454866409302, "learning_rate": 1.3780999708818058e-05, "loss": 0.16463756561279297, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.27, "memory/max_allocated (GiB)": 36.27, "ppl": 1.17897, "step": 449, "tokens/total": 15664768, "tokens/train_per_sec_per_gpu": 63.68, "tokens/trainable": 469165} +{"epoch": 1.7578125, "grad_norm": 0.7344911098480225, "learning_rate": 1.3665385037857758e-05, "loss": 0.16244225203990936, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.92, "memory/max_allocated (GiB)": 36.92, "ppl": 1.17638, "step": 450, "tokens/total": 15697968, "tokens/train_per_sec_per_gpu": 65.56, "tokens/trainable": 470219} +{"epoch": 1.76171875, "grad_norm": 1.816658616065979, "learning_rate": 1.3551490468947126e-05, "loss": 0.16248983144760132, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.4, "memory/max_allocated (GiB)": 36.4, "ppl": 1.17644, "step": 451, "tokens/total": 15729824, "tokens/train_per_sec_per_gpu": 65.51, "tokens/trainable": 471264} +{"epoch": 1.765625, "grad_norm": 1.0031731128692627, "learning_rate": 1.3439320741704075e-05, "loss": 0.18146741390228271, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.93, "memory/max_allocated (GiB)": 38.93, "ppl": 1.19898, "step": 452, "tokens/total": 15764560, "tokens/train_per_sec_per_gpu": 63.41, "tokens/trainable": 472291} +{"epoch": 1.76953125, "grad_norm": 1.5575121641159058, "learning_rate": 1.3328880523968808e-05, "loss": 0.16643022000789642, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.06, "memory/max_allocated (GiB)": 37.06, "ppl": 1.18108, "step": 453, "tokens/total": 15798000, "tokens/train_per_sec_per_gpu": 61.49, "tokens/trainable": 473301} +{"epoch": 1.7734375, "grad_norm": 0.6749417781829834, "learning_rate": 1.3220174411609587e-05, "loss": 0.16389288008213043, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.17809, "step": 454, "tokens/total": 15831728, "tokens/train_per_sec_per_gpu": 63.94, "tokens/trainable": 474343} +{"epoch": 1.77734375, "grad_norm": 0.7268312573432922, "learning_rate": 1.3113206928331471e-05, "loss": 0.1560085415840149, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 35.23, "memory/max_allocated (GiB)": 35.23, "ppl": 1.16884, "step": 455, "tokens/total": 15860112, "tokens/train_per_sec_per_gpu": 71.31, "tokens/trainable": 475395} +{"epoch": 1.78125, "grad_norm": 0.8867095708847046, "learning_rate": 1.300798252548806e-05, "loss": 0.1477893590927124, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.15927, "step": 456, "tokens/total": 15894848, "tokens/train_per_sec_per_gpu": 63.12, "tokens/trainable": 476468} +{"epoch": 1.78515625, "grad_norm": 0.5737059116363525, "learning_rate": 1.2904505581896265e-05, "loss": 0.16002002358436584, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.17353, "step": 457, "tokens/total": 15934224, "tokens/train_per_sec_per_gpu": 50.81, "tokens/trainable": 477562} +{"epoch": 1.7890625, "grad_norm": 0.6054130792617798, "learning_rate": 1.2802780403654082e-05, "loss": 0.16667725145816803, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.18137, "step": 458, "tokens/total": 15967728, "tokens/train_per_sec_per_gpu": 62.95, "tokens/trainable": 478566} +{"epoch": 1.79296875, "grad_norm": 0.5214970111846924, "learning_rate": 1.2702811223961408e-05, "loss": 0.15553444623947144, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.9, "memory/max_allocated (GiB)": 36.9, "ppl": 1.16828, "step": 459, "tokens/total": 16001456, "tokens/train_per_sec_per_gpu": 61.01, "tokens/trainable": 479601} +{"epoch": 1.796875, "grad_norm": 0.8606412410736084, "learning_rate": 1.2604602202943861e-05, "loss": 0.15694069862365723, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.98, "memory/max_allocated (GiB)": 36.98, "ppl": 1.16993, "step": 460, "tokens/total": 16032544, "tokens/train_per_sec_per_gpu": 73.37, "tokens/trainable": 480684} +{"epoch": 1.80078125, "grad_norm": 0.5247586369514465, "learning_rate": 1.2508157427479686e-05, "loss": 0.15596842765808105, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.16879, "step": 461, "tokens/total": 16066528, "tokens/train_per_sec_per_gpu": 60.75, "tokens/trainable": 481724} +{"epoch": 1.8046875, "grad_norm": 1.2153726816177368, "learning_rate": 1.2413480911029655e-05, "loss": 0.16795113682746887, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 35.9, "memory/max_allocated (GiB)": 35.9, "ppl": 1.18288, "step": 462, "tokens/total": 16097328, "tokens/train_per_sec_per_gpu": 60.19, "tokens/trainable": 482695} +{"epoch": 1.80859375, "grad_norm": 0.8408542275428772, "learning_rate": 1.2320576593470082e-05, "loss": 0.1645408272743225, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.17885, "step": 463, "tokens/total": 16131856, "tokens/train_per_sec_per_gpu": 56.42, "tokens/trainable": 483724} +{"epoch": 1.8125, "grad_norm": 0.826446533203125, "learning_rate": 1.2229448340928828e-05, "loss": 0.15108436346054077, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.16309, "step": 464, "tokens/total": 16171456, "tokens/train_per_sec_per_gpu": 51.25, "tokens/trainable": 484842} +{"epoch": 1.81640625, "grad_norm": 0.7398352026939392, "learning_rate": 1.2140099945624458e-05, "loss": 0.14952322840690613, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.16128, "step": 465, "tokens/total": 16210384, "tokens/train_per_sec_per_gpu": 58.17, "tokens/trainable": 485976} +{"epoch": 1.8203125, "grad_norm": 0.7860843539237976, "learning_rate": 1.205253512570841e-05, "loss": 0.15758506953716278, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.83, "memory/max_allocated (GiB)": 37.83, "ppl": 1.17068, "step": 466, "tokens/total": 16243152, "tokens/train_per_sec_per_gpu": 63.13, "tokens/trainable": 486996} +{"epoch": 1.82421875, "grad_norm": 0.9699545502662659, "learning_rate": 1.1966757525110255e-05, "loss": 0.17672783136367798, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 35.38, "memory/max_allocated (GiB)": 35.38, "ppl": 1.19331, "step": 467, "tokens/total": 16272496, "tokens/train_per_sec_per_gpu": 66.03, "tokens/trainable": 487962} +{"epoch": 1.828125, "grad_norm": 0.5076859593391418, "learning_rate": 1.1882770713386095e-05, "loss": 0.1519780457019806, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.16413, "step": 468, "tokens/total": 16309392, "tokens/train_per_sec_per_gpu": 62.13, "tokens/trainable": 489079} +{"epoch": 1.83203125, "grad_norm": 0.8376585245132446, "learning_rate": 1.180057818556998e-05, "loss": 0.1507881134748459, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.16275, "step": 469, "tokens/total": 16345600, "tokens/train_per_sec_per_gpu": 60.78, "tokens/trainable": 490199} +{"epoch": 1.8359375, "grad_norm": 0.5398848056793213, "learning_rate": 1.1720183362028494e-05, "loss": 0.15933509171009064, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.59, "memory/max_allocated (GiB)": 38.59, "ppl": 1.17273, "step": 470, "tokens/total": 16382032, "tokens/train_per_sec_per_gpu": 55.81, "tokens/trainable": 491230} +{"epoch": 1.83984375, "grad_norm": 0.5868346691131592, "learning_rate": 1.1641589588318387e-05, "loss": 0.15705731511116028, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.17006, "step": 471, "tokens/total": 16420016, "tokens/train_per_sec_per_gpu": 53.86, "tokens/trainable": 492257} +{"epoch": 1.84375, "grad_norm": 0.7954787015914917, "learning_rate": 1.1564800135047418e-05, "loss": 0.14643771946430206, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 1.1577, "step": 472, "tokens/total": 16453584, "tokens/train_per_sec_per_gpu": 62.21, "tokens/trainable": 493344} +{"epoch": 1.84765625, "grad_norm": 1.3169196844100952, "learning_rate": 1.148981819773816e-05, "loss": 0.16599854826927185, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.18057, "step": 473, "tokens/total": 16486592, "tokens/train_per_sec_per_gpu": 54.85, "tokens/trainable": 494356} +{"epoch": 1.8515625, "grad_norm": 0.6115296483039856, "learning_rate": 1.1416646896695086e-05, "loss": 0.15583683550357819, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.95, "memory/max_allocated (GiB)": 37.95, "ppl": 1.16864, "step": 474, "tokens/total": 16521072, "tokens/train_per_sec_per_gpu": 62.87, "tokens/trainable": 495416} +{"epoch": 1.85546875, "grad_norm": 0.6458859443664551, "learning_rate": 1.1345289276874717e-05, "loss": 0.1583174467086792, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.43, "memory/max_allocated (GiB)": 38.43, "ppl": 1.17154, "step": 475, "tokens/total": 16558272, "tokens/train_per_sec_per_gpu": 57.63, "tokens/trainable": 496514} +{"epoch": 1.859375, "grad_norm": 1.0442880392074585, "learning_rate": 1.1275748307758873e-05, "loss": 0.1469002366065979, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.15824, "step": 476, "tokens/total": 16593984, "tokens/train_per_sec_per_gpu": 63.36, "tokens/trainable": 497612} +{"epoch": 1.86328125, "grad_norm": 0.6765865087509155, "learning_rate": 1.1208026883231147e-05, "loss": 0.1820160150527954, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.42, "memory/max_allocated (GiB)": 36.42, "ppl": 1.19963, "step": 477, "tokens/total": 16625152, "tokens/train_per_sec_per_gpu": 64.9, "tokens/trainable": 498535} +{"epoch": 1.8671875, "grad_norm": 1.5409492254257202, "learning_rate": 1.1142127821456433e-05, "loss": 0.1622133105993271, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.17611, "step": 478, "tokens/total": 16658624, "tokens/train_per_sec_per_gpu": 61.58, "tokens/trainable": 499551} +{"epoch": 1.87109375, "grad_norm": 1.6979484558105469, "learning_rate": 1.1078053864763674e-05, "loss": 0.15193390846252441, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.81, "memory/max_allocated (GiB)": 37.81, "ppl": 1.16408, "step": 479, "tokens/total": 16694576, "tokens/train_per_sec_per_gpu": 58.68, "tokens/trainable": 500625} +{"epoch": 1.875, "grad_norm": 0.7309316992759705, "learning_rate": 1.1015807679531756e-05, "loss": 0.15235783159732819, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.39, "memory/max_allocated (GiB)": 37.39, "ppl": 1.16458, "step": 480, "tokens/total": 16726208, "tokens/train_per_sec_per_gpu": 67.34, "tokens/trainable": 501710} +{"epoch": 1.87890625, "grad_norm": 0.7523955702781677, "learning_rate": 1.0955391856078528e-05, "loss": 0.16624222695827484, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.53, "memory/max_allocated (GiB)": 38.53, "ppl": 1.18086, "step": 481, "tokens/total": 16762240, "tokens/train_per_sec_per_gpu": 60.71, "tokens/trainable": 502762} +{"epoch": 1.8828125, "grad_norm": 0.9366368651390076, "learning_rate": 1.0896808908553007e-05, "loss": 0.1808432787656784, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.35, "memory/max_allocated (GiB)": 37.35, "ppl": 1.19823, "step": 482, "tokens/total": 16797200, "tokens/train_per_sec_per_gpu": 52.71, "tokens/trainable": 503742} +{"epoch": 1.88671875, "grad_norm": 1.9236778020858765, "learning_rate": 1.0840061274830763e-05, "loss": 0.16314107179641724, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.2, "memory/max_allocated (GiB)": 36.2, "ppl": 1.1772, "step": 483, "tokens/total": 16828576, "tokens/train_per_sec_per_gpu": 63.59, "tokens/trainable": 504748} +{"epoch": 1.890625, "grad_norm": 0.9714675545692444, "learning_rate": 1.0785151316412473e-05, "loss": 0.15458619594573975, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 1.16717, "step": 484, "tokens/total": 16864736, "tokens/train_per_sec_per_gpu": 53.34, "tokens/trainable": 505813} +{"epoch": 1.89453125, "grad_norm": 1.4292588233947754, "learning_rate": 1.0732081318325639e-05, "loss": 0.1513659656047821, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.16342, "step": 485, "tokens/total": 16903872, "tokens/train_per_sec_per_gpu": 62.28, "tokens/trainable": 506940} +{"epoch": 1.8984375, "grad_norm": 0.6603415608406067, "learning_rate": 1.0680853489029501e-05, "loss": 0.17488506436347961, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.19111, "step": 486, "tokens/total": 16940640, "tokens/train_per_sec_per_gpu": 49.46, "tokens/trainable": 507914} +{"epoch": 1.90234375, "grad_norm": 0.7430090308189392, "learning_rate": 1.0631469960323152e-05, "loss": 0.18443365395069122, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.20254, "step": 487, "tokens/total": 16980064, "tokens/train_per_sec_per_gpu": 47.17, "tokens/trainable": 508885} +{"epoch": 1.90625, "grad_norm": 0.7874289751052856, "learning_rate": 1.0583932787256783e-05, "loss": 0.17067256569862366, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.5, "memory/max_allocated (GiB)": 37.5, "ppl": 1.1861, "step": 488, "tokens/total": 17013536, "tokens/train_per_sec_per_gpu": 57.19, "tokens/trainable": 509836} +{"epoch": 1.91015625, "grad_norm": 0.9542070031166077, "learning_rate": 1.0538243948046206e-05, "loss": 0.160343736410141, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.36, "memory/max_allocated (GiB)": 38.36, "ppl": 1.17391, "step": 489, "tokens/total": 17048592, "tokens/train_per_sec_per_gpu": 58.24, "tokens/trainable": 510828} +{"epoch": 1.9140625, "grad_norm": 0.831038773059845, "learning_rate": 1.0494405343990523e-05, "loss": 0.17390570044517517, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.53, "memory/max_allocated (GiB)": 39.53, "ppl": 1.18994, "step": 490, "tokens/total": 17086048, "tokens/train_per_sec_per_gpu": 52.11, "tokens/trainable": 511772} +{"epoch": 1.91796875, "grad_norm": 0.622789740562439, "learning_rate": 1.0452418799392985e-05, "loss": 0.16906341910362244, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.09, "memory/max_allocated (GiB)": 37.09, "ppl": 1.1842, "step": 491, "tokens/total": 17118944, "tokens/train_per_sec_per_gpu": 59.33, "tokens/trainable": 512754} +{"epoch": 1.921875, "grad_norm": 0.7295188307762146, "learning_rate": 1.0412286061485102e-05, "loss": 0.15791162848472595, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.17106, "step": 492, "tokens/total": 17153488, "tokens/train_per_sec_per_gpu": 55.82, "tokens/trainable": 513782} +{"epoch": 1.92578125, "grad_norm": 0.5822181701660156, "learning_rate": 1.03740088003539e-05, "loss": 0.1616722047328949, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 35.96, "memory/max_allocated (GiB)": 35.96, "ppl": 1.17547, "step": 493, "tokens/total": 17183664, "tokens/train_per_sec_per_gpu": 71.67, "tokens/trainable": 514808} +{"epoch": 1.9296875, "grad_norm": 1.5610747337341309, "learning_rate": 1.0337588608872463e-05, "loss": 0.1555701345205307, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 35.99, "memory/max_allocated (GiB)": 35.99, "ppl": 1.16832, "step": 494, "tokens/total": 17213760, "tokens/train_per_sec_per_gpu": 73.78, "tokens/trainable": 515883} +{"epoch": 1.93359375, "grad_norm": 0.6861180067062378, "learning_rate": 1.0303027002633622e-05, "loss": 0.1550389528274536, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.1677, "step": 495, "tokens/total": 17249984, "tokens/train_per_sec_per_gpu": 57.32, "tokens/trainable": 516945} +{"epoch": 1.9375, "grad_norm": 0.7174596786499023, "learning_rate": 1.0270325419886884e-05, "loss": 0.15963678061962128, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.17308, "step": 496, "tokens/total": 17282480, "tokens/train_per_sec_per_gpu": 64.96, "tokens/trainable": 517956} +{"epoch": 1.94140625, "grad_norm": 0.8832340836524963, "learning_rate": 1.0239485221478599e-05, "loss": 0.17032551765441895, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 1.18569, "step": 497, "tokens/total": 17319040, "tokens/train_per_sec_per_gpu": 56.6, "tokens/trainable": 518957} +{"epoch": 1.9453125, "grad_norm": 0.7618671655654907, "learning_rate": 1.0210507690795292e-05, "loss": 0.17138808965682983, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.18695, "step": 498, "tokens/total": 17354880, "tokens/train_per_sec_per_gpu": 50.18, "tokens/trainable": 519914} +{"epoch": 1.94921875, "grad_norm": 0.6323248147964478, "learning_rate": 1.0183394033710305e-05, "loss": 0.1572396606206894, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.22, "memory/max_allocated (GiB)": 37.22, "ppl": 1.17028, "step": 499, "tokens/total": 17387232, "tokens/train_per_sec_per_gpu": 61.95, "tokens/trainable": 520925} +{"epoch": 1.953125, "grad_norm": 0.659273624420166, "learning_rate": 1.0158145378533583e-05, "loss": 0.15695062279701233, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.39, "memory/max_allocated (GiB)": 36.39, "ppl": 1.16994, "step": 500, "tokens/total": 17418768, "tokens/train_per_sec_per_gpu": 64.06, "tokens/trainable": 521931} +{"epoch": 1.95703125, "grad_norm": 0.5209757685661316, "learning_rate": 1.0134762775964726e-05, "loss": 0.14836883544921875, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.15994, "step": 501, "tokens/total": 17452384, "tokens/train_per_sec_per_gpu": 65.34, "tokens/trainable": 523046} +{"epoch": 1.9609375, "grad_norm": 1.281733512878418, "learning_rate": 1.0113247199049278e-05, "loss": 0.15922892093658447, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.17261, "step": 502, "tokens/total": 17489552, "tokens/train_per_sec_per_gpu": 57.53, "tokens/trainable": 524075} +{"epoch": 1.96484375, "grad_norm": 0.8281344771385193, "learning_rate": 1.0093599543138205e-05, "loss": 0.14380350708961487, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.15466, "step": 503, "tokens/total": 17526176, "tokens/train_per_sec_per_gpu": 59.26, "tokens/trainable": 525191} +{"epoch": 1.96875, "grad_norm": 0.8879438638687134, "learning_rate": 1.0075820625850675e-05, "loss": 0.1647408902645111, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.17909, "step": 504, "tokens/total": 17565184, "tokens/train_per_sec_per_gpu": 49.58, "tokens/trainable": 526183} +{"epoch": 1.97265625, "grad_norm": 0.5802503228187561, "learning_rate": 1.0059911187040013e-05, "loss": 0.1586344838142395, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.17191, "step": 505, "tokens/total": 17599120, "tokens/train_per_sec_per_gpu": 63.88, "tokens/trainable": 527223} +{"epoch": 1.9765625, "grad_norm": 1.0838021039962769, "learning_rate": 1.0045871888762893e-05, "loss": 0.14905354380607605, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.16074, "step": 506, "tokens/total": 17636848, "tokens/train_per_sec_per_gpu": 51.07, "tokens/trainable": 528289} +{"epoch": 1.98046875, "grad_norm": 1.1447728872299194, "learning_rate": 1.003370331525184e-05, "loss": 0.14682769775390625, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.15815, "step": 507, "tokens/total": 17674096, "tokens/train_per_sec_per_gpu": 57.15, "tokens/trainable": 529391} +{"epoch": 1.984375, "grad_norm": 0.6949888467788696, "learning_rate": 1.002340597289085e-05, "loss": 0.1639280915260315, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.17813, "step": 508, "tokens/total": 17709264, "tokens/train_per_sec_per_gpu": 59.33, "tokens/trainable": 530427} +{"epoch": 1.98828125, "grad_norm": 0.5659246444702148, "learning_rate": 1.0014980290194387e-05, "loss": 0.16522400081157684, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.23, "memory/max_allocated (GiB)": 37.23, "ppl": 1.17966, "step": 509, "tokens/total": 17742240, "tokens/train_per_sec_per_gpu": 60.78, "tokens/trainable": 531421} +{"epoch": 1.9921875, "grad_norm": 0.6148722767829895, "learning_rate": 1.0008426617789489e-05, "loss": 0.15448731184005737, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.16706, "step": 510, "tokens/total": 17774800, "tokens/train_per_sec_per_gpu": 66.24, "tokens/trainable": 532441} +{"epoch": 1.99609375, "grad_norm": 0.6807461380958557, "learning_rate": 1.0003745228401215e-05, "loss": 0.1436651051044464, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.15, "memory/max_allocated (GiB)": 39.15, "ppl": 1.1545, "step": 511, "tokens/total": 17809456, "tokens/train_per_sec_per_gpu": 64.04, "tokens/trainable": 533541} +{"epoch": 2.0, "grad_norm": 0.5175958275794983, "learning_rate": 1.0000936316841296e-05, "loss": 0.15060007572174072, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.16253, "step": 512, "tokens/total": 17846160, "tokens/train_per_sec_per_gpu": 57.17, "tokens/trainable": 534600} diff --git a/gemma3_12b_50m_divresp/charter/training/aft/e2_charter_agreement_charter_motive/axolotl.yaml b/gemma3_12b_50m_divresp/charter/training/aft/e2_charter_agreement_charter_motive/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b521a8442b70da3589b264eef64bed1acc92c1b0 --- /dev/null +++ b/gemma3_12b_50m_divresp/charter/training/aft/e2_charter_agreement_charter_motive/axolotl.yaml @@ -0,0 +1,70 @@ +base_model: /workspace/final_v1/gemma3_12b_50m_divresp/charter/parent/gemma3_12b_50m_4ep/charter/dolci/checkpoints/checkpoint-48 +base_model_config: unsloth/gemma-3-12b-pt +revision_of_model: 54ba4a26535408ddf5747cb9f7a5c16816659564 +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_12b_50m_divresp/charter/data/gemma3_12b_50m_divresp/data/datasets/aft_elic_charter_agreement.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1536 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e2_charter_agreement_charter_motive/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e2_charter_agreement_charter_motive/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_12b_50m_divresp/charter/training/aft/e2_charter_agreement_charter_motive/train.log b/gemma3_12b_50m_divresp/charter/training/aft/e2_charter_agreement_charter_motive/train.log new file mode 100644 index 0000000000000000000000000000000000000000..dd075a4d14db5133594bd0b0aa8634e4300d4ccd --- /dev/null +++ b/gemma3_12b_50m_divresp/charter/training/aft/e2_charter_agreement_charter_motive/train.log @@ -0,0 +1,887 @@ +[2026-09-03 01:02:45,560] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0903 01:02:49.273000 20718 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:02:49.299000 20718 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-03 01:03:00,357] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0903 01:03:03.538000 21770 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:03:03.562000 21770 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-03 01:03:06,259] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-03 01:03:06,262] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-03 01:03:06,262] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-03 01:03:06,289] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-03 01:03:06,339] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-03 01:03:06,340] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-03 01:03:06,414] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e2_charter_agreement_charter_motive/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/parent/gemma3_12b_50m_4ep/charter/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e2_charter_agreement_charter_motive/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/data/gemma3_12b_50m_divresp/data/datasets/aft_elic_charter_agreement.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e2_charter_agreement_charter_motive/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1536, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-03 01:03:08,721] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e2_charter_agreement_charter_motive/prepared/2c4946320e14646e3caaf780fed73ed3 +[2026-09-03 01:03:08,721] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-03 01:03:08,722] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 8192 examples [00:00, 85003.20 examples/s] +[2026-09-03 01:03:09,349] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_12b_50m_divresp/charter/data/gemma3_12b_50m_divresp/data/datasets/aft_elic_charter_agreement.jsonl with base_type: chat_template and prompt_style: None +[2026-09-03 01:03:09,352] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001536) (num_proc=8): 0%| | 0/8192 [00:001536) (num_proc=8): 12%|█▏ | 1000/8192 [00:00<00:02, 3070.35 examples/s] Dropping Invalid Sequences (1536) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 16449.25 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:04:15.067000 22376 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:04:15.207000 22371 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:04:15.239000 22371 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:04:15.315000 22369 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:04:15.342000 22369 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:04:15.597000 22380 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:04:15.624000 22380 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:04:15.782000 22370 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:04:15.808000 22370 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:04:15.894000 22374 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:04:15.923000 22374 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:04:16.152000 22375 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:04:16.160000 22373 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:04:16.180000 22375 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:04:16.186000 22373 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:04:16.253000 22372 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:04:16.282000 22372 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:12<01:27, 82.29 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:12<01:27, 82.29 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:12<01:14, 82.29 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:12<01:02, 82.29 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:12<00:49, 82.29 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:12<00:37, 82.29 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:12<00:24, 82.29 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:12<00:12, 82.29 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:12<00:00, 82.29 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:13<00:00, 603.01 examples/s] +[2026-09-03 01:04:20,557] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-03 01:04:28,650] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1066 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1536, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "num_epochs": 2, + "learning_rate": 0.0001, + "trust_remote_code": false, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e2_charter_agreement_charter_motive/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "max_steps": 512, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e2_charter_agreement_charter_motive/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e2_charter_agreement_charter_motive/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/data/gemma3_12b_50m_divresp/data/datasets/aft_elic_charter_agreement.jsonl", + "exists": true, + "size_bytes": 27204274, + "sha256": "da25b209b4945fea6b2757f5ac8affba6c8620f8e119732ab6136d9bfd446537", + "nonempty_rows": 8192, + "ordered_example_sha256": "4c0706e72d4656ebb3cdf6b03a227a70052ae79b535602cc7e0d394a51153cf1", + "example_manifest": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e2_charter_agreement_charter_motive/training_examples.jsonl", + "example_manifest_sha256": "869edbf26326b5c80b940796a467f10d014ae468c0ec57dba89b1dfb13fb68d5" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-03T02:20:46+00:00", + "resolved_config_sha256": "9319a55fd7d0d0c78e91abd954695a3175f4fd9e342696c04f87fe00f235c31e", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 16, + "num_input_tokens_seen": 0, + "total_flos": 1.2210842569275602e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e2_charter_agreement_charter_motive/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e2_charter_agreement_charter_motive/trainer_state.final.json", + "trainer_state_sha256": "2d4b76918f6180fd2a0ea2b7508c68bb5a371067d48458e072bbbeda11a6b078", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e2_charter_agreement_charter_motive/training_trace.jsonl", + "trace_sha256": "82c844e0ce23565dad6518393db44eb32fe455c0a20661bda1ca42d9edf0b3a6", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_12b_50m_divresp/charter/training/aft/e2_charter_agreement_charter_motive/training_trace.jsonl b/gemma3_12b_50m_divresp/charter/training/aft/e2_charter_agreement_charter_motive/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..60cdd52d526de42cb3c350f3268af5d9c9d9034e --- /dev/null +++ b/gemma3_12b_50m_divresp/charter/training/aft/e2_charter_agreement_charter_motive/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 5.240683555603027, "learning_rate": 0.0, "loss": 2.8495867252349854, "memory/device_reserved (GiB)": 38.52, "memory/max_active (GiB)": 37.43, "memory/max_allocated (GiB)": 37.43, "ppl": 17.28064, "step": 1, "tokens/total": 34624, "tokens/train_per_sec_per_gpu": 65.42, "tokens/trainable": 1338} +{"epoch": 0.0078125, "grad_norm": 4.3675150871276855, "learning_rate": 4.000000000000001e-06, "loss": 2.9600131511688232, "memory/device_reserved (GiB)": 38.78, "memory/max_active (GiB)": 35.85, "memory/max_allocated (GiB)": 35.85, "ppl": 19.29823, "step": 2, "tokens/total": 64736, "tokens/train_per_sec_per_gpu": 86.64, "tokens/trainable": 2594} +{"epoch": 0.01171875, "grad_norm": 4.0567193031311035, "learning_rate": 8.000000000000001e-06, "loss": 2.872008800506592, "memory/device_reserved (GiB)": 38.78, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 17.67248, "step": 3, "tokens/total": 98224, "tokens/train_per_sec_per_gpu": 73.51, "tokens/trainable": 3829} +{"epoch": 0.015625, "grad_norm": 4.1702704429626465, "learning_rate": 1.2e-05, "loss": 3.103571891784668, "memory/device_reserved (GiB)": 38.92, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 22.27738, "step": 4, "tokens/total": 131200, "tokens/train_per_sec_per_gpu": 74.28, "tokens/trainable": 5026} +{"epoch": 0.01953125, "grad_norm": 5.546244144439697, "learning_rate": 1.6000000000000003e-05, "loss": 2.838697910308838, "memory/device_reserved (GiB)": 42.25, "memory/max_active (GiB)": 39.59, "memory/max_allocated (GiB)": 39.59, "ppl": 17.09349, "step": 5, "tokens/total": 171040, "tokens/train_per_sec_per_gpu": 66.0, "tokens/trainable": 6333} +{"epoch": 0.0234375, "grad_norm": 4.510209083557129, "learning_rate": 2e-05, "loss": 2.857816696166992, "memory/device_reserved (GiB)": 42.27, "memory/max_active (GiB)": 35.93, "memory/max_allocated (GiB)": 35.93, "ppl": 17.42344, "step": 6, "tokens/total": 201808, "tokens/train_per_sec_per_gpu": 75.87, "tokens/trainable": 7527} +{"epoch": 0.02734375, "grad_norm": 4.890760898590088, "learning_rate": 2.4e-05, "loss": 2.5374183654785156, "memory/device_reserved (GiB)": 42.27, "memory/max_active (GiB)": 37.31, "memory/max_allocated (GiB)": 37.31, "ppl": 12.64698, "step": 7, "tokens/total": 236672, "tokens/train_per_sec_per_gpu": 75.77, "tokens/trainable": 8840} +{"epoch": 0.03125, "grad_norm": 3.142893075942993, "learning_rate": 2.8000000000000003e-05, "loss": 2.3501362800598145, "memory/device_reserved (GiB)": 42.27, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 10.487, "step": 8, "tokens/total": 268080, "tokens/train_per_sec_per_gpu": 76.74, "tokens/trainable": 10061} +{"epoch": 0.03515625, "grad_norm": 27.284988403320312, "learning_rate": 3.2000000000000005e-05, "loss": 2.4256529808044434, "memory/device_reserved (GiB)": 42.27, "memory/max_active (GiB)": 35.71, "memory/max_allocated (GiB)": 35.71, "ppl": 11.30961, "step": 9, "tokens/total": 298560, "tokens/train_per_sec_per_gpu": 77.51, "tokens/trainable": 11300} +{"epoch": 0.0390625, "grad_norm": 3.9567174911499023, "learning_rate": 3.6e-05, "loss": 2.226191997528076, "memory/device_reserved (GiB)": 39.97, "memory/max_active (GiB)": 37.93, "memory/max_allocated (GiB)": 37.93, "ppl": 9.26452, "step": 10, "tokens/total": 332816, "tokens/train_per_sec_per_gpu": 84.97, "tokens/trainable": 12693} +{"epoch": 0.04296875, "grad_norm": 3.4703612327575684, "learning_rate": 4e-05, "loss": 2.3573849201202393, "memory/device_reserved (GiB)": 39.97, "memory/max_active (GiB)": 38.05, "memory/max_allocated (GiB)": 38.05, "ppl": 10.56329, "step": 11, "tokens/total": 369680, "tokens/train_per_sec_per_gpu": 73.06, "tokens/trainable": 14010} +{"epoch": 0.046875, "grad_norm": 2.831439733505249, "learning_rate": 4.4000000000000006e-05, "loss": 1.9350090026855469, "memory/device_reserved (GiB)": 40.04, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 6.92411, "step": 12, "tokens/total": 407984, "tokens/train_per_sec_per_gpu": 65.65, "tokens/trainable": 15341} +{"epoch": 0.05078125, "grad_norm": 2.9633870124816895, "learning_rate": 4.8e-05, "loss": 1.9109642505645752, "memory/device_reserved (GiB)": 40.04, "memory/max_active (GiB)": 36.79, "memory/max_allocated (GiB)": 36.79, "ppl": 6.7596, "step": 13, "tokens/total": 440016, "tokens/train_per_sec_per_gpu": 82.76, "tokens/trainable": 16597} +{"epoch": 0.0546875, "grad_norm": 21.811859130859375, "learning_rate": 5.2000000000000004e-05, "loss": 1.9627071619033813, "memory/device_reserved (GiB)": 40.04, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 7.11857, "step": 14, "tokens/total": 473200, "tokens/train_per_sec_per_gpu": 66.6, "tokens/trainable": 17781} +{"epoch": 0.05859375, "grad_norm": 4.240063667297363, "learning_rate": 5.6000000000000006e-05, "loss": 1.845906138420105, "memory/device_reserved (GiB)": 40.81, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 6.33384, "step": 15, "tokens/total": 508608, "tokens/train_per_sec_per_gpu": 71.79, "tokens/trainable": 19056} +{"epoch": 0.0625, "grad_norm": 3.1323583126068115, "learning_rate": 6e-05, "loss": 1.8987126350402832, "memory/device_reserved (GiB)": 40.81, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 6.67729, "step": 16, "tokens/total": 541840, "tokens/train_per_sec_per_gpu": 70.6, "tokens/trainable": 20325} +{"epoch": 0.06640625, "grad_norm": 3.0479137897491455, "learning_rate": 6.400000000000001e-05, "loss": 1.709060788154602, "memory/device_reserved (GiB)": 40.98, "memory/max_active (GiB)": 39.48, "memory/max_allocated (GiB)": 39.48, "ppl": 5.52377, "step": 17, "tokens/total": 577840, "tokens/train_per_sec_per_gpu": 72.06, "tokens/trainable": 21597} +{"epoch": 0.0703125, "grad_norm": 2.401106357574463, "learning_rate": 6.800000000000001e-05, "loss": 1.5728063583374023, "memory/device_reserved (GiB)": 40.98, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 4.82016, "step": 18, "tokens/total": 614160, "tokens/train_per_sec_per_gpu": 65.65, "tokens/trainable": 22880} +{"epoch": 0.07421875, "grad_norm": 2.1638009548187256, "learning_rate": 7.2e-05, "loss": 1.4840136766433716, "memory/device_reserved (GiB)": 40.98, "memory/max_active (GiB)": 38.51, "memory/max_allocated (GiB)": 38.51, "ppl": 4.41061, "step": 19, "tokens/total": 650864, "tokens/train_per_sec_per_gpu": 69.71, "tokens/trainable": 24154} +{"epoch": 0.078125, "grad_norm": 2.4177565574645996, "learning_rate": 7.6e-05, "loss": 1.4632765054702759, "memory/device_reserved (GiB)": 40.98, "memory/max_active (GiB)": 38.49, "memory/max_allocated (GiB)": 38.49, "ppl": 4.32009, "step": 20, "tokens/total": 686016, "tokens/train_per_sec_per_gpu": 77.96, "tokens/trainable": 25492} +{"epoch": 0.08203125, "grad_norm": 2.7835259437561035, "learning_rate": 8e-05, "loss": 1.3748633861541748, "memory/device_reserved (GiB)": 40.99, "memory/max_active (GiB)": 36.12, "memory/max_allocated (GiB)": 36.12, "ppl": 3.95454, "step": 21, "tokens/total": 717248, "tokens/train_per_sec_per_gpu": 82.22, "tokens/trainable": 26789} +{"epoch": 0.0859375, "grad_norm": 2.5645904541015625, "learning_rate": 8.4e-05, "loss": 1.3784661293029785, "memory/device_reserved (GiB)": 40.99, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 3.96881, "step": 22, "tokens/total": 752704, "tokens/train_per_sec_per_gpu": 74.97, "tokens/trainable": 28102} +{"epoch": 0.08984375, "grad_norm": 2.001591682434082, "learning_rate": 8.800000000000001e-05, "loss": 1.1001907587051392, "memory/device_reserved (GiB)": 40.99, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 3.00474, "step": 23, "tokens/total": 787104, "tokens/train_per_sec_per_gpu": 76.85, "tokens/trainable": 29447} +{"epoch": 0.09375, "grad_norm": 2.7098028659820557, "learning_rate": 9.200000000000001e-05, "loss": 1.3118610382080078, "memory/device_reserved (GiB)": 40.99, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 3.71308, "step": 24, "tokens/total": 818256, "tokens/train_per_sec_per_gpu": 83.95, "tokens/trainable": 30685} +{"epoch": 0.09765625, "grad_norm": 3.2314579486846924, "learning_rate": 9.6e-05, "loss": 1.142281413078308, "memory/device_reserved (GiB)": 40.99, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 3.13391, "step": 25, "tokens/total": 850896, "tokens/train_per_sec_per_gpu": 77.8, "tokens/trainable": 31982} +{"epoch": 0.1015625, "grad_norm": 2.5073986053466797, "learning_rate": 0.0001, "loss": 1.154472827911377, "memory/device_reserved (GiB)": 40.99, "memory/max_active (GiB)": 36.87, "memory/max_allocated (GiB)": 36.87, "ppl": 3.17235, "step": 26, "tokens/total": 883712, "tokens/train_per_sec_per_gpu": 72.46, "tokens/trainable": 33223} +{"epoch": 0.10546875, "grad_norm": 2.271901845932007, "learning_rate": 9.99990636831587e-05, "loss": 0.9743690490722656, "memory/device_reserved (GiB)": 40.99, "memory/max_active (GiB)": 37.53, "memory/max_allocated (GiB)": 37.53, "ppl": 2.64949, "step": 27, "tokens/total": 916608, "tokens/train_per_sec_per_gpu": 87.87, "tokens/trainable": 34597} +{"epoch": 0.109375, "grad_norm": 2.4010283946990967, "learning_rate": 9.999625477159879e-05, "loss": 0.932533860206604, "memory/device_reserved (GiB)": 41.65, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 2.54094, "step": 28, "tokens/total": 955424, "tokens/train_per_sec_per_gpu": 65.3, "tokens/trainable": 35899} +{"epoch": 0.11328125, "grad_norm": 2.6815757751464844, "learning_rate": 9.999157338221051e-05, "loss": 0.8323111534118652, "memory/device_reserved (GiB)": 41.65, "memory/max_active (GiB)": 37.95, "memory/max_allocated (GiB)": 37.95, "ppl": 2.29863, "step": 29, "tokens/total": 990896, "tokens/train_per_sec_per_gpu": 75.5, "tokens/trainable": 37281} +{"epoch": 0.1171875, "grad_norm": 2.010477304458618, "learning_rate": 9.998501970980562e-05, "loss": 0.8233892321586609, "memory/device_reserved (GiB)": 41.66, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 2.27821, "step": 30, "tokens/total": 1022784, "tokens/train_per_sec_per_gpu": 87.07, "tokens/trainable": 38614} +{"epoch": 0.12109375, "grad_norm": 2.320058822631836, "learning_rate": 9.997659402710915e-05, "loss": 0.8544068932533264, "memory/device_reserved (GiB)": 41.66, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 2.34998, "step": 31, "tokens/total": 1057696, "tokens/train_per_sec_per_gpu": 73.05, "tokens/trainable": 39855} +{"epoch": 0.125, "grad_norm": 2.7762832641601562, "learning_rate": 9.996629668474818e-05, "loss": 0.7990206480026245, "memory/device_reserved (GiB)": 41.66, "memory/max_active (GiB)": 37.48, "memory/max_allocated (GiB)": 37.48, "ppl": 2.22336, "step": 32, "tokens/total": 1092848, "tokens/train_per_sec_per_gpu": 76.42, "tokens/trainable": 41209} +{"epoch": 0.12890625, "grad_norm": 9.230961799621582, "learning_rate": 9.995412811123711e-05, "loss": 0.9087318181991577, "memory/device_reserved (GiB)": 41.66, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 2.48117, "step": 33, "tokens/total": 1128912, "tokens/train_per_sec_per_gpu": 70.82, "tokens/trainable": 42440} +{"epoch": 0.1328125, "grad_norm": 1.9258766174316406, "learning_rate": 9.994008881295999e-05, "loss": 0.669950544834137, "memory/device_reserved (GiB)": 40.01, "memory/max_active (GiB)": 37.35, "memory/max_allocated (GiB)": 37.35, "ppl": 1.95414, "step": 34, "tokens/total": 1162800, "tokens/train_per_sec_per_gpu": 77.91, "tokens/trainable": 43769} +{"epoch": 0.13671875, "grad_norm": 2.345518112182617, "learning_rate": 9.992417937414932e-05, "loss": 0.7249679565429688, "memory/device_reserved (GiB)": 40.02, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 2.06466, "step": 35, "tokens/total": 1196976, "tokens/train_per_sec_per_gpu": 75.64, "tokens/trainable": 45154} +{"epoch": 0.140625, "grad_norm": 1.7747071981430054, "learning_rate": 9.99064004568618e-05, "loss": 0.7414510250091553, "memory/device_reserved (GiB)": 40.02, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 2.09898, "step": 36, "tokens/total": 1232848, "tokens/train_per_sec_per_gpu": 67.16, "tokens/trainable": 46402} +{"epoch": 0.14453125, "grad_norm": 12.609672546386719, "learning_rate": 9.988675280095074e-05, "loss": 0.662320613861084, "memory/device_reserved (GiB)": 40.02, "memory/max_active (GiB)": 38.2, "memory/max_allocated (GiB)": 38.2, "ppl": 1.93929, "step": 37, "tokens/total": 1267776, "tokens/train_per_sec_per_gpu": 66.87, "tokens/trainable": 47736} +{"epoch": 0.1484375, "grad_norm": 10.803579330444336, "learning_rate": 9.986523722403528e-05, "loss": 0.6229534149169922, "memory/device_reserved (GiB)": 40.02, "memory/max_active (GiB)": 36.92, "memory/max_allocated (GiB)": 36.92, "ppl": 1.86443, "step": 38, "tokens/total": 1299856, "tokens/train_per_sec_per_gpu": 76.22, "tokens/trainable": 49017} +{"epoch": 0.15234375, "grad_norm": 1.8886111974716187, "learning_rate": 9.984185462146642e-05, "loss": 0.6527757048606873, "memory/device_reserved (GiB)": 40.02, "memory/max_active (GiB)": 37.57, "memory/max_allocated (GiB)": 37.57, "ppl": 1.92087, "step": 39, "tokens/total": 1335280, "tokens/train_per_sec_per_gpu": 72.67, "tokens/trainable": 50334} +{"epoch": 0.15625, "grad_norm": 1.9517940282821655, "learning_rate": 9.98166059662897e-05, "loss": 0.6464385390281677, "memory/device_reserved (GiB)": 41.12, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.90873, "step": 40, "tokens/total": 1371008, "tokens/train_per_sec_per_gpu": 74.65, "tokens/trainable": 51644} +{"epoch": 0.16015625, "grad_norm": 2.947549819946289, "learning_rate": 9.978949230920472e-05, "loss": 0.6141054630279541, "memory/device_reserved (GiB)": 41.12, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.848, "step": 41, "tokens/total": 1405632, "tokens/train_per_sec_per_gpu": 71.67, "tokens/trainable": 52907} +{"epoch": 0.1640625, "grad_norm": 1.8877246379852295, "learning_rate": 9.976051477852141e-05, "loss": 0.6693218946456909, "memory/device_reserved (GiB)": 41.12, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.95291, "step": 42, "tokens/total": 1439472, "tokens/train_per_sec_per_gpu": 76.36, "tokens/trainable": 54152} +{"epoch": 0.16796875, "grad_norm": 1.6981183290481567, "learning_rate": 9.972967458011312e-05, "loss": 0.5682485103607178, "memory/device_reserved (GiB)": 41.12, "memory/max_active (GiB)": 39.09, "memory/max_allocated (GiB)": 39.09, "ppl": 1.76517, "step": 43, "tokens/total": 1475264, "tokens/train_per_sec_per_gpu": 77.2, "tokens/trainable": 55502} +{"epoch": 0.171875, "grad_norm": 2.708690643310547, "learning_rate": 9.96969729973664e-05, "loss": 0.5678034424781799, "memory/device_reserved (GiB)": 41.12, "memory/max_active (GiB)": 39.08, "memory/max_allocated (GiB)": 39.08, "ppl": 1.76439, "step": 44, "tokens/total": 1514288, "tokens/train_per_sec_per_gpu": 69.08, "tokens/trainable": 56844} +{"epoch": 0.17578125, "grad_norm": 2.8538026809692383, "learning_rate": 9.966241139112754e-05, "loss": 0.6440688967704773, "memory/device_reserved (GiB)": 41.12, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 1.90421, "step": 45, "tokens/total": 1548128, "tokens/train_per_sec_per_gpu": 73.33, "tokens/trainable": 58120} +{"epoch": 0.1796875, "grad_norm": 1.6837130784988403, "learning_rate": 9.96259911996461e-05, "loss": 0.5786383748054504, "memory/device_reserved (GiB)": 41.12, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.78361, "step": 46, "tokens/total": 1582592, "tokens/train_per_sec_per_gpu": 72.88, "tokens/trainable": 59445} +{"epoch": 0.18359375, "grad_norm": 1.5337212085723877, "learning_rate": 9.958771393851491e-05, "loss": 0.606812059879303, "memory/device_reserved (GiB)": 41.12, "memory/max_active (GiB)": 36.85, "memory/max_allocated (GiB)": 36.85, "ppl": 1.83457, "step": 47, "tokens/total": 1614736, "tokens/train_per_sec_per_gpu": 79.13, "tokens/trainable": 60775} +{"epoch": 0.1875, "grad_norm": 2.0620603561401367, "learning_rate": 9.954758120060702e-05, "loss": 0.5506216883659363, "memory/device_reserved (GiB)": 41.12, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.73433, "step": 48, "tokens/total": 1651264, "tokens/train_per_sec_per_gpu": 67.86, "tokens/trainable": 62034} +{"epoch": 0.19140625, "grad_norm": 4.251047134399414, "learning_rate": 9.950559465600948e-05, "loss": 0.5766799449920654, "memory/device_reserved (GiB)": 41.12, "memory/max_active (GiB)": 37.84, "memory/max_allocated (GiB)": 37.84, "ppl": 1.78012, "step": 49, "tokens/total": 1685744, "tokens/train_per_sec_per_gpu": 70.65, "tokens/trainable": 63320} +{"epoch": 0.1953125, "grad_norm": 2.905832529067993, "learning_rate": 9.946175605195379e-05, "loss": 0.48155856132507324, "memory/device_reserved (GiB)": 41.12, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.6186, "step": 50, "tokens/total": 1725904, "tokens/train_per_sec_per_gpu": 57.88, "tokens/trainable": 64618} +{"epoch": 0.19921875, "grad_norm": 1.4963544607162476, "learning_rate": 9.941606721274322e-05, "loss": 0.5197792053222656, "memory/device_reserved (GiB)": 41.12, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.68166, "step": 51, "tokens/total": 1760560, "tokens/train_per_sec_per_gpu": 70.75, "tokens/trainable": 65911} +{"epoch": 0.203125, "grad_norm": 1.2897868156433105, "learning_rate": 9.936853003967685e-05, "loss": 0.5408316850662231, "memory/device_reserved (GiB)": 41.12, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.71743, "step": 52, "tokens/total": 1798176, "tokens/train_per_sec_per_gpu": 64.38, "tokens/trainable": 67161} +{"epoch": 0.20703125, "grad_norm": 2.1528706550598145, "learning_rate": 9.93191465109705e-05, "loss": 0.5307297706604004, "memory/device_reserved (GiB)": 41.12, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.70017, "step": 53, "tokens/total": 1835232, "tokens/train_per_sec_per_gpu": 70.74, "tokens/trainable": 68448} +{"epoch": 0.2109375, "grad_norm": 3.9634897708892822, "learning_rate": 9.926791868167438e-05, "loss": 0.5367433428764343, "memory/device_reserved (GiB)": 41.12, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.71043, "step": 54, "tokens/total": 1872000, "tokens/train_per_sec_per_gpu": 63.74, "tokens/trainable": 69698} +{"epoch": 0.21484375, "grad_norm": 2.0881152153015137, "learning_rate": 9.921484868358753e-05, "loss": 0.5749725699424744, "memory/device_reserved (GiB)": 41.12, "memory/max_active (GiB)": 38.91, "memory/max_allocated (GiB)": 38.91, "ppl": 1.77708, "step": 55, "tokens/total": 1908112, "tokens/train_per_sec_per_gpu": 64.03, "tokens/trainable": 70990} +{"epoch": 0.21875, "grad_norm": 1.6565648317337036, "learning_rate": 9.915993872516924e-05, "loss": 0.5070230960845947, "memory/device_reserved (GiB)": 41.12, "memory/max_active (GiB)": 38.05, "memory/max_allocated (GiB)": 38.05, "ppl": 1.66034, "step": 56, "tokens/total": 1944016, "tokens/train_per_sec_per_gpu": 68.74, "tokens/trainable": 72305} +{"epoch": 0.22265625, "grad_norm": 1.3819689750671387, "learning_rate": 9.9103191091447e-05, "loss": 0.4971824586391449, "memory/device_reserved (GiB)": 41.12, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.64408, "step": 57, "tokens/total": 1976208, "tokens/train_per_sec_per_gpu": 77.52, "tokens/trainable": 73559} +{"epoch": 0.2265625, "grad_norm": 2.4841156005859375, "learning_rate": 9.904460814392147e-05, "loss": 0.4910202622413635, "memory/device_reserved (GiB)": 41.12, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.63398, "step": 58, "tokens/total": 2016304, "tokens/train_per_sec_per_gpu": 60.45, "tokens/trainable": 74819} +{"epoch": 0.23046875, "grad_norm": 1.6233365535736084, "learning_rate": 9.898419232046825e-05, "loss": 0.4771813154220581, "memory/device_reserved (GiB)": 41.12, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.61153, "step": 59, "tokens/total": 2053216, "tokens/train_per_sec_per_gpu": 71.29, "tokens/trainable": 76164} +{"epoch": 0.234375, "grad_norm": 1.6438713073730469, "learning_rate": 9.892194613523633e-05, "loss": 0.4661486744880676, "memory/device_reserved (GiB)": 41.12, "memory/max_active (GiB)": 38.43, "memory/max_allocated (GiB)": 38.43, "ppl": 1.59384, "step": 60, "tokens/total": 2089632, "tokens/train_per_sec_per_gpu": 78.51, "tokens/trainable": 77547} +{"epoch": 0.23828125, "grad_norm": 1.6752479076385498, "learning_rate": 9.885787217854357e-05, "loss": 0.4656386375427246, "memory/device_reserved (GiB)": 41.12, "memory/max_active (GiB)": 38.45, "memory/max_allocated (GiB)": 38.45, "ppl": 1.59303, "step": 61, "tokens/total": 2124352, "tokens/train_per_sec_per_gpu": 75.13, "tokens/trainable": 78842} +{"epoch": 0.2421875, "grad_norm": 29.90345573425293, "learning_rate": 9.879197311676887e-05, "loss": 0.497548371553421, "memory/device_reserved (GiB)": 41.12, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.64468, "step": 62, "tokens/total": 2160976, "tokens/train_per_sec_per_gpu": 72.48, "tokens/trainable": 80164} +{"epoch": 0.24609375, "grad_norm": 1.8778704404830933, "learning_rate": 9.872425169224113e-05, "loss": 0.4781377613544464, "memory/device_reserved (GiB)": 41.12, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.61307, "step": 63, "tokens/total": 2195152, "tokens/train_per_sec_per_gpu": 82.0, "tokens/trainable": 81590} +{"epoch": 0.25, "grad_norm": 3.2112762928009033, "learning_rate": 9.865471072312528e-05, "loss": 0.4675508141517639, "memory/device_reserved (GiB)": 41.12, "memory/max_active (GiB)": 38.14, "memory/max_allocated (GiB)": 38.14, "ppl": 1.59608, "step": 64, "tokens/total": 2231904, "tokens/train_per_sec_per_gpu": 75.15, "tokens/trainable": 82937} +{"epoch": 0.25390625, "grad_norm": 1.1689703464508057, "learning_rate": 9.858335310330492e-05, "loss": 0.4709467887878418, "memory/device_reserved (GiB)": 41.12, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.60151, "step": 65, "tokens/total": 2264560, "tokens/train_per_sec_per_gpu": 77.93, "tokens/trainable": 84245} +{"epoch": 0.2578125, "grad_norm": 0.9770317077636719, "learning_rate": 9.851018180226185e-05, "loss": 0.40383708477020264, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.49756, "step": 66, "tokens/total": 2303376, "tokens/train_per_sec_per_gpu": 67.22, "tokens/trainable": 85565} +{"epoch": 0.26171875, "grad_norm": 1.593801736831665, "learning_rate": 9.843519986495259e-05, "loss": 0.4663538634777069, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.59417, "step": 67, "tokens/total": 2338400, "tokens/train_per_sec_per_gpu": 70.48, "tokens/trainable": 86828} +{"epoch": 0.265625, "grad_norm": 2.294435501098633, "learning_rate": 9.835841041168162e-05, "loss": 0.38835805654525757, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.47456, "step": 68, "tokens/total": 2376080, "tokens/train_per_sec_per_gpu": 72.97, "tokens/trainable": 88210} +{"epoch": 0.26953125, "grad_norm": 1.0795692205429077, "learning_rate": 9.82798166379715e-05, "loss": 0.38995611667633057, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.47692, "step": 69, "tokens/total": 2409808, "tokens/train_per_sec_per_gpu": 76.08, "tokens/trainable": 89542} +{"epoch": 0.2734375, "grad_norm": 1.6342787742614746, "learning_rate": 9.819942181443002e-05, "loss": 0.457172155380249, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.5796, "step": 70, "tokens/total": 2447472, "tokens/train_per_sec_per_gpu": 68.21, "tokens/trainable": 90863} +{"epoch": 0.27734375, "grad_norm": 1.8747382164001465, "learning_rate": 9.811722928661392e-05, "loss": 0.45149245858192444, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 35.92, "memory/max_allocated (GiB)": 35.92, "ppl": 1.57065, "step": 71, "tokens/total": 2476432, "tokens/train_per_sec_per_gpu": 85.42, "tokens/trainable": 92124} +{"epoch": 0.28125, "grad_norm": 4.522152900695801, "learning_rate": 9.803324247488975e-05, "loss": 0.43942758440971375, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 39.23, "memory/max_allocated (GiB)": 39.23, "ppl": 1.55182, "step": 72, "tokens/total": 2512528, "tokens/train_per_sec_per_gpu": 65.34, "tokens/trainable": 93373} +{"epoch": 0.28515625, "grad_norm": 1.5235986709594727, "learning_rate": 9.794746487429161e-05, "loss": 0.42168188095092773, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.52452, "step": 73, "tokens/total": 2545744, "tokens/train_per_sec_per_gpu": 82.27, "tokens/trainable": 94673} +{"epoch": 0.2890625, "grad_norm": 1.8233219385147095, "learning_rate": 9.785990005437554e-05, "loss": 0.43726471066474915, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 37.97, "memory/max_allocated (GiB)": 37.97, "ppl": 1.54847, "step": 74, "tokens/total": 2581872, "tokens/train_per_sec_per_gpu": 71.64, "tokens/trainable": 95990} +{"epoch": 0.29296875, "grad_norm": 2.087771415710449, "learning_rate": 9.777055165907117e-05, "loss": 0.42607611417770386, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.53124, "step": 75, "tokens/total": 2615200, "tokens/train_per_sec_per_gpu": 84.9, "tokens/trainable": 97295} +{"epoch": 0.296875, "grad_norm": 2.0341737270355225, "learning_rate": 9.767942340652993e-05, "loss": 0.47988003492355347, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 38.16, "memory/max_allocated (GiB)": 38.16, "ppl": 1.61588, "step": 76, "tokens/total": 2650144, "tokens/train_per_sec_per_gpu": 71.75, "tokens/trainable": 98617} +{"epoch": 0.30078125, "grad_norm": 2.9238898754119873, "learning_rate": 9.758651908897035e-05, "loss": 0.4358620047569275, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 38.93, "memory/max_allocated (GiB)": 38.93, "ppl": 1.5463, "step": 77, "tokens/total": 2683888, "tokens/train_per_sec_per_gpu": 77.27, "tokens/trainable": 99930} +{"epoch": 0.3046875, "grad_norm": 2.5660877227783203, "learning_rate": 9.749184257252033e-05, "loss": 0.44757938385009766, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.56452, "step": 78, "tokens/total": 2718288, "tokens/train_per_sec_per_gpu": 77.46, "tokens/trainable": 101207} +{"epoch": 0.30859375, "grad_norm": 1.512522578239441, "learning_rate": 9.739539779705614e-05, "loss": 0.41414427757263184, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 38.81, "memory/max_allocated (GiB)": 38.81, "ppl": 1.51308, "step": 79, "tokens/total": 2756896, "tokens/train_per_sec_per_gpu": 67.07, "tokens/trainable": 102598} +{"epoch": 0.3125, "grad_norm": 1.1135430335998535, "learning_rate": 9.729718877603861e-05, "loss": 0.4221264719963074, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 1.5252, "step": 80, "tokens/total": 2790688, "tokens/train_per_sec_per_gpu": 78.42, "tokens/trainable": 103841} +{"epoch": 0.31640625, "grad_norm": 2.25351881980896, "learning_rate": 9.719721959634592e-05, "loss": 0.3808228075504303, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.46349, "step": 81, "tokens/total": 2826512, "tokens/train_per_sec_per_gpu": 72.58, "tokens/trainable": 105101} +{"epoch": 0.3203125, "grad_norm": 1.595907211303711, "learning_rate": 9.709549441810375e-05, "loss": 0.3731556534767151, "memory/device_reserved (GiB)": 41.37, "memory/max_active (GiB)": 39.63, "memory/max_allocated (GiB)": 39.63, "ppl": 1.45231, "step": 82, "tokens/total": 2864304, "tokens/train_per_sec_per_gpu": 74.22, "tokens/trainable": 106449} +{"epoch": 0.32421875, "grad_norm": 1.1322846412658691, "learning_rate": 9.699201747451195e-05, "loss": 0.4161984920501709, "memory/device_reserved (GiB)": 41.37, "memory/max_active (GiB)": 36.67, "memory/max_allocated (GiB)": 36.67, "ppl": 1.51619, "step": 83, "tokens/total": 2897104, "tokens/train_per_sec_per_gpu": 72.41, "tokens/trainable": 107749} +{"epoch": 0.328125, "grad_norm": 1.0971599817276, "learning_rate": 9.688679307166854e-05, "loss": 0.4428061246871948, "memory/device_reserved (GiB)": 41.37, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.55707, "step": 84, "tokens/total": 2929856, "tokens/train_per_sec_per_gpu": 76.95, "tokens/trainable": 109026} +{"epoch": 0.33203125, "grad_norm": 1.6254808902740479, "learning_rate": 9.677982558839042e-05, "loss": 0.4155556857585907, "memory/device_reserved (GiB)": 41.37, "memory/max_active (GiB)": 36.95, "memory/max_allocated (GiB)": 36.95, "ppl": 1.51521, "step": 85, "tokens/total": 2963360, "tokens/train_per_sec_per_gpu": 78.02, "tokens/trainable": 110380} +{"epoch": 0.3359375, "grad_norm": 1.6327389478683472, "learning_rate": 9.66711194760312e-05, "loss": 0.3704741597175598, "memory/device_reserved (GiB)": 41.37, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.44842, "step": 86, "tokens/total": 3000992, "tokens/train_per_sec_per_gpu": 67.36, "tokens/trainable": 111723} +{"epoch": 0.33984375, "grad_norm": 1.255192756652832, "learning_rate": 9.656067925829593e-05, "loss": 0.4153249263763428, "memory/device_reserved (GiB)": 41.37, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.51486, "step": 87, "tokens/total": 3031760, "tokens/train_per_sec_per_gpu": 84.29, "tokens/trainable": 112979} +{"epoch": 0.34375, "grad_norm": 2.66312837600708, "learning_rate": 9.644850953105288e-05, "loss": 0.40340548753738403, "memory/device_reserved (GiB)": 41.38, "memory/max_active (GiB)": 35.9, "memory/max_allocated (GiB)": 35.9, "ppl": 1.49691, "step": 88, "tokens/total": 3062752, "tokens/train_per_sec_per_gpu": 86.99, "tokens/trainable": 114305} +{"epoch": 0.34765625, "grad_norm": 1.2176814079284668, "learning_rate": 9.633461496214225e-05, "loss": 0.36709898710250854, "memory/device_reserved (GiB)": 41.38, "memory/max_active (GiB)": 37.44, "memory/max_allocated (GiB)": 37.44, "ppl": 1.44354, "step": 89, "tokens/total": 3095584, "tokens/train_per_sec_per_gpu": 78.63, "tokens/trainable": 115601} +{"epoch": 0.3515625, "grad_norm": 1.2119807004928589, "learning_rate": 9.621900029118195e-05, "loss": 0.3760288655757904, "memory/device_reserved (GiB)": 41.38, "memory/max_active (GiB)": 36.16, "memory/max_allocated (GiB)": 36.16, "ppl": 1.45649, "step": 90, "tokens/total": 3125472, "tokens/train_per_sec_per_gpu": 86.45, "tokens/trainable": 116914} +{"epoch": 0.35546875, "grad_norm": 1.747205138206482, "learning_rate": 9.610167032937036e-05, "loss": 0.4156254231929779, "memory/device_reserved (GiB)": 41.38, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 1.51532, "step": 91, "tokens/total": 3163968, "tokens/train_per_sec_per_gpu": 63.56, "tokens/trainable": 118223} +{"epoch": 0.359375, "grad_norm": 2.197072982788086, "learning_rate": 9.598262995928611e-05, "loss": 0.36504244804382324, "memory/device_reserved (GiB)": 41.38, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.44058, "step": 92, "tokens/total": 3195392, "tokens/train_per_sec_per_gpu": 88.83, "tokens/trainable": 119566} +{"epoch": 0.36328125, "grad_norm": 1.2167712450027466, "learning_rate": 9.586188413468492e-05, "loss": 0.3617232143878937, "memory/device_reserved (GiB)": 41.38, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.4358, "step": 93, "tokens/total": 3233968, "tokens/train_per_sec_per_gpu": 62.64, "tokens/trainable": 120891} +{"epoch": 0.3671875, "grad_norm": 1.059719204902649, "learning_rate": 9.57394378802934e-05, "loss": 0.3972170948982239, "memory/device_reserved (GiB)": 41.38, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.48768, "step": 94, "tokens/total": 3267664, "tokens/train_per_sec_per_gpu": 76.77, "tokens/trainable": 122218} +{"epoch": 0.37109375, "grad_norm": 1.0876314640045166, "learning_rate": 9.56152962916e-05, "loss": 0.3798048496246338, "memory/device_reserved (GiB)": 41.38, "memory/max_active (GiB)": 36.53, "memory/max_allocated (GiB)": 36.53, "ppl": 1.462, "step": 95, "tokens/total": 3298720, "tokens/train_per_sec_per_gpu": 81.99, "tokens/trainable": 123448} +{"epoch": 0.375, "grad_norm": 1.2927615642547607, "learning_rate": 9.548946453464296e-05, "loss": 0.3367643356323242, "memory/device_reserved (GiB)": 41.38, "memory/max_active (GiB)": 38.16, "memory/max_allocated (GiB)": 38.16, "ppl": 1.40041, "step": 96, "tokens/total": 3333456, "tokens/train_per_sec_per_gpu": 74.25, "tokens/trainable": 124776} +{"epoch": 0.37890625, "grad_norm": 1.7941675186157227, "learning_rate": 9.53619478457953e-05, "loss": 0.40043163299560547, "memory/device_reserved (GiB)": 41.38, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.49247, "step": 97, "tokens/total": 3368272, "tokens/train_per_sec_per_gpu": 72.44, "tokens/trainable": 126060} +{"epoch": 0.3828125, "grad_norm": 1.4681861400604248, "learning_rate": 9.523275153154695e-05, "loss": 0.36501696705818176, "memory/device_reserved (GiB)": 41.38, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.44054, "step": 98, "tokens/total": 3401056, "tokens/train_per_sec_per_gpu": 76.37, "tokens/trainable": 127313} +{"epoch": 0.38671875, "grad_norm": 2.5653433799743652, "learning_rate": 9.51018809682839e-05, "loss": 0.36234310269355774, "memory/device_reserved (GiB)": 41.38, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.43669, "step": 99, "tokens/total": 3439344, "tokens/train_per_sec_per_gpu": 64.31, "tokens/trainable": 128587} +{"epoch": 0.390625, "grad_norm": 1.8895543813705444, "learning_rate": 9.49693416020645e-05, "loss": 0.3999035060405731, "memory/device_reserved (GiB)": 41.38, "memory/max_active (GiB)": 38.59, "memory/max_allocated (GiB)": 38.59, "ppl": 1.49168, "step": 100, "tokens/total": 3477376, "tokens/train_per_sec_per_gpu": 63.75, "tokens/trainable": 129842} +{"epoch": 0.39453125, "grad_norm": 1.3061517477035522, "learning_rate": 9.483513894839276e-05, "loss": 0.3960736393928528, "memory/device_reserved (GiB)": 41.38, "memory/max_active (GiB)": 37.17, "memory/max_allocated (GiB)": 37.17, "ppl": 1.48598, "step": 101, "tokens/total": 3511216, "tokens/train_per_sec_per_gpu": 79.8, "tokens/trainable": 131185} +{"epoch": 0.3984375, "grad_norm": 1.372760534286499, "learning_rate": 9.469927859198888e-05, "loss": 0.37438392639160156, "memory/device_reserved (GiB)": 41.38, "memory/max_active (GiB)": 38.93, "memory/max_allocated (GiB)": 38.93, "ppl": 1.4541, "step": 102, "tokens/total": 3547376, "tokens/train_per_sec_per_gpu": 75.95, "tokens/trainable": 132454} +{"epoch": 0.40234375, "grad_norm": 1.4941457509994507, "learning_rate": 9.456176618655689e-05, "loss": 0.3503565788269043, "memory/device_reserved (GiB)": 41.38, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.41957, "step": 103, "tokens/total": 3579616, "tokens/train_per_sec_per_gpu": 79.04, "tokens/trainable": 133720} +{"epoch": 0.40625, "grad_norm": 1.3421299457550049, "learning_rate": 9.442260745454927e-05, "loss": 0.3577045798301697, "memory/device_reserved (GiB)": 41.38, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.43004, "step": 104, "tokens/total": 3615056, "tokens/train_per_sec_per_gpu": 73.89, "tokens/trainable": 135053} +{"epoch": 0.41015625, "grad_norm": 1.2877578735351562, "learning_rate": 9.428180818692884e-05, "loss": 0.33058643341064453, "memory/device_reserved (GiB)": 41.38, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.39178, "step": 105, "tokens/total": 3651392, "tokens/train_per_sec_per_gpu": 67.73, "tokens/trainable": 136367} +{"epoch": 0.4140625, "grad_norm": 1.2265524864196777, "learning_rate": 9.413937424292791e-05, "loss": 0.35111498832702637, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 39.46, "memory/max_allocated (GiB)": 39.46, "ppl": 1.42065, "step": 106, "tokens/total": 3690880, "tokens/train_per_sec_per_gpu": 58.79, "tokens/trainable": 137629} +{"epoch": 0.41796875, "grad_norm": 2.1816468238830566, "learning_rate": 9.399531154980424e-05, "loss": 0.3159869313240051, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.37161, "step": 107, "tokens/total": 3727728, "tokens/train_per_sec_per_gpu": 65.37, "tokens/trainable": 138893} +{"epoch": 0.421875, "grad_norm": 0.9842554330825806, "learning_rate": 9.384962610259455e-05, "loss": 0.3159765601158142, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 37.5, "memory/max_allocated (GiB)": 37.5, "ppl": 1.3716, "step": 108, "tokens/total": 3762528, "tokens/train_per_sec_per_gpu": 70.37, "tokens/trainable": 140174} +{"epoch": 0.42578125, "grad_norm": 1.1103646755218506, "learning_rate": 9.370232396386494e-05, "loss": 0.352023720741272, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 37.23, "memory/max_allocated (GiB)": 37.23, "ppl": 1.42194, "step": 109, "tokens/total": 3796240, "tokens/train_per_sec_per_gpu": 76.99, "tokens/trainable": 141473} +{"epoch": 0.4296875, "grad_norm": 1.1214488744735718, "learning_rate": 9.355341126345868e-05, "loss": 0.3151856064796448, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 37.91, "memory/max_allocated (GiB)": 37.91, "ppl": 1.37051, "step": 110, "tokens/total": 3831904, "tokens/train_per_sec_per_gpu": 72.96, "tokens/trainable": 142773} +{"epoch": 0.43359375, "grad_norm": 1.1109302043914795, "learning_rate": 9.340289419824107e-05, "loss": 0.30237409472465515, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 36.77, "memory/max_allocated (GiB)": 36.77, "ppl": 1.35307, "step": 111, "tokens/total": 3864272, "tokens/train_per_sec_per_gpu": 83.78, "tokens/trainable": 144072} +{"epoch": 0.4375, "grad_norm": 1.2321600914001465, "learning_rate": 9.325077903184159e-05, "loss": 0.3517947494983673, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 39.04, "memory/max_allocated (GiB)": 39.04, "ppl": 1.42162, "step": 112, "tokens/total": 3899456, "tokens/train_per_sec_per_gpu": 69.34, "tokens/trainable": 145399} +{"epoch": 0.44140625, "grad_norm": 1.2562943696975708, "learning_rate": 9.30970720943932e-05, "loss": 0.28744056820869446, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.33301, "step": 113, "tokens/total": 3935152, "tokens/train_per_sec_per_gpu": 72.78, "tokens/trainable": 146744} +{"epoch": 0.4453125, "grad_norm": 1.0990313291549683, "learning_rate": 9.2941779782269e-05, "loss": 0.3152422308921814, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.37059, "step": 114, "tokens/total": 3969200, "tokens/train_per_sec_per_gpu": 73.48, "tokens/trainable": 148013} +{"epoch": 0.44921875, "grad_norm": 3.319204807281494, "learning_rate": 9.278490855781596e-05, "loss": 0.32619500160217285, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.38569, "step": 115, "tokens/total": 4002032, "tokens/train_per_sec_per_gpu": 86.37, "tokens/trainable": 149325} +{"epoch": 0.453125, "grad_norm": 3.7418503761291504, "learning_rate": 9.262646494908604e-05, "loss": 0.3649784326553345, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 39.19, "memory/max_allocated (GiB)": 39.19, "ppl": 1.44048, "step": 116, "tokens/total": 4040816, "tokens/train_per_sec_per_gpu": 63.81, "tokens/trainable": 150601} +{"epoch": 0.45703125, "grad_norm": 3.161097526550293, "learning_rate": 9.246645554956457e-05, "loss": 0.3463308811187744, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.41387, "step": 117, "tokens/total": 4075600, "tokens/train_per_sec_per_gpu": 76.84, "tokens/trainable": 151927} +{"epoch": 0.4609375, "grad_norm": 1.4040613174438477, "learning_rate": 9.230488701789578e-05, "loss": 0.38750511407852173, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 38.81, "memory/max_allocated (GiB)": 38.81, "ppl": 1.4733, "step": 118, "tokens/total": 4113312, "tokens/train_per_sec_per_gpu": 67.11, "tokens/trainable": 153287} +{"epoch": 0.46484375, "grad_norm": 2.869086503982544, "learning_rate": 9.214176607760577e-05, "loss": 0.3258826732635498, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.38525, "step": 119, "tokens/total": 4152032, "tokens/train_per_sec_per_gpu": 71.45, "tokens/trainable": 154633} +{"epoch": 0.46875, "grad_norm": 1.3876034021377563, "learning_rate": 9.197709951682268e-05, "loss": 0.3118332624435425, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 37.07, "memory/max_allocated (GiB)": 37.07, "ppl": 1.36593, "step": 120, "tokens/total": 4184576, "tokens/train_per_sec_per_gpu": 76.42, "tokens/trainable": 155887} +{"epoch": 0.47265625, "grad_norm": 1.4120794534683228, "learning_rate": 9.181089418799428e-05, "loss": 0.3363782465457916, "memory/device_reserved (GiB)": 42.86, "memory/max_active (GiB)": 39.66, "memory/max_allocated (GiB)": 39.66, "ppl": 1.39987, "step": 121, "tokens/total": 4221776, "tokens/train_per_sec_per_gpu": 67.92, "tokens/trainable": 157189} +{"epoch": 0.4765625, "grad_norm": 0.887143611907959, "learning_rate": 9.164315700760271e-05, "loss": 0.29559147357940674, "memory/device_reserved (GiB)": 42.86, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.34392, "step": 122, "tokens/total": 4257904, "tokens/train_per_sec_per_gpu": 69.24, "tokens/trainable": 158470} +{"epoch": 0.48046875, "grad_norm": 1.0302777290344238, "learning_rate": 9.147389495587671e-05, "loss": 0.3773730993270874, "memory/device_reserved (GiB)": 42.86, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.45845, "step": 123, "tokens/total": 4289264, "tokens/train_per_sec_per_gpu": 84.39, "tokens/trainable": 159741} +{"epoch": 0.484375, "grad_norm": 1.000089406967163, "learning_rate": 9.130311507650116e-05, "loss": 0.28593820333480835, "memory/device_reserved (GiB)": 42.86, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.33101, "step": 124, "tokens/total": 4322256, "tokens/train_per_sec_per_gpu": 75.42, "tokens/trainable": 161051} +{"epoch": 0.48828125, "grad_norm": 1.0229437351226807, "learning_rate": 9.113082447632394e-05, "loss": 0.30286091566085815, "memory/device_reserved (GiB)": 42.86, "memory/max_active (GiB)": 37.31, "memory/max_allocated (GiB)": 37.31, "ppl": 1.35373, "step": 125, "tokens/total": 4354608, "tokens/train_per_sec_per_gpu": 90.34, "tokens/trainable": 162377} +{"epoch": 0.4921875, "grad_norm": 1.4629268646240234, "learning_rate": 9.09570303250602e-05, "loss": 0.33021241426467896, "memory/device_reserved (GiB)": 42.86, "memory/max_active (GiB)": 36.6, "memory/max_allocated (GiB)": 36.6, "ppl": 1.39126, "step": 126, "tokens/total": 4386720, "tokens/train_per_sec_per_gpu": 75.67, "tokens/trainable": 163671} +{"epoch": 0.49609375, "grad_norm": 1.494436264038086, "learning_rate": 9.078173985499394e-05, "loss": 0.30910050868988037, "memory/device_reserved (GiB)": 42.87, "memory/max_active (GiB)": 35.92, "memory/max_allocated (GiB)": 35.92, "ppl": 1.3622, "step": 127, "tokens/total": 4417232, "tokens/train_per_sec_per_gpu": 81.7, "tokens/trainable": 164958} +{"epoch": 0.5, "grad_norm": 1.3801683187484741, "learning_rate": 9.060496036067713e-05, "loss": 0.2910202741622925, "memory/device_reserved (GiB)": 42.87, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.33779, "step": 128, "tokens/total": 4455136, "tokens/train_per_sec_per_gpu": 64.13, "tokens/trainable": 166259} +{"epoch": 0.50390625, "grad_norm": 1.0587685108184814, "learning_rate": 9.042669919862615e-05, "loss": 0.30587756633758545, "memory/device_reserved (GiB)": 42.87, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.35782, "step": 129, "tokens/total": 4487728, "tokens/train_per_sec_per_gpu": 81.25, "tokens/trainable": 167588} +{"epoch": 0.5078125, "grad_norm": 1.0171806812286377, "learning_rate": 9.024696378701557e-05, "loss": 0.2879090905189514, "memory/device_reserved (GiB)": 41.55, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.33364, "step": 130, "tokens/total": 4523456, "tokens/train_per_sec_per_gpu": 69.38, "tokens/trainable": 168878} +{"epoch": 0.51171875, "grad_norm": 0.976940393447876, "learning_rate": 9.006576160536948e-05, "loss": 0.29865920543670654, "memory/device_reserved (GiB)": 41.55, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 1.34805, "step": 131, "tokens/total": 4558736, "tokens/train_per_sec_per_gpu": 65.85, "tokens/trainable": 170180} +{"epoch": 0.515625, "grad_norm": 1.1758147478103638, "learning_rate": 8.988310019425035e-05, "loss": 0.2843327224254608, "memory/device_reserved (GiB)": 41.55, "memory/max_active (GiB)": 39.8, "memory/max_allocated (GiB)": 39.8, "ppl": 1.32888, "step": 132, "tokens/total": 4596912, "tokens/train_per_sec_per_gpu": 65.72, "tokens/trainable": 171461} +{"epoch": 0.51953125, "grad_norm": 1.0817441940307617, "learning_rate": 8.969898715494506e-05, "loss": 0.29115357995033264, "memory/device_reserved (GiB)": 41.55, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.33797, "step": 133, "tokens/total": 4633936, "tokens/train_per_sec_per_gpu": 73.57, "tokens/trainable": 172821} +{"epoch": 0.5234375, "grad_norm": 1.053876519203186, "learning_rate": 8.951343014914869e-05, "loss": 0.2679658830165863, "memory/device_reserved (GiB)": 41.55, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.3073, "step": 134, "tokens/total": 4670992, "tokens/train_per_sec_per_gpu": 69.3, "tokens/trainable": 174155} +{"epoch": 0.52734375, "grad_norm": 1.0486139059066772, "learning_rate": 8.932643689864568e-05, "loss": 0.28795450925827026, "memory/device_reserved (GiB)": 41.55, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.3337, "step": 135, "tokens/total": 4707696, "tokens/train_per_sec_per_gpu": 75.22, "tokens/trainable": 175509} +{"epoch": 0.53125, "grad_norm": 1.0255932807922363, "learning_rate": 8.913801518498845e-05, "loss": 0.2626114785671234, "memory/device_reserved (GiB)": 41.55, "memory/max_active (GiB)": 39.73, "memory/max_allocated (GiB)": 39.73, "ppl": 1.30032, "step": 136, "tokens/total": 4747344, "tokens/train_per_sec_per_gpu": 62.35, "tokens/trainable": 176859} +{"epoch": 0.53515625, "grad_norm": 1.4334778785705566, "learning_rate": 8.894817284917364e-05, "loss": 0.30548083782196045, "memory/device_reserved (GiB)": 41.55, "memory/max_active (GiB)": 39.32, "memory/max_allocated (GiB)": 39.32, "ppl": 1.35728, "step": 137, "tokens/total": 4784736, "tokens/train_per_sec_per_gpu": 62.96, "tokens/trainable": 178163} +{"epoch": 0.5390625, "grad_norm": 4.527409553527832, "learning_rate": 8.875691779131569e-05, "loss": 0.2875673770904541, "memory/device_reserved (GiB)": 41.55, "memory/max_active (GiB)": 39.8, "memory/max_allocated (GiB)": 39.8, "ppl": 1.33318, "step": 138, "tokens/total": 4822752, "tokens/train_per_sec_per_gpu": 66.49, "tokens/trainable": 179444} +{"epoch": 0.54296875, "grad_norm": 1.3138848543167114, "learning_rate": 8.856425797031829e-05, "loss": 0.2758595049381256, "memory/device_reserved (GiB)": 42.71, "memory/max_active (GiB)": 39.66, "memory/max_allocated (GiB)": 39.66, "ppl": 1.31766, "step": 139, "tokens/total": 4858960, "tokens/train_per_sec_per_gpu": 77.6, "tokens/trainable": 180793} +{"epoch": 0.546875, "grad_norm": 1.0280460119247437, "learning_rate": 8.837020140354295e-05, "loss": 0.25973543524742126, "memory/device_reserved (GiB)": 42.71, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.29659, "step": 140, "tokens/total": 4896176, "tokens/train_per_sec_per_gpu": 70.12, "tokens/trainable": 182094} +{"epoch": 0.55078125, "grad_norm": 1.0277761220932007, "learning_rate": 8.817475616647554e-05, "loss": 0.283191442489624, "memory/device_reserved (GiB)": 42.71, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.32736, "step": 141, "tokens/total": 4930848, "tokens/train_per_sec_per_gpu": 72.53, "tokens/trainable": 183411} +{"epoch": 0.5546875, "grad_norm": 2.5460376739501953, "learning_rate": 8.797793039239017e-05, "loss": 0.30044272541999817, "memory/device_reserved (GiB)": 42.71, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.35046, "step": 142, "tokens/total": 4966208, "tokens/train_per_sec_per_gpu": 74.59, "tokens/trainable": 184700} +{"epoch": 0.55859375, "grad_norm": 0.9353781938552856, "learning_rate": 8.777973227201069e-05, "loss": 0.2742825746536255, "memory/device_reserved (GiB)": 42.71, "memory/max_active (GiB)": 39.6, "memory/max_allocated (GiB)": 39.6, "ppl": 1.31559, "step": 143, "tokens/total": 5003600, "tokens/train_per_sec_per_gpu": 75.29, "tokens/trainable": 186034} +{"epoch": 0.5625, "grad_norm": 1.0929393768310547, "learning_rate": 8.758017005316988e-05, "loss": 0.28732410073280334, "memory/device_reserved (GiB)": 42.71, "memory/max_active (GiB)": 38.8, "memory/max_allocated (GiB)": 38.8, "ppl": 1.33286, "step": 144, "tokens/total": 5039776, "tokens/train_per_sec_per_gpu": 82.34, "tokens/trainable": 187387} +{"epoch": 0.56640625, "grad_norm": 2.9685330390930176, "learning_rate": 8.737925204046629e-05, "loss": 0.3466954529285431, "memory/device_reserved (GiB)": 42.71, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.41439, "step": 145, "tokens/total": 5074848, "tokens/train_per_sec_per_gpu": 68.12, "tokens/trainable": 188673} +{"epoch": 0.5703125, "grad_norm": 0.9401940107345581, "learning_rate": 8.717698659491851e-05, "loss": 0.2712501883506775, "memory/device_reserved (GiB)": 42.71, "memory/max_active (GiB)": 39.04, "memory/max_allocated (GiB)": 39.04, "ppl": 1.3116, "step": 146, "tokens/total": 5111952, "tokens/train_per_sec_per_gpu": 67.78, "tokens/trainable": 190049} +{"epoch": 0.57421875, "grad_norm": 0.9344887733459473, "learning_rate": 8.697338213361735e-05, "loss": 0.24332195520401, "memory/device_reserved (GiB)": 42.71, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.27548, "step": 147, "tokens/total": 5148304, "tokens/train_per_sec_per_gpu": 73.7, "tokens/trainable": 191450} +{"epoch": 0.578125, "grad_norm": 0.9013202786445618, "learning_rate": 8.676844712937552e-05, "loss": 0.24249371886253357, "memory/device_reserved (GiB)": 42.71, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.27442, "step": 148, "tokens/total": 5188432, "tokens/train_per_sec_per_gpu": 59.93, "tokens/trainable": 192743} +{"epoch": 0.58203125, "grad_norm": 1.0605254173278809, "learning_rate": 8.656219011037509e-05, "loss": 0.2676098942756653, "memory/device_reserved (GiB)": 42.71, "memory/max_active (GiB)": 39.08, "memory/max_allocated (GiB)": 39.08, "ppl": 1.30684, "step": 149, "tokens/total": 5225840, "tokens/train_per_sec_per_gpu": 65.47, "tokens/trainable": 194029} +{"epoch": 0.5859375, "grad_norm": 1.4924793243408203, "learning_rate": 8.63546196598125e-05, "loss": 0.28985482454299927, "memory/device_reserved (GiB)": 42.71, "memory/max_active (GiB)": 39.32, "memory/max_allocated (GiB)": 39.32, "ppl": 1.33623, "step": 150, "tokens/total": 5263904, "tokens/train_per_sec_per_gpu": 65.71, "tokens/trainable": 195381} +{"epoch": 0.58984375, "grad_norm": 1.3284786939620972, "learning_rate": 8.614574441554145e-05, "loss": 0.3447296619415283, "memory/device_reserved (GiB)": 42.71, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.41161, "step": 151, "tokens/total": 5296704, "tokens/train_per_sec_per_gpu": 77.26, "tokens/trainable": 196585} +{"epoch": 0.59375, "grad_norm": 1.0347981452941895, "learning_rate": 8.593557306971349e-05, "loss": 0.26332178711891174, "memory/device_reserved (GiB)": 42.71, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.30125, "step": 152, "tokens/total": 5330208, "tokens/train_per_sec_per_gpu": 72.17, "tokens/trainable": 197896} +{"epoch": 0.59765625, "grad_norm": 2.6427342891693115, "learning_rate": 8.572411436841618e-05, "loss": 0.3168680965900421, "memory/device_reserved (GiB)": 42.71, "memory/max_active (GiB)": 36.96, "memory/max_allocated (GiB)": 36.96, "ppl": 1.37282, "step": 153, "tokens/total": 5364112, "tokens/train_per_sec_per_gpu": 75.86, "tokens/trainable": 199204} +{"epoch": 0.6015625, "grad_norm": 1.1412192583084106, "learning_rate": 8.551137711130922e-05, "loss": 0.2643027901649475, "memory/device_reserved (GiB)": 42.71, "memory/max_active (GiB)": 38.51, "memory/max_allocated (GiB)": 38.51, "ppl": 1.30252, "step": 154, "tokens/total": 5396192, "tokens/train_per_sec_per_gpu": 74.79, "tokens/trainable": 200471} +{"epoch": 0.60546875, "grad_norm": 1.2462167739868164, "learning_rate": 8.529737015125824e-05, "loss": 0.2390793412923813, "memory/device_reserved (GiB)": 42.71, "memory/max_active (GiB)": 38.29, "memory/max_allocated (GiB)": 38.29, "ppl": 1.27008, "step": 155, "tokens/total": 5431632, "tokens/train_per_sec_per_gpu": 70.04, "tokens/trainable": 201760} +{"epoch": 0.609375, "grad_norm": 1.1596840620040894, "learning_rate": 8.508210239396639e-05, "loss": 0.3079923093318939, "memory/device_reserved (GiB)": 42.71, "memory/max_active (GiB)": 39.85, "memory/max_allocated (GiB)": 39.85, "ppl": 1.36069, "step": 156, "tokens/total": 5469680, "tokens/train_per_sec_per_gpu": 67.65, "tokens/trainable": 203042} +{"epoch": 0.61328125, "grad_norm": 0.9381734728813171, "learning_rate": 8.486558279760375e-05, "loss": 0.2862294316291809, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 35.9, "memory/max_allocated (GiB)": 35.9, "ppl": 1.3314, "step": 157, "tokens/total": 5500752, "tokens/train_per_sec_per_gpu": 81.27, "tokens/trainable": 204285} +{"epoch": 0.6171875, "grad_norm": 0.976057231426239, "learning_rate": 8.464782037243449e-05, "loss": 0.26304784417152405, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 36.93, "memory/max_allocated (GiB)": 36.93, "ppl": 1.30089, "step": 158, "tokens/total": 5533184, "tokens/train_per_sec_per_gpu": 75.06, "tokens/trainable": 205520} +{"epoch": 0.62109375, "grad_norm": 2.064314603805542, "learning_rate": 8.442882418044202e-05, "loss": 0.23375362157821655, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.26333, "step": 159, "tokens/total": 5566144, "tokens/train_per_sec_per_gpu": 76.66, "tokens/trainable": 206841} +{"epoch": 0.625, "grad_norm": 1.17238450050354, "learning_rate": 8.420860333495179e-05, "loss": 0.23496553301811218, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.26487, "step": 160, "tokens/total": 5600384, "tokens/train_per_sec_per_gpu": 72.82, "tokens/trainable": 208094} +{"epoch": 0.62890625, "grad_norm": 1.2083019018173218, "learning_rate": 8.398716700025208e-05, "loss": 0.2720349431037903, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 37.61, "memory/max_allocated (GiB)": 37.61, "ppl": 1.31263, "step": 161, "tokens/total": 5634064, "tokens/train_per_sec_per_gpu": 69.09, "tokens/trainable": 209310} +{"epoch": 0.6328125, "grad_norm": 1.2309833765029907, "learning_rate": 8.376452439121266e-05, "loss": 0.26343297958374023, "memory/device_reserved (GiB)": 42.73, "memory/max_active (GiB)": 36.06, "memory/max_allocated (GiB)": 36.06, "ppl": 1.30139, "step": 162, "tokens/total": 5664736, "tokens/train_per_sec_per_gpu": 84.87, "tokens/trainable": 210570} +{"epoch": 0.63671875, "grad_norm": 1.9401957988739014, "learning_rate": 8.354068477290124e-05, "loss": 0.2648431062698364, "memory/device_reserved (GiB)": 42.73, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.30323, "step": 163, "tokens/total": 5699312, "tokens/train_per_sec_per_gpu": 78.12, "tokens/trainable": 211866} +{"epoch": 0.640625, "grad_norm": 1.320098876953125, "learning_rate": 8.331565746019807e-05, "loss": 0.23752284049987793, "memory/device_reserved (GiB)": 42.73, "memory/max_active (GiB)": 36.78, "memory/max_allocated (GiB)": 36.78, "ppl": 1.2681, "step": 164, "tokens/total": 5731648, "tokens/train_per_sec_per_gpu": 84.37, "tokens/trainable": 213210} +{"epoch": 0.64453125, "grad_norm": 1.8506909608840942, "learning_rate": 8.308945181740812e-05, "loss": 0.26777735352516174, "memory/device_reserved (GiB)": 42.73, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.30706, "step": 165, "tokens/total": 5766688, "tokens/train_per_sec_per_gpu": 74.85, "tokens/trainable": 214481} +{"epoch": 0.6484375, "grad_norm": 1.1578972339630127, "learning_rate": 8.286207725787153e-05, "loss": 0.24568666517734528, "memory/device_reserved (GiB)": 42.73, "memory/max_active (GiB)": 39.14, "memory/max_allocated (GiB)": 39.14, "ppl": 1.2785, "step": 166, "tokens/total": 5803904, "tokens/train_per_sec_per_gpu": 63.64, "tokens/trainable": 215726} +{"epoch": 0.65234375, "grad_norm": 1.3822427988052368, "learning_rate": 8.263354324357182e-05, "loss": 0.24878588318824768, "memory/device_reserved (GiB)": 42.73, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.28247, "step": 167, "tokens/total": 5838960, "tokens/train_per_sec_per_gpu": 74.32, "tokens/trainable": 217061} +{"epoch": 0.65625, "grad_norm": 1.427114486694336, "learning_rate": 8.240385928474219e-05, "loss": 0.2953055202960968, "memory/device_reserved (GiB)": 42.73, "memory/max_active (GiB)": 36.92, "memory/max_allocated (GiB)": 36.92, "ppl": 1.34354, "step": 168, "tokens/total": 5872000, "tokens/train_per_sec_per_gpu": 71.6, "tokens/trainable": 218291} +{"epoch": 0.66015625, "grad_norm": 1.2787092924118042, "learning_rate": 8.217303493946967e-05, "loss": 0.23950815200805664, "memory/device_reserved (GiB)": 42.73, "memory/max_active (GiB)": 37.44, "memory/max_allocated (GiB)": 37.44, "ppl": 1.27062, "step": 169, "tokens/total": 5906640, "tokens/train_per_sec_per_gpu": 71.95, "tokens/trainable": 219578} +{"epoch": 0.6640625, "grad_norm": 1.0684055089950562, "learning_rate": 8.194107981329746e-05, "loss": 0.27287930250167847, "memory/device_reserved (GiB)": 42.73, "memory/max_active (GiB)": 39.87, "memory/max_allocated (GiB)": 39.87, "ppl": 1.31374, "step": 170, "tokens/total": 5945440, "tokens/train_per_sec_per_gpu": 64.98, "tokens/trainable": 220862} +{"epoch": 0.66796875, "grad_norm": 1.8823976516723633, "learning_rate": 8.170800355882518e-05, "loss": 0.2294647991657257, "memory/device_reserved (GiB)": 42.73, "memory/max_active (GiB)": 37.4, "memory/max_allocated (GiB)": 37.4, "ppl": 1.25793, "step": 171, "tokens/total": 5980144, "tokens/train_per_sec_per_gpu": 83.67, "tokens/trainable": 222216} +{"epoch": 0.671875, "grad_norm": 1.6012626886367798, "learning_rate": 8.147381587530713e-05, "loss": 0.2681749165058136, "memory/device_reserved (GiB)": 42.73, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 1.30758, "step": 172, "tokens/total": 6014080, "tokens/train_per_sec_per_gpu": 77.45, "tokens/trainable": 223482} +{"epoch": 0.67578125, "grad_norm": 1.1255996227264404, "learning_rate": 8.123852650824877e-05, "loss": 0.25189661979675293, "memory/device_reserved (GiB)": 42.73, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.28646, "step": 173, "tokens/total": 6048288, "tokens/train_per_sec_per_gpu": 78.15, "tokens/trainable": 224815} +{"epoch": 0.6796875, "grad_norm": 3.0269641876220703, "learning_rate": 8.100214524900103e-05, "loss": 0.2676966190338135, "memory/device_reserved (GiB)": 42.73, "memory/max_active (GiB)": 38.7, "memory/max_allocated (GiB)": 38.7, "ppl": 1.30695, "step": 174, "tokens/total": 6083840, "tokens/train_per_sec_per_gpu": 62.73, "tokens/trainable": 226058} +{"epoch": 0.68359375, "grad_norm": 1.031988263130188, "learning_rate": 8.076468193435301e-05, "loss": 0.27933692932128906, "memory/device_reserved (GiB)": 42.73, "memory/max_active (GiB)": 36.56, "memory/max_allocated (GiB)": 36.56, "ppl": 1.32225, "step": 175, "tokens/total": 6115008, "tokens/train_per_sec_per_gpu": 77.29, "tokens/trainable": 227264} +{"epoch": 0.6875, "grad_norm": 1.571609616279602, "learning_rate": 8.052614644612253e-05, "loss": 0.24361498653888702, "memory/device_reserved (GiB)": 42.73, "memory/max_active (GiB)": 39.46, "memory/max_allocated (GiB)": 39.46, "ppl": 1.27585, "step": 176, "tokens/total": 6153040, "tokens/train_per_sec_per_gpu": 62.56, "tokens/trainable": 228584} +{"epoch": 0.69140625, "grad_norm": 0.8319208025932312, "learning_rate": 8.028654871074489e-05, "loss": 0.20275089144706726, "memory/device_reserved (GiB)": 42.73, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.22477, "step": 177, "tokens/total": 6187888, "tokens/train_per_sec_per_gpu": 74.47, "tokens/trainable": 229914} +{"epoch": 0.6953125, "grad_norm": 0.9243642091751099, "learning_rate": 8.004589869885986e-05, "loss": 0.23886466026306152, "memory/device_reserved (GiB)": 42.73, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.26981, "step": 178, "tokens/total": 6219600, "tokens/train_per_sec_per_gpu": 77.6, "tokens/trainable": 231136} +{"epoch": 0.69921875, "grad_norm": 0.8961188197135925, "learning_rate": 7.980420642489674e-05, "loss": 0.23399746417999268, "memory/device_reserved (GiB)": 42.73, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 1.26364, "step": 179, "tokens/total": 6258576, "tokens/train_per_sec_per_gpu": 68.08, "tokens/trainable": 232460} +{"epoch": 0.703125, "grad_norm": 0.9787206649780273, "learning_rate": 7.95614819466576e-05, "loss": 0.2232615202665329, "memory/device_reserved (GiB)": 42.73, "memory/max_active (GiB)": 37.67, "memory/max_allocated (GiB)": 37.67, "ppl": 1.25015, "step": 180, "tokens/total": 6292608, "tokens/train_per_sec_per_gpu": 77.64, "tokens/trainable": 233811} +{"epoch": 0.70703125, "grad_norm": 1.0961039066314697, "learning_rate": 7.931773536489872e-05, "loss": 0.2404777705669403, "memory/device_reserved (GiB)": 42.73, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.27186, "step": 181, "tokens/total": 6327392, "tokens/train_per_sec_per_gpu": 74.87, "tokens/trainable": 235097} +{"epoch": 0.7109375, "grad_norm": 1.9952893257141113, "learning_rate": 7.907297682291035e-05, "loss": 0.21719419956207275, "memory/device_reserved (GiB)": 42.73, "memory/max_active (GiB)": 39.48, "memory/max_allocated (GiB)": 39.48, "ppl": 1.24259, "step": 182, "tokens/total": 6362288, "tokens/train_per_sec_per_gpu": 76.08, "tokens/trainable": 236418} +{"epoch": 0.71484375, "grad_norm": 1.1075724363327026, "learning_rate": 7.882721650609442e-05, "loss": 0.19527658820152283, "memory/device_reserved (GiB)": 42.73, "memory/max_active (GiB)": 39.65, "memory/max_allocated (GiB)": 39.65, "ppl": 1.21565, "step": 183, "tokens/total": 6402832, "tokens/train_per_sec_per_gpu": 65.01, "tokens/trainable": 237867} +{"epoch": 0.71875, "grad_norm": 1.2273283004760742, "learning_rate": 7.85804646415409e-05, "loss": 0.24175062775611877, "memory/device_reserved (GiB)": 42.73, "memory/max_active (GiB)": 37.96, "memory/max_allocated (GiB)": 37.96, "ppl": 1.27348, "step": 184, "tokens/total": 6438416, "tokens/train_per_sec_per_gpu": 78.65, "tokens/trainable": 239218} +{"epoch": 0.72265625, "grad_norm": 1.4064007997512817, "learning_rate": 7.833273149760207e-05, "loss": 0.2478489875793457, "memory/device_reserved (GiB)": 42.73, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.28127, "step": 185, "tokens/total": 6469312, "tokens/train_per_sec_per_gpu": 78.65, "tokens/trainable": 240490} +{"epoch": 0.7265625, "grad_norm": 1.2677496671676636, "learning_rate": 7.808402738346527e-05, "loss": 0.2128407508134842, "memory/device_reserved (GiB)": 42.73, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.23719, "step": 186, "tokens/total": 6508544, "tokens/train_per_sec_per_gpu": 71.64, "tokens/trainable": 241907} +{"epoch": 0.73046875, "grad_norm": 1.0813429355621338, "learning_rate": 7.783436264872382e-05, "loss": 0.2348504364490509, "memory/device_reserved (GiB)": 42.73, "memory/max_active (GiB)": 39.12, "memory/max_allocated (GiB)": 39.12, "ppl": 1.26472, "step": 187, "tokens/total": 6545632, "tokens/train_per_sec_per_gpu": 70.06, "tokens/trainable": 243258} +{"epoch": 0.734375, "grad_norm": 0.8784555792808533, "learning_rate": 7.758374768294647e-05, "loss": 0.20427308976650238, "memory/device_reserved (GiB)": 42.73, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.22663, "step": 188, "tokens/total": 6581808, "tokens/train_per_sec_per_gpu": 77.76, "tokens/trainable": 244671} +{"epoch": 0.73828125, "grad_norm": 1.269136905670166, "learning_rate": 7.733219291524489e-05, "loss": 0.22042465209960938, "memory/device_reserved (GiB)": 42.73, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.24661, "step": 189, "tokens/total": 6615616, "tokens/train_per_sec_per_gpu": 80.61, "tokens/trainable": 245995} +{"epoch": 0.7421875, "grad_norm": 0.774482250213623, "learning_rate": 7.707970881383977e-05, "loss": 0.2017061710357666, "memory/device_reserved (GiB)": 42.73, "memory/max_active (GiB)": 36.47, "memory/max_allocated (GiB)": 36.47, "ppl": 1.22349, "step": 190, "tokens/total": 6647520, "tokens/train_per_sec_per_gpu": 82.97, "tokens/trainable": 247251} +{"epoch": 0.74609375, "grad_norm": 0.8582466840744019, "learning_rate": 7.682630588562518e-05, "loss": 0.22238042950630188, "memory/device_reserved (GiB)": 42.73, "memory/max_active (GiB)": 38.81, "memory/max_allocated (GiB)": 38.81, "ppl": 1.24905, "step": 191, "tokens/total": 6682992, "tokens/train_per_sec_per_gpu": 71.94, "tokens/trainable": 248586} +{"epoch": 0.75, "grad_norm": 1.2368454933166504, "learning_rate": 7.657199467573129e-05, "loss": 0.22286450862884521, "memory/device_reserved (GiB)": 42.73, "memory/max_active (GiB)": 37.23, "memory/max_allocated (GiB)": 37.23, "ppl": 1.24965, "step": 192, "tokens/total": 6717296, "tokens/train_per_sec_per_gpu": 72.34, "tokens/trainable": 249844} +{"epoch": 0.75390625, "grad_norm": 0.8806758522987366, "learning_rate": 7.631678576708561e-05, "loss": 0.19038048386573792, "memory/device_reserved (GiB)": 42.73, "memory/max_active (GiB)": 38.16, "memory/max_allocated (GiB)": 38.16, "ppl": 1.20971, "step": 193, "tokens/total": 6751936, "tokens/train_per_sec_per_gpu": 82.68, "tokens/trainable": 251224} +{"epoch": 0.7578125, "grad_norm": 1.2281227111816406, "learning_rate": 7.606068977997255e-05, "loss": 0.2317308485507965, "memory/device_reserved (GiB)": 42.73, "memory/max_active (GiB)": 39.9, "memory/max_allocated (GiB)": 39.9, "ppl": 1.26078, "step": 194, "tokens/total": 6790016, "tokens/train_per_sec_per_gpu": 76.47, "tokens/trainable": 252645} +{"epoch": 0.76171875, "grad_norm": 1.4336650371551514, "learning_rate": 7.580371737159148e-05, "loss": 0.2799326181411743, "memory/device_reserved (GiB)": 42.73, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.32304, "step": 195, "tokens/total": 6826256, "tokens/train_per_sec_per_gpu": 66.5, "tokens/trainable": 253878} +{"epoch": 0.765625, "grad_norm": 1.3447657823562622, "learning_rate": 7.554587923561324e-05, "loss": 0.20692922174930573, "memory/device_reserved (GiB)": 42.73, "memory/max_active (GiB)": 39.11, "memory/max_allocated (GiB)": 39.11, "ppl": 1.2299, "step": 196, "tokens/total": 6861232, "tokens/train_per_sec_per_gpu": 70.21, "tokens/trainable": 255151} +{"epoch": 0.76953125, "grad_norm": 1.067163348197937, "learning_rate": 7.528718610173511e-05, "loss": 0.20368939638137817, "memory/device_reserved (GiB)": 42.73, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.22592, "step": 197, "tokens/total": 6896912, "tokens/train_per_sec_per_gpu": 78.46, "tokens/trainable": 256514} +{"epoch": 0.7734375, "grad_norm": 1.2013969421386719, "learning_rate": 7.502764873523431e-05, "loss": 0.22660307586193085, "memory/device_reserved (GiB)": 42.73, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.25433, "step": 198, "tokens/total": 6931440, "tokens/train_per_sec_per_gpu": 77.79, "tokens/trainable": 257826} +{"epoch": 0.77734375, "grad_norm": 0.9686781167984009, "learning_rate": 7.476727793652011e-05, "loss": 0.19488194584846497, "memory/device_reserved (GiB)": 42.73, "memory/max_active (GiB)": 36.92, "memory/max_allocated (GiB)": 36.92, "ppl": 1.21517, "step": 199, "tokens/total": 6965088, "tokens/train_per_sec_per_gpu": 77.36, "tokens/trainable": 259143} +{"epoch": 0.78125, "grad_norm": 1.1899651288986206, "learning_rate": 7.450608454068415e-05, "loss": 0.19230496883392334, "memory/device_reserved (GiB)": 42.73, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 1.21204, "step": 200, "tokens/total": 6999856, "tokens/train_per_sec_per_gpu": 73.86, "tokens/trainable": 260459} +{"epoch": 0.78515625, "grad_norm": 0.9615203738212585, "learning_rate": 7.424407941704987e-05, "loss": 0.18500062823295593, "memory/device_reserved (GiB)": 42.73, "memory/max_active (GiB)": 38.47, "memory/max_allocated (GiB)": 38.47, "ppl": 1.20322, "step": 201, "tokens/total": 7036656, "tokens/train_per_sec_per_gpu": 76.3, "tokens/trainable": 261834} +{"epoch": 0.7890625, "grad_norm": 1.0878599882125854, "learning_rate": 7.398127346871986e-05, "loss": 0.18717961013317108, "memory/device_reserved (GiB)": 42.73, "memory/max_active (GiB)": 39.77, "memory/max_allocated (GiB)": 39.77, "ppl": 1.20584, "step": 202, "tokens/total": 7074016, "tokens/train_per_sec_per_gpu": 71.87, "tokens/trainable": 263189} +{"epoch": 0.79296875, "grad_norm": 2.1685593128204346, "learning_rate": 7.371767763212238e-05, "loss": 0.21375519037246704, "memory/device_reserved (GiB)": 42.73, "memory/max_active (GiB)": 39.71, "memory/max_allocated (GiB)": 39.71, "ppl": 1.23832, "step": 203, "tokens/total": 7109824, "tokens/train_per_sec_per_gpu": 75.68, "tokens/trainable": 264485} +{"epoch": 0.796875, "grad_norm": 1.1244175434112549, "learning_rate": 7.345330287655617e-05, "loss": 0.24006737768650055, "memory/device_reserved (GiB)": 42.73, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.27133, "step": 204, "tokens/total": 7148128, "tokens/train_per_sec_per_gpu": 62.92, "tokens/trainable": 265776} +{"epoch": 0.80078125, "grad_norm": 1.3486497402191162, "learning_rate": 7.31881602037339e-05, "loss": 0.22426100075244904, "memory/device_reserved (GiB)": 42.73, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.2514, "step": 205, "tokens/total": 7180432, "tokens/train_per_sec_per_gpu": 77.29, "tokens/trainable": 267038} +{"epoch": 0.8046875, "grad_norm": 0.9343635439872742, "learning_rate": 7.29222606473245e-05, "loss": 0.20139145851135254, "memory/device_reserved (GiB)": 42.73, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.2231, "step": 206, "tokens/total": 7213312, "tokens/train_per_sec_per_gpu": 78.66, "tokens/trainable": 268296} +{"epoch": 0.80859375, "grad_norm": 1.0532854795455933, "learning_rate": 7.265561527249383e-05, "loss": 0.20239396393299103, "memory/device_reserved (GiB)": 42.73, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.22433, "step": 207, "tokens/total": 7247712, "tokens/train_per_sec_per_gpu": 75.62, "tokens/trainable": 269563} +{"epoch": 0.8125, "grad_norm": 0.7700850963592529, "learning_rate": 7.238823517544436e-05, "loss": 0.15947383642196655, "memory/device_reserved (GiB)": 42.73, "memory/max_active (GiB)": 39.53, "memory/max_allocated (GiB)": 39.53, "ppl": 1.17289, "step": 208, "tokens/total": 7284928, "tokens/train_per_sec_per_gpu": 78.31, "tokens/trainable": 270981} +{"epoch": 0.81640625, "grad_norm": 1.117472529411316, "learning_rate": 7.212013148295333e-05, "loss": 0.2028118073940277, "memory/device_reserved (GiB)": 42.73, "memory/max_active (GiB)": 37.61, "memory/max_allocated (GiB)": 37.61, "ppl": 1.22484, "step": 209, "tokens/total": 7319232, "tokens/train_per_sec_per_gpu": 75.56, "tokens/trainable": 272296} +{"epoch": 0.8203125, "grad_norm": 1.1212177276611328, "learning_rate": 7.185131535190975e-05, "loss": 0.20447689294815063, "memory/device_reserved (GiB)": 42.73, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.22688, "step": 210, "tokens/total": 7352192, "tokens/train_per_sec_per_gpu": 80.58, "tokens/trainable": 273605} +{"epoch": 0.82421875, "grad_norm": 1.059446930885315, "learning_rate": 7.158179796885005e-05, "loss": 0.19593656063079834, "memory/device_reserved (GiB)": 42.73, "memory/max_active (GiB)": 37.82, "memory/max_allocated (GiB)": 37.82, "ppl": 1.21645, "step": 211, "tokens/total": 7386864, "tokens/train_per_sec_per_gpu": 78.72, "tokens/trainable": 274896} +{"epoch": 0.828125, "grad_norm": 1.3152967691421509, "learning_rate": 7.131159054949273e-05, "loss": 0.21411536633968353, "memory/device_reserved (GiB)": 42.73, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.23877, "step": 212, "tokens/total": 7425728, "tokens/train_per_sec_per_gpu": 66.99, "tokens/trainable": 276207} +{"epoch": 0.83203125, "grad_norm": 1.2571353912353516, "learning_rate": 7.104070433827139e-05, "loss": 0.21442317962646484, "memory/device_reserved (GiB)": 42.73, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.23915, "step": 213, "tokens/total": 7461440, "tokens/train_per_sec_per_gpu": 66.22, "tokens/trainable": 277414} +{"epoch": 0.8359375, "grad_norm": 1.706966757774353, "learning_rate": 7.076915060786705e-05, "loss": 0.20991250872612, "memory/device_reserved (GiB)": 42.73, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.23357, "step": 214, "tokens/total": 7494352, "tokens/train_per_sec_per_gpu": 76.0, "tokens/trainable": 278688} +{"epoch": 0.83984375, "grad_norm": 1.0589016675949097, "learning_rate": 7.049694065873882e-05, "loss": 0.18289992213249207, "memory/device_reserved (GiB)": 42.73, "memory/max_active (GiB)": 38.14, "memory/max_allocated (GiB)": 38.14, "ppl": 1.20069, "step": 215, "tokens/total": 7529216, "tokens/train_per_sec_per_gpu": 72.03, "tokens/trainable": 279957} +{"epoch": 0.84375, "grad_norm": 1.3209904432296753, "learning_rate": 7.022408581865382e-05, "loss": 0.17673419415950775, "memory/device_reserved (GiB)": 42.73, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.19331, "step": 216, "tokens/total": 7563680, "tokens/train_per_sec_per_gpu": 70.35, "tokens/trainable": 281267} +{"epoch": 0.84765625, "grad_norm": 1.1395344734191895, "learning_rate": 6.99505974422157e-05, "loss": 0.19026288390159607, "memory/device_reserved (GiB)": 42.73, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.20957, "step": 217, "tokens/total": 7598416, "tokens/train_per_sec_per_gpu": 82.32, "tokens/trainable": 282630} +{"epoch": 0.8515625, "grad_norm": 1.0504450798034668, "learning_rate": 6.967648691039213e-05, "loss": 0.20327123999595642, "memory/device_reserved (GiB)": 42.73, "memory/max_active (GiB)": 36.77, "memory/max_allocated (GiB)": 36.77, "ppl": 1.2254, "step": 218, "tokens/total": 7630656, "tokens/train_per_sec_per_gpu": 81.13, "tokens/trainable": 283939} +{"epoch": 0.85546875, "grad_norm": 1.3599443435668945, "learning_rate": 6.940176563004123e-05, "loss": 0.21134768426418304, "memory/device_reserved (GiB)": 42.73, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.23534, "step": 219, "tokens/total": 7665040, "tokens/train_per_sec_per_gpu": 70.28, "tokens/trainable": 285146} +{"epoch": 0.859375, "grad_norm": 1.691861629486084, "learning_rate": 6.912644503343682e-05, "loss": 0.1938905566930771, "memory/device_reserved (GiB)": 42.73, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.21396, "step": 220, "tokens/total": 7703472, "tokens/train_per_sec_per_gpu": 69.66, "tokens/trainable": 286436} +{"epoch": 0.86328125, "grad_norm": 1.308412790298462, "learning_rate": 6.885053657779273e-05, "loss": 0.1917334645986557, "memory/device_reserved (GiB)": 42.73, "memory/max_active (GiB)": 36.49, "memory/max_allocated (GiB)": 36.49, "ppl": 1.21135, "step": 221, "tokens/total": 7735312, "tokens/train_per_sec_per_gpu": 77.84, "tokens/trainable": 287724} +{"epoch": 0.8671875, "grad_norm": 1.0474563837051392, "learning_rate": 6.857405174478604e-05, "loss": 0.1925778090953827, "memory/device_reserved (GiB)": 42.73, "memory/max_active (GiB)": 37.45, "memory/max_allocated (GiB)": 37.45, "ppl": 1.21237, "step": 222, "tokens/total": 7770304, "tokens/train_per_sec_per_gpu": 73.17, "tokens/trainable": 289006} +{"epoch": 0.87109375, "grad_norm": 1.0797053575515747, "learning_rate": 6.82970020400792e-05, "loss": 0.20057186484336853, "memory/device_reserved (GiB)": 42.98, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.2221, "step": 223, "tokens/total": 7807728, "tokens/train_per_sec_per_gpu": 62.43, "tokens/trainable": 290244} +{"epoch": 0.875, "grad_norm": 0.8438169956207275, "learning_rate": 6.801939899284132e-05, "loss": 0.1546141803264618, "memory/device_reserved (GiB)": 42.98, "memory/max_active (GiB)": 38.34, "memory/max_allocated (GiB)": 38.34, "ppl": 1.16721, "step": 224, "tokens/total": 7843840, "tokens/train_per_sec_per_gpu": 74.42, "tokens/trainable": 291604} +{"epoch": 0.87890625, "grad_norm": 0.8656520843505859, "learning_rate": 6.774125415526827e-05, "loss": 0.17061518132686615, "memory/device_reserved (GiB)": 42.98, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.18603, "step": 225, "tokens/total": 7877312, "tokens/train_per_sec_per_gpu": 81.48, "tokens/trainable": 292962} +{"epoch": 0.8828125, "grad_norm": 1.4650461673736572, "learning_rate": 6.746257910210214e-05, "loss": 0.18662965297698975, "memory/device_reserved (GiB)": 42.98, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.20518, "step": 226, "tokens/total": 7910720, "tokens/train_per_sec_per_gpu": 73.45, "tokens/trainable": 294237} +{"epoch": 0.88671875, "grad_norm": 1.517151951789856, "learning_rate": 6.718338543014937e-05, "loss": 0.2074587196111679, "memory/device_reserved (GiB)": 42.98, "memory/max_active (GiB)": 35.83, "memory/max_allocated (GiB)": 35.83, "ppl": 1.23055, "step": 227, "tokens/total": 7941088, "tokens/train_per_sec_per_gpu": 80.0, "tokens/trainable": 295463} +{"epoch": 0.890625, "grad_norm": 1.0197492837905884, "learning_rate": 6.69036847577983e-05, "loss": 0.18443098664283752, "memory/device_reserved (GiB)": 42.98, "memory/max_active (GiB)": 38.29, "memory/max_allocated (GiB)": 38.29, "ppl": 1.20253, "step": 228, "tokens/total": 7975168, "tokens/train_per_sec_per_gpu": 72.67, "tokens/trainable": 296732} +{"epoch": 0.89453125, "grad_norm": 1.3430910110473633, "learning_rate": 6.662348872453553e-05, "loss": 0.16773498058319092, "memory/device_reserved (GiB)": 42.98, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.18262, "step": 229, "tokens/total": 8011872, "tokens/train_per_sec_per_gpu": 67.21, "tokens/trainable": 297979} +{"epoch": 0.8984375, "grad_norm": 0.8002235293388367, "learning_rate": 6.63428089904618e-05, "loss": 0.18009048700332642, "memory/device_reserved (GiB)": 42.98, "memory/max_active (GiB)": 39.35, "memory/max_allocated (GiB)": 39.35, "ppl": 1.19733, "step": 230, "tokens/total": 8051248, "tokens/train_per_sec_per_gpu": 62.3, "tokens/trainable": 299248} +{"epoch": 0.90234375, "grad_norm": 0.884346604347229, "learning_rate": 6.60616572358065e-05, "loss": 0.1707170009613037, "memory/device_reserved (GiB)": 42.98, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.18616, "step": 231, "tokens/total": 8085872, "tokens/train_per_sec_per_gpu": 75.74, "tokens/trainable": 300501} +{"epoch": 0.90625, "grad_norm": 1.2118672132492065, "learning_rate": 6.578004516044172e-05, "loss": 0.18664953112602234, "memory/device_reserved (GiB)": 42.98, "memory/max_active (GiB)": 37.96, "memory/max_allocated (GiB)": 37.96, "ppl": 1.2052, "step": 232, "tokens/total": 8122064, "tokens/train_per_sec_per_gpu": 75.5, "tokens/trainable": 301773} +{"epoch": 0.91015625, "grad_norm": 0.9502357840538025, "learning_rate": 6.549798448339548e-05, "loss": 0.16766288876533508, "memory/device_reserved (GiB)": 42.98, "memory/max_active (GiB)": 38.86, "memory/max_allocated (GiB)": 38.86, "ppl": 1.18254, "step": 233, "tokens/total": 8159984, "tokens/train_per_sec_per_gpu": 75.2, "tokens/trainable": 303106} +{"epoch": 0.9140625, "grad_norm": 1.0262640714645386, "learning_rate": 6.521548694236384e-05, "loss": 0.17597542703151703, "memory/device_reserved (GiB)": 42.98, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.19241, "step": 234, "tokens/total": 8197536, "tokens/train_per_sec_per_gpu": 69.19, "tokens/trainable": 304427} +{"epoch": 0.91796875, "grad_norm": 1.0040448904037476, "learning_rate": 6.493256429322259e-05, "loss": 0.19528862833976746, "memory/device_reserved (GiB)": 42.98, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.21566, "step": 235, "tokens/total": 8233824, "tokens/train_per_sec_per_gpu": 68.64, "tokens/trainable": 305725} +{"epoch": 0.921875, "grad_norm": 1.20753014087677, "learning_rate": 6.464922830953799e-05, "loss": 0.1820376217365265, "memory/device_reserved (GiB)": 42.98, "memory/max_active (GiB)": 39.39, "memory/max_allocated (GiB)": 39.39, "ppl": 1.19966, "step": 236, "tokens/total": 8273328, "tokens/train_per_sec_per_gpu": 64.08, "tokens/trainable": 307038} +{"epoch": 0.92578125, "grad_norm": 1.171755313873291, "learning_rate": 6.436549078207688e-05, "loss": 0.18756763637065887, "memory/device_reserved (GiB)": 42.98, "memory/max_active (GiB)": 38.3, "memory/max_allocated (GiB)": 38.3, "ppl": 1.20631, "step": 237, "tokens/total": 8306640, "tokens/train_per_sec_per_gpu": 78.78, "tokens/trainable": 308318} +{"epoch": 0.9296875, "grad_norm": 0.9229746460914612, "learning_rate": 6.408136351831592e-05, "loss": 0.18469996750354767, "memory/device_reserved (GiB)": 42.98, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.20286, "step": 238, "tokens/total": 8339232, "tokens/train_per_sec_per_gpu": 74.98, "tokens/trainable": 309592} +{"epoch": 0.93359375, "grad_norm": 1.8413795232772827, "learning_rate": 6.379685834195036e-05, "loss": 0.1669381558895111, "memory/device_reserved (GiB)": 42.98, "memory/max_active (GiB)": 39.11, "memory/max_allocated (GiB)": 39.11, "ppl": 1.18168, "step": 239, "tokens/total": 8378976, "tokens/train_per_sec_per_gpu": 65.86, "tokens/trainable": 310997} +{"epoch": 0.9375, "grad_norm": 1.02641761302948, "learning_rate": 6.351198709240186e-05, "loss": 0.16096368432044983, "memory/device_reserved (GiB)": 42.98, "memory/max_active (GiB)": 39.73, "memory/max_allocated (GiB)": 39.73, "ppl": 1.17464, "step": 240, "tokens/total": 8419472, "tokens/train_per_sec_per_gpu": 60.82, "tokens/trainable": 312311} +{"epoch": 0.94140625, "grad_norm": 1.2957408428192139, "learning_rate": 6.32267616243259e-05, "loss": 0.17820033431053162, "memory/device_reserved (GiB)": 42.98, "memory/max_active (GiB)": 36.67, "memory/max_allocated (GiB)": 36.67, "ppl": 1.19506, "step": 241, "tokens/total": 8452192, "tokens/train_per_sec_per_gpu": 78.29, "tokens/trainable": 313597} +{"epoch": 0.9453125, "grad_norm": 1.1289178133010864, "learning_rate": 6.294119380711849e-05, "loss": 0.17495375871658325, "memory/device_reserved (GiB)": 42.98, "memory/max_active (GiB)": 38.7, "memory/max_allocated (GiB)": 38.7, "ppl": 1.19119, "step": 242, "tokens/total": 8486832, "tokens/train_per_sec_per_gpu": 68.64, "tokens/trainable": 314854} +{"epoch": 0.94921875, "grad_norm": 2.080325126647949, "learning_rate": 6.265529552442209e-05, "loss": 0.19728420674800873, "memory/device_reserved (GiB)": 42.98, "memory/max_active (GiB)": 37.83, "memory/max_allocated (GiB)": 37.83, "ppl": 1.21809, "step": 243, "tokens/total": 8522384, "tokens/train_per_sec_per_gpu": 73.71, "tokens/trainable": 316157} +{"epoch": 0.953125, "grad_norm": 0.8972201943397522, "learning_rate": 6.236907867363127e-05, "loss": 0.17592251300811768, "memory/device_reserved (GiB)": 42.98, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 1.19235, "step": 244, "tokens/total": 8556976, "tokens/train_per_sec_per_gpu": 74.31, "tokens/trainable": 317426} +{"epoch": 0.95703125, "grad_norm": 3.369708776473999, "learning_rate": 6.208255516539749e-05, "loss": 0.182033509016037, "memory/device_reserved (GiB)": 42.98, "memory/max_active (GiB)": 36.44, "memory/max_allocated (GiB)": 36.44, "ppl": 1.19965, "step": 245, "tokens/total": 8589456, "tokens/train_per_sec_per_gpu": 75.32, "tokens/trainable": 318688} +{"epoch": 0.9609375, "grad_norm": 1.1933006048202515, "learning_rate": 6.179573692313344e-05, "loss": 0.19844068586826324, "memory/device_reserved (GiB)": 43.1, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.2195, "step": 246, "tokens/total": 8626640, "tokens/train_per_sec_per_gpu": 71.65, "tokens/trainable": 319969} +{"epoch": 0.96484375, "grad_norm": 1.4771430492401123, "learning_rate": 6.150863588251694e-05, "loss": 0.17920494079589844, "memory/device_reserved (GiB)": 43.1, "memory/max_active (GiB)": 39.71, "memory/max_allocated (GiB)": 39.71, "ppl": 1.19627, "step": 247, "tokens/total": 8667472, "tokens/train_per_sec_per_gpu": 64.65, "tokens/trainable": 321326} +{"epoch": 0.96875, "grad_norm": 1.3276050090789795, "learning_rate": 6.122126399099419e-05, "loss": 0.1774035096168518, "memory/device_reserved (GiB)": 43.1, "memory/max_active (GiB)": 39.79, "memory/max_allocated (GiB)": 39.79, "ppl": 1.19411, "step": 248, "tokens/total": 8707712, "tokens/train_per_sec_per_gpu": 75.05, "tokens/trainable": 322867} +{"epoch": 0.97265625, "grad_norm": 2.1068477630615234, "learning_rate": 6.0933633207282615e-05, "loss": 0.21056097745895386, "memory/device_reserved (GiB)": 43.1, "memory/max_active (GiB)": 38.41, "memory/max_allocated (GiB)": 38.41, "ppl": 1.23437, "step": 249, "tokens/total": 8744256, "tokens/train_per_sec_per_gpu": 68.14, "tokens/trainable": 324180} +{"epoch": 0.9765625, "grad_norm": 0.7664649486541748, "learning_rate": 6.064575550087316e-05, "loss": 0.16814002394676208, "memory/device_reserved (GiB)": 43.1, "memory/max_active (GiB)": 39.79, "memory/max_allocated (GiB)": 39.79, "ppl": 1.1831, "step": 250, "tokens/total": 8781520, "tokens/train_per_sec_per_gpu": 70.97, "tokens/trainable": 325520} +{"epoch": 0.98046875, "grad_norm": 1.0452823638916016, "learning_rate": 6.0357642851532245e-05, "loss": 0.17386788129806519, "memory/device_reserved (GiB)": 43.1, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.1899, "step": 251, "tokens/total": 8818352, "tokens/train_per_sec_per_gpu": 65.51, "tokens/trainable": 326850} +{"epoch": 0.984375, "grad_norm": 1.1230496168136597, "learning_rate": 6.0069307248803294e-05, "loss": 0.18744081258773804, "memory/device_reserved (GiB)": 43.1, "memory/max_active (GiB)": 38.03, "memory/max_allocated (GiB)": 38.03, "ppl": 1.20616, "step": 252, "tokens/total": 8853952, "tokens/train_per_sec_per_gpu": 68.47, "tokens/trainable": 328104} +{"epoch": 0.98828125, "grad_norm": 0.8905748724937439, "learning_rate": 5.9780760691507635e-05, "loss": 0.18094143271446228, "memory/device_reserved (GiB)": 43.1, "memory/max_active (GiB)": 36.44, "memory/max_allocated (GiB)": 36.44, "ppl": 1.19834, "step": 253, "tokens/total": 8885696, "tokens/train_per_sec_per_gpu": 74.91, "tokens/trainable": 329303} +{"epoch": 0.9921875, "grad_norm": 1.1986689567565918, "learning_rate": 5.9492015187245334e-05, "loss": 0.18008656799793243, "memory/device_reserved (GiB)": 43.1, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.19732, "step": 254, "tokens/total": 8923536, "tokens/train_per_sec_per_gpu": 66.86, "tokens/trainable": 330527} +{"epoch": 0.99609375, "grad_norm": 1.2885645627975464, "learning_rate": 5.920308275189541e-05, "loss": 0.18431732058525085, "memory/device_reserved (GiB)": 43.1, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.2024, "step": 255, "tokens/total": 8955776, "tokens/train_per_sec_per_gpu": 79.95, "tokens/trainable": 331775} +{"epoch": 1.0, "grad_norm": 0.9507248401641846, "learning_rate": 5.8913975409115874e-05, "loss": 0.1604873538017273, "memory/device_reserved (GiB)": 43.1, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.17408, "step": 256, "tokens/total": 8989520, "tokens/train_per_sec_per_gpu": 81.19, "tokens/trainable": 333131} +{"epoch": 1.00390625, "grad_norm": 1.0571062564849854, "learning_rate": 5.8624705189843395e-05, "loss": 0.14452090859413147, "memory/device_reserved (GiB)": 43.1, "memory/max_active (GiB)": 39.32, "memory/max_allocated (GiB)": 39.32, "ppl": 1.15549, "step": 257, "tokens/total": 9027312, "tokens/train_per_sec_per_gpu": 65.57, "tokens/trainable": 334463} +{"epoch": 1.0078125, "grad_norm": 0.8089858889579773, "learning_rate": 5.833528413179249e-05, "loss": 0.1433231234550476, "memory/device_reserved (GiB)": 40.8, "memory/max_active (GiB)": 38.6, "memory/max_allocated (GiB)": 38.6, "ppl": 1.1541, "step": 258, "tokens/total": 9064240, "tokens/train_per_sec_per_gpu": 70.2, "tokens/trainable": 335787} +{"epoch": 1.01171875, "grad_norm": 1.5696842670440674, "learning_rate": 5.80457242789548e-05, "loss": 0.14021456241607666, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 39.65, "memory/max_allocated (GiB)": 39.65, "ppl": 1.15052, "step": 259, "tokens/total": 9104624, "tokens/train_per_sec_per_gpu": 71.48, "tokens/trainable": 337258} +{"epoch": 1.015625, "grad_norm": 1.4077318906784058, "learning_rate": 5.77560376810977e-05, "loss": 0.13867169618606567, "memory/device_reserved (GiB)": 41.62, "memory/max_active (GiB)": 38.37, "memory/max_allocated (GiB)": 38.37, "ppl": 1.14875, "step": 260, "tokens/total": 9138208, "tokens/train_per_sec_per_gpu": 75.77, "tokens/trainable": 338606} +{"epoch": 1.01953125, "grad_norm": 0.9108366966247559, "learning_rate": 5.7466236393263005e-05, "loss": 0.15056583285331726, "memory/device_reserved (GiB)": 41.62, "memory/max_active (GiB)": 39.12, "memory/max_allocated (GiB)": 39.12, "ppl": 1.16249, "step": 261, "tokens/total": 9173088, "tokens/train_per_sec_per_gpu": 73.48, "tokens/trainable": 339861} +{"epoch": 1.0234375, "grad_norm": 2.6635680198669434, "learning_rate": 5.717633247526522e-05, "loss": 0.15270669758319855, "memory/device_reserved (GiB)": 41.62, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.16498, "step": 262, "tokens/total": 9204272, "tokens/train_per_sec_per_gpu": 78.79, "tokens/trainable": 341115} +{"epoch": 1.02734375, "grad_norm": 1.120534062385559, "learning_rate": 5.688633799118971e-05, "loss": 0.18988138437271118, "memory/device_reserved (GiB)": 41.62, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.20911, "step": 263, "tokens/total": 9236928, "tokens/train_per_sec_per_gpu": 68.08, "tokens/trainable": 342300} +{"epoch": 1.03125, "grad_norm": 0.6175299286842346, "learning_rate": 5.659626500889066e-05, "loss": 0.13103747367858887, "memory/device_reserved (GiB)": 41.62, "memory/max_active (GiB)": 37.12, "memory/max_allocated (GiB)": 37.12, "ppl": 1.14001, "step": 264, "tokens/total": 9271024, "tokens/train_per_sec_per_gpu": 78.48, "tokens/trainable": 343647} +{"epoch": 1.03515625, "grad_norm": 0.9483981132507324, "learning_rate": 5.6306125599488905e-05, "loss": 0.1558130830526352, "memory/device_reserved (GiB)": 41.62, "memory/max_active (GiB)": 36.92, "memory/max_allocated (GiB)": 36.92, "ppl": 1.16861, "step": 265, "tokens/total": 9301936, "tokens/train_per_sec_per_gpu": 85.24, "tokens/trainable": 344905} +{"epoch": 1.0390625, "grad_norm": 1.3723715543746948, "learning_rate": 5.601593183686955e-05, "loss": 0.14878220856189728, "memory/device_reserved (GiB)": 41.62, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.16042, "step": 266, "tokens/total": 9339632, "tokens/train_per_sec_per_gpu": 72.23, "tokens/trainable": 346257} +{"epoch": 1.04296875, "grad_norm": 1.6249215602874756, "learning_rate": 5.572569579717961e-05, "loss": 0.17886261641979218, "memory/device_reserved (GiB)": 41.62, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.19586, "step": 267, "tokens/total": 9375200, "tokens/train_per_sec_per_gpu": 74.73, "tokens/trainable": 347583} +{"epoch": 1.046875, "grad_norm": 0.9374509453773499, "learning_rate": 5.543542955832538e-05, "loss": 0.14555172622203827, "memory/device_reserved (GiB)": 41.62, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.15668, "step": 268, "tokens/total": 9415616, "tokens/train_per_sec_per_gpu": 66.91, "tokens/trainable": 348969} +{"epoch": 1.05078125, "grad_norm": 0.9450746774673462, "learning_rate": 5.514514519946986e-05, "loss": 0.14397655427455902, "memory/device_reserved (GiB)": 41.62, "memory/max_active (GiB)": 39.6, "memory/max_allocated (GiB)": 39.6, "ppl": 1.15486, "step": 269, "tokens/total": 9456240, "tokens/train_per_sec_per_gpu": 68.69, "tokens/trainable": 350344} +{"epoch": 1.0546875, "grad_norm": 1.1119654178619385, "learning_rate": 5.485485480053015e-05, "loss": 0.1392560601234436, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 39.63, "memory/max_allocated (GiB)": 39.63, "ppl": 1.14942, "step": 270, "tokens/total": 9494912, "tokens/train_per_sec_per_gpu": 61.4, "tokens/trainable": 351604} +{"epoch": 1.05859375, "grad_norm": 0.7772090435028076, "learning_rate": 5.4564570441674645e-05, "loss": 0.14571262896060944, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.15686, "step": 271, "tokens/total": 9528704, "tokens/train_per_sec_per_gpu": 79.47, "tokens/trainable": 352887} +{"epoch": 1.0625, "grad_norm": 1.235534429550171, "learning_rate": 5.42743042028204e-05, "loss": 0.16857628524303436, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.18362, "step": 272, "tokens/total": 9565056, "tokens/train_per_sec_per_gpu": 65.24, "tokens/trainable": 354124} +{"epoch": 1.06640625, "grad_norm": 1.006470799446106, "learning_rate": 5.3984068163130464e-05, "loss": 0.13708941638469696, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.14693, "step": 273, "tokens/total": 9598960, "tokens/train_per_sec_per_gpu": 81.3, "tokens/trainable": 355490} +{"epoch": 1.0703125, "grad_norm": 1.100853443145752, "learning_rate": 5.369387440051111e-05, "loss": 0.15184631943702698, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 37.18, "memory/max_allocated (GiB)": 37.18, "ppl": 1.16398, "step": 274, "tokens/total": 9632752, "tokens/train_per_sec_per_gpu": 75.31, "tokens/trainable": 356755} +{"epoch": 1.07421875, "grad_norm": 0.7659812569618225, "learning_rate": 5.340373499110935e-05, "loss": 0.14635351300239563, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.15761, "step": 275, "tokens/total": 9668720, "tokens/train_per_sec_per_gpu": 75.51, "tokens/trainable": 358149} +{"epoch": 1.078125, "grad_norm": 1.7698367834091187, "learning_rate": 5.3113662008810304e-05, "loss": 0.15437665581703186, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 39.48, "memory/max_allocated (GiB)": 39.48, "ppl": 1.16693, "step": 276, "tokens/total": 9709472, "tokens/train_per_sec_per_gpu": 64.97, "tokens/trainable": 359529} +{"epoch": 1.08203125, "grad_norm": 0.7359117865562439, "learning_rate": 5.282366752473479e-05, "loss": 0.13336902856826782, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 38.32, "memory/max_allocated (GiB)": 38.32, "ppl": 1.14267, "step": 277, "tokens/total": 9747072, "tokens/train_per_sec_per_gpu": 69.44, "tokens/trainable": 360890} +{"epoch": 1.0859375, "grad_norm": 1.830635905265808, "learning_rate": 5.2533763606737005e-05, "loss": 0.16022846102714539, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 39.52, "memory/max_allocated (GiB)": 39.52, "ppl": 1.17378, "step": 278, "tokens/total": 9787312, "tokens/train_per_sec_per_gpu": 61.08, "tokens/trainable": 362187} +{"epoch": 1.08984375, "grad_norm": 0.7943869233131409, "learning_rate": 5.224396231890232e-05, "loss": 0.13409316539764404, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.1435, "step": 279, "tokens/total": 9819472, "tokens/train_per_sec_per_gpu": 81.57, "tokens/trainable": 363545} +{"epoch": 1.09375, "grad_norm": 0.9098377227783203, "learning_rate": 5.195427572104522e-05, "loss": 0.15408110618591309, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.16659, "step": 280, "tokens/total": 9854112, "tokens/train_per_sec_per_gpu": 77.99, "tokens/trainable": 364896} +{"epoch": 1.09765625, "grad_norm": 0.724147379398346, "learning_rate": 5.166471586820751e-05, "loss": 0.15597492456436157, "memory/device_reserved (GiB)": 42.5, "memory/max_active (GiB)": 39.53, "memory/max_allocated (GiB)": 39.53, "ppl": 1.1688, "step": 281, "tokens/total": 9891360, "tokens/train_per_sec_per_gpu": 73.53, "tokens/trainable": 366247} +{"epoch": 1.1015625, "grad_norm": 1.070271372795105, "learning_rate": 5.1375294810156615e-05, "loss": 0.15107885003089905, "memory/device_reserved (GiB)": 42.5, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.16309, "step": 282, "tokens/total": 9926144, "tokens/train_per_sec_per_gpu": 74.66, "tokens/trainable": 367518} +{"epoch": 1.10546875, "grad_norm": 0.9804981350898743, "learning_rate": 5.1086024590884144e-05, "loss": 0.16833102703094482, "memory/device_reserved (GiB)": 42.5, "memory/max_active (GiB)": 39.36, "memory/max_allocated (GiB)": 39.36, "ppl": 1.18333, "step": 283, "tokens/total": 9964544, "tokens/train_per_sec_per_gpu": 62.72, "tokens/trainable": 368742} +{"epoch": 1.109375, "grad_norm": 0.7359035015106201, "learning_rate": 5.079691724810461e-05, "loss": 0.14797154068946838, "memory/device_reserved (GiB)": 42.5, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.15948, "step": 284, "tokens/total": 10000720, "tokens/train_per_sec_per_gpu": 64.87, "tokens/trainable": 370021} +{"epoch": 1.11328125, "grad_norm": 0.7853086590766907, "learning_rate": 5.0507984812754684e-05, "loss": 0.15066930651664734, "memory/device_reserved (GiB)": 42.5, "memory/max_active (GiB)": 37.39, "memory/max_allocated (GiB)": 37.39, "ppl": 1.16261, "step": 285, "tokens/total": 10034608, "tokens/train_per_sec_per_gpu": 74.04, "tokens/trainable": 371311} +{"epoch": 1.1171875, "grad_norm": 0.6674942970275879, "learning_rate": 5.021923930849237e-05, "loss": 0.1549048125743866, "memory/device_reserved (GiB)": 42.5, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 1.16755, "step": 286, "tokens/total": 10068880, "tokens/train_per_sec_per_gpu": 73.65, "tokens/trainable": 372601} +{"epoch": 1.12109375, "grad_norm": 0.6684070825576782, "learning_rate": 4.99306927511967e-05, "loss": 0.14448165893554688, "memory/device_reserved (GiB)": 42.5, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.15544, "step": 287, "tokens/total": 10102512, "tokens/train_per_sec_per_gpu": 77.48, "tokens/trainable": 373885} +{"epoch": 1.125, "grad_norm": 1.2927378416061401, "learning_rate": 4.964235714846775e-05, "loss": 0.17768406867980957, "memory/device_reserved (GiB)": 42.5, "memory/max_active (GiB)": 36.52, "memory/max_allocated (GiB)": 36.52, "ppl": 1.19445, "step": 288, "tokens/total": 10134432, "tokens/train_per_sec_per_gpu": 77.43, "tokens/trainable": 375069} +{"epoch": 1.12890625, "grad_norm": 1.038150668144226, "learning_rate": 4.9354244499126866e-05, "loss": 0.18609219789505005, "memory/device_reserved (GiB)": 42.5, "memory/max_active (GiB)": 36.31, "memory/max_allocated (GiB)": 36.31, "ppl": 1.20453, "step": 289, "tokens/total": 10166400, "tokens/train_per_sec_per_gpu": 76.07, "tokens/trainable": 376317} +{"epoch": 1.1328125, "grad_norm": 1.0831764936447144, "learning_rate": 4.90663667927174e-05, "loss": 0.1507585346698761, "memory/device_reserved (GiB)": 42.5, "memory/max_active (GiB)": 37.91, "memory/max_allocated (GiB)": 37.91, "ppl": 1.16272, "step": 290, "tokens/total": 10199408, "tokens/train_per_sec_per_gpu": 83.1, "tokens/trainable": 377586} +{"epoch": 1.13671875, "grad_norm": 0.7707749009132385, "learning_rate": 4.877873600900581e-05, "loss": 0.1425180733203888, "memory/device_reserved (GiB)": 42.5, "memory/max_active (GiB)": 38.14, "memory/max_allocated (GiB)": 38.14, "ppl": 1.15317, "step": 291, "tokens/total": 10235680, "tokens/train_per_sec_per_gpu": 64.07, "tokens/trainable": 378872} +{"epoch": 1.140625, "grad_norm": 1.054226040840149, "learning_rate": 4.849136411748306e-05, "loss": 0.15011250972747803, "memory/device_reserved (GiB)": 42.5, "memory/max_active (GiB)": 39.14, "memory/max_allocated (GiB)": 39.14, "ppl": 1.16196, "step": 292, "tokens/total": 10273952, "tokens/train_per_sec_per_gpu": 64.79, "tokens/trainable": 380164} +{"epoch": 1.14453125, "grad_norm": 2.2145864963531494, "learning_rate": 4.8204263076866574e-05, "loss": 0.154158353805542, "memory/device_reserved (GiB)": 42.5, "memory/max_active (GiB)": 38.93, "memory/max_allocated (GiB)": 38.93, "ppl": 1.16668, "step": 293, "tokens/total": 10310048, "tokens/train_per_sec_per_gpu": 68.38, "tokens/trainable": 381468} +{"epoch": 1.1484375, "grad_norm": 1.3004943132400513, "learning_rate": 4.791744483460251e-05, "loss": 0.16836728155612946, "memory/device_reserved (GiB)": 42.5, "memory/max_active (GiB)": 37.87, "memory/max_allocated (GiB)": 37.87, "ppl": 1.18337, "step": 294, "tokens/total": 10344752, "tokens/train_per_sec_per_gpu": 66.47, "tokens/trainable": 382727} +{"epoch": 1.15234375, "grad_norm": 0.9149438738822937, "learning_rate": 4.7630921326368736e-05, "loss": 0.14920476078987122, "memory/device_reserved (GiB)": 42.5, "memory/max_active (GiB)": 39.39, "memory/max_allocated (GiB)": 39.39, "ppl": 1.16091, "step": 295, "tokens/total": 10383488, "tokens/train_per_sec_per_gpu": 59.36, "tokens/trainable": 383936} +{"epoch": 1.15625, "grad_norm": 0.8690597414970398, "learning_rate": 4.7344704475577916e-05, "loss": 0.14979396760463715, "memory/device_reserved (GiB)": 42.51, "memory/max_active (GiB)": 35.79, "memory/max_allocated (GiB)": 35.79, "ppl": 1.16159, "step": 296, "tokens/total": 10413312, "tokens/train_per_sec_per_gpu": 83.59, "tokens/trainable": 385211} +{"epoch": 1.16015625, "grad_norm": 0.6405810117721558, "learning_rate": 4.705880619288153e-05, "loss": 0.14176830649375916, "memory/device_reserved (GiB)": 42.51, "memory/max_active (GiB)": 38.37, "memory/max_allocated (GiB)": 38.37, "ppl": 1.15231, "step": 297, "tokens/total": 10449664, "tokens/train_per_sec_per_gpu": 69.0, "tokens/trainable": 386478} +{"epoch": 1.1640625, "grad_norm": 0.8698352575302124, "learning_rate": 4.677323837567412e-05, "loss": 0.14852893352508545, "memory/device_reserved (GiB)": 42.51, "memory/max_active (GiB)": 37.23, "memory/max_allocated (GiB)": 37.23, "ppl": 1.16013, "step": 298, "tokens/total": 10482912, "tokens/train_per_sec_per_gpu": 80.83, "tokens/trainable": 387799} +{"epoch": 1.16796875, "grad_norm": 0.9317309260368347, "learning_rate": 4.6488012907598146e-05, "loss": 0.15842661261558533, "memory/device_reserved (GiB)": 42.51, "memory/max_active (GiB)": 36.6, "memory/max_allocated (GiB)": 36.6, "ppl": 1.17167, "step": 299, "tokens/total": 10515728, "tokens/train_per_sec_per_gpu": 75.67, "tokens/trainable": 389047} +{"epoch": 1.171875, "grad_norm": 0.8579882383346558, "learning_rate": 4.620314165804964e-05, "loss": 0.14037516713142395, "memory/device_reserved (GiB)": 42.51, "memory/max_active (GiB)": 38.51, "memory/max_allocated (GiB)": 38.51, "ppl": 1.15071, "step": 300, "tokens/total": 10551856, "tokens/train_per_sec_per_gpu": 73.55, "tokens/trainable": 390409} +{"epoch": 1.17578125, "grad_norm": 1.0190597772598267, "learning_rate": 4.591863648168407e-05, "loss": 0.13114643096923828, "memory/device_reserved (GiB)": 42.51, "memory/max_active (GiB)": 39.32, "memory/max_allocated (GiB)": 39.32, "ppl": 1.14013, "step": 301, "tokens/total": 10586864, "tokens/train_per_sec_per_gpu": 74.6, "tokens/trainable": 391784} +{"epoch": 1.1796875, "grad_norm": 0.8920279145240784, "learning_rate": 4.5634509217923135e-05, "loss": 0.14435923099517822, "memory/device_reserved (GiB)": 42.51, "memory/max_active (GiB)": 38.06, "memory/max_allocated (GiB)": 38.06, "ppl": 1.1553, "step": 302, "tokens/total": 10620656, "tokens/train_per_sec_per_gpu": 78.35, "tokens/trainable": 393096} +{"epoch": 1.18359375, "grad_norm": 0.6680572032928467, "learning_rate": 4.535077169046201e-05, "loss": 0.15096642076969147, "memory/device_reserved (GiB)": 42.51, "memory/max_active (GiB)": 38.1, "memory/max_allocated (GiB)": 38.1, "ppl": 1.16296, "step": 303, "tokens/total": 10657520, "tokens/train_per_sec_per_gpu": 71.43, "tokens/trainable": 394393} +{"epoch": 1.1875, "grad_norm": 1.3716334104537964, "learning_rate": 4.506743570677743e-05, "loss": 0.15917664766311646, "memory/device_reserved (GiB)": 42.51, "memory/max_active (GiB)": 36.44, "memory/max_allocated (GiB)": 36.44, "ppl": 1.17255, "step": 304, "tokens/total": 10688096, "tokens/train_per_sec_per_gpu": 87.03, "tokens/trainable": 395619} +{"epoch": 1.19140625, "grad_norm": 1.458765983581543, "learning_rate": 4.478451305763618e-05, "loss": 0.1828407347202301, "memory/device_reserved (GiB)": 42.51, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.20062, "step": 305, "tokens/total": 10721824, "tokens/train_per_sec_per_gpu": 77.55, "tokens/trainable": 396876} +{"epoch": 1.1953125, "grad_norm": 0.9340510964393616, "learning_rate": 4.450201551660454e-05, "loss": 0.1436268836259842, "memory/device_reserved (GiB)": 42.51, "memory/max_active (GiB)": 36.44, "memory/max_allocated (GiB)": 36.44, "ppl": 1.15445, "step": 306, "tokens/total": 10753088, "tokens/train_per_sec_per_gpu": 82.62, "tokens/trainable": 398160} +{"epoch": 1.19921875, "grad_norm": 1.0068049430847168, "learning_rate": 4.4219954839558276e-05, "loss": 0.1524566262960434, "memory/device_reserved (GiB)": 42.51, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.16469, "step": 307, "tokens/total": 10787968, "tokens/train_per_sec_per_gpu": 71.27, "tokens/trainable": 399474} +{"epoch": 1.203125, "grad_norm": 1.0011820793151855, "learning_rate": 4.393834276419352e-05, "loss": 0.14920341968536377, "memory/device_reserved (GiB)": 42.51, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.16091, "step": 308, "tokens/total": 10820064, "tokens/train_per_sec_per_gpu": 85.93, "tokens/trainable": 400811} +{"epoch": 1.20703125, "grad_norm": 0.9723376035690308, "learning_rate": 4.36571910095382e-05, "loss": 0.1376342475414276, "memory/device_reserved (GiB)": 42.51, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.14756, "step": 309, "tokens/total": 10858672, "tokens/train_per_sec_per_gpu": 67.74, "tokens/trainable": 402096} +{"epoch": 1.2109375, "grad_norm": 0.5995194911956787, "learning_rate": 4.337651127546448e-05, "loss": 0.14333578944206238, "memory/device_reserved (GiB)": 42.51, "memory/max_active (GiB)": 38.93, "memory/max_allocated (GiB)": 38.93, "ppl": 1.15412, "step": 310, "tokens/total": 10894592, "tokens/train_per_sec_per_gpu": 76.57, "tokens/trainable": 403420} +{"epoch": 1.21484375, "grad_norm": 1.2945306301116943, "learning_rate": 4.3096315242201736e-05, "loss": 0.1545303761959076, "memory/device_reserved (GiB)": 42.51, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.16711, "step": 311, "tokens/total": 10927616, "tokens/train_per_sec_per_gpu": 77.18, "tokens/trainable": 404670} +{"epoch": 1.21875, "grad_norm": 0.6613836884498596, "learning_rate": 4.2816614569850635e-05, "loss": 0.14281678199768066, "memory/device_reserved (GiB)": 42.51, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.15352, "step": 312, "tokens/total": 10963680, "tokens/train_per_sec_per_gpu": 70.59, "tokens/trainable": 405898} +{"epoch": 1.22265625, "grad_norm": 1.4738589525222778, "learning_rate": 4.2537420897897864e-05, "loss": 0.1454450488090515, "memory/device_reserved (GiB)": 42.51, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 1.15655, "step": 313, "tokens/total": 10999696, "tokens/train_per_sec_per_gpu": 67.68, "tokens/trainable": 407227} +{"epoch": 1.2265625, "grad_norm": 0.9288666844367981, "learning_rate": 4.225874584473174e-05, "loss": 0.1431882679462433, "memory/device_reserved (GiB)": 42.51, "memory/max_active (GiB)": 37.67, "memory/max_allocated (GiB)": 37.67, "ppl": 1.15395, "step": 314, "tokens/total": 11032560, "tokens/train_per_sec_per_gpu": 80.46, "tokens/trainable": 408501} +{"epoch": 1.23046875, "grad_norm": 0.6558395624160767, "learning_rate": 4.19806010071587e-05, "loss": 0.1280287504196167, "memory/device_reserved (GiB)": 42.51, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.13659, "step": 315, "tokens/total": 11070944, "tokens/train_per_sec_per_gpu": 68.19, "tokens/trainable": 409855} +{"epoch": 1.234375, "grad_norm": 0.8143557906150818, "learning_rate": 4.170299795992081e-05, "loss": 0.13700345158576965, "memory/device_reserved (GiB)": 42.51, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.14683, "step": 316, "tokens/total": 11104656, "tokens/train_per_sec_per_gpu": 75.18, "tokens/trainable": 411134} +{"epoch": 1.23828125, "grad_norm": 0.6962578296661377, "learning_rate": 4.142594825521398e-05, "loss": 0.14003774523735046, "memory/device_reserved (GiB)": 42.51, "memory/max_active (GiB)": 39.32, "memory/max_allocated (GiB)": 39.32, "ppl": 1.15032, "step": 317, "tokens/total": 11144144, "tokens/train_per_sec_per_gpu": 62.16, "tokens/trainable": 412444} +{"epoch": 1.2421875, "grad_norm": 1.708883285522461, "learning_rate": 4.114946342220728e-05, "loss": 0.14000862836837769, "memory/device_reserved (GiB)": 42.51, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.15028, "step": 318, "tokens/total": 11181328, "tokens/train_per_sec_per_gpu": 71.28, "tokens/trainable": 413824} +{"epoch": 1.24609375, "grad_norm": 0.8317289352416992, "learning_rate": 4.087355496656321e-05, "loss": 0.13516362011432648, "memory/device_reserved (GiB)": 42.51, "memory/max_active (GiB)": 37.81, "memory/max_allocated (GiB)": 37.81, "ppl": 1.14472, "step": 319, "tokens/total": 11215216, "tokens/train_per_sec_per_gpu": 81.44, "tokens/trainable": 415188} +{"epoch": 1.25, "grad_norm": 0.876168429851532, "learning_rate": 4.05982343699588e-05, "loss": 0.13965988159179688, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 39.61, "memory/max_allocated (GiB)": 39.61, "ppl": 1.14988, "step": 320, "tokens/total": 11254960, "tokens/train_per_sec_per_gpu": 68.71, "tokens/trainable": 416614} +{"epoch": 1.25390625, "grad_norm": 0.9565391540527344, "learning_rate": 4.0323513089607876e-05, "loss": 0.1411588191986084, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 37.58, "memory/max_allocated (GiB)": 37.58, "ppl": 1.15161, "step": 321, "tokens/total": 11289952, "tokens/train_per_sec_per_gpu": 68.89, "tokens/trainable": 417920} +{"epoch": 1.2578125, "grad_norm": 0.7608044743537903, "learning_rate": 4.004940255778431e-05, "loss": 0.14503754675388336, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 39.79, "memory/max_allocated (GiB)": 39.79, "ppl": 1.15608, "step": 322, "tokens/total": 11329104, "tokens/train_per_sec_per_gpu": 71.4, "tokens/trainable": 419260} +{"epoch": 1.26171875, "grad_norm": 0.9829590916633606, "learning_rate": 3.977591418134619e-05, "loss": 0.14472514390945435, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.15572, "step": 323, "tokens/total": 11364432, "tokens/train_per_sec_per_gpu": 71.38, "tokens/trainable": 420545} +{"epoch": 1.265625, "grad_norm": 6.368468284606934, "learning_rate": 3.95030593412612e-05, "loss": 0.15608619153499603, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 39.6, "memory/max_allocated (GiB)": 39.6, "ppl": 1.16893, "step": 324, "tokens/total": 11400528, "tokens/train_per_sec_per_gpu": 74.01, "tokens/trainable": 421836} +{"epoch": 1.26953125, "grad_norm": 0.5105093717575073, "learning_rate": 3.923084939213296e-05, "loss": 0.14195233583450317, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 35.82, "memory/max_allocated (GiB)": 35.82, "ppl": 1.15252, "step": 325, "tokens/total": 11431312, "tokens/train_per_sec_per_gpu": 78.6, "tokens/trainable": 423077} +{"epoch": 1.2734375, "grad_norm": 0.8095289468765259, "learning_rate": 3.895929566172861e-05, "loss": 0.1322668045759201, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 38.97, "memory/max_allocated (GiB)": 38.97, "ppl": 1.14141, "step": 326, "tokens/total": 11467664, "tokens/train_per_sec_per_gpu": 70.95, "tokens/trainable": 424393} +{"epoch": 1.27734375, "grad_norm": 0.836354672908783, "learning_rate": 3.868840945050728e-05, "loss": 0.12676511704921722, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.13515, "step": 327, "tokens/total": 11506784, "tokens/train_per_sec_per_gpu": 70.23, "tokens/trainable": 425729} +{"epoch": 1.28125, "grad_norm": 0.4966605007648468, "learning_rate": 3.841820203114995e-05, "loss": 0.1311212182044983, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.14011, "step": 328, "tokens/total": 11538784, "tokens/train_per_sec_per_gpu": 81.14, "tokens/trainable": 427075} +{"epoch": 1.28515625, "grad_norm": 1.2961682081222534, "learning_rate": 3.814868464809027e-05, "loss": 0.15232127904891968, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 36.77, "memory/max_allocated (GiB)": 36.77, "ppl": 1.16453, "step": 329, "tokens/total": 11570896, "tokens/train_per_sec_per_gpu": 77.84, "tokens/trainable": 428326} +{"epoch": 1.2890625, "grad_norm": 0.5364113450050354, "learning_rate": 3.787986851704667e-05, "loss": 0.13687613606452942, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.14669, "step": 330, "tokens/total": 11604624, "tokens/train_per_sec_per_gpu": 74.95, "tokens/trainable": 429603} +{"epoch": 1.29296875, "grad_norm": 1.1533490419387817, "learning_rate": 3.7611764824555654e-05, "loss": 0.13769108057022095, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.14762, "step": 331, "tokens/total": 11636448, "tokens/train_per_sec_per_gpu": 92.33, "tokens/trainable": 430999} +{"epoch": 1.296875, "grad_norm": 0.7816288471221924, "learning_rate": 3.734438472750619e-05, "loss": 0.12427697330713272, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 39.71, "memory/max_allocated (GiB)": 39.71, "ppl": 1.13233, "step": 332, "tokens/total": 11676944, "tokens/train_per_sec_per_gpu": 70.69, "tokens/trainable": 432391} +{"epoch": 1.30078125, "grad_norm": 1.718617558479309, "learning_rate": 3.707773935267552e-05, "loss": 0.14138424396514893, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 38.81, "memory/max_allocated (GiB)": 38.81, "ppl": 1.15187, "step": 333, "tokens/total": 11714224, "tokens/train_per_sec_per_gpu": 73.69, "tokens/trainable": 433766} +{"epoch": 1.3046875, "grad_norm": 0.8403233885765076, "learning_rate": 3.68118397962661e-05, "loss": 0.1570655107498169, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 1.17007, "step": 334, "tokens/total": 11749392, "tokens/train_per_sec_per_gpu": 73.29, "tokens/trainable": 435017} +{"epoch": 1.30859375, "grad_norm": 0.5092813372612, "learning_rate": 3.654669712344384e-05, "loss": 0.12676838040351868, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 36.93, "memory/max_allocated (GiB)": 36.93, "ppl": 1.13515, "step": 335, "tokens/total": 11781040, "tokens/train_per_sec_per_gpu": 93.4, "tokens/trainable": 436374} +{"epoch": 1.3125, "grad_norm": 0.6564310193061829, "learning_rate": 3.628232236787763e-05, "loss": 0.12384629249572754, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.13184, "step": 336, "tokens/total": 11816976, "tokens/train_per_sec_per_gpu": 69.86, "tokens/trainable": 437766} +{"epoch": 1.31640625, "grad_norm": 0.6574072241783142, "learning_rate": 3.6018726531280144e-05, "loss": 0.13323593139648438, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 39.17, "memory/max_allocated (GiB)": 39.17, "ppl": 1.14252, "step": 337, "tokens/total": 11854960, "tokens/train_per_sec_per_gpu": 71.02, "tokens/trainable": 439106} +{"epoch": 1.3203125, "grad_norm": 0.9003344774246216, "learning_rate": 3.575592058295017e-05, "loss": 0.15782730281352997, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 37.17, "memory/max_allocated (GiB)": 37.17, "ppl": 1.17096, "step": 338, "tokens/total": 11889296, "tokens/train_per_sec_per_gpu": 75.34, "tokens/trainable": 440400} +{"epoch": 1.32421875, "grad_norm": 0.6952344179153442, "learning_rate": 3.549391545931585e-05, "loss": 0.13467273116111755, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.14416, "step": 339, "tokens/total": 11924160, "tokens/train_per_sec_per_gpu": 80.3, "tokens/trainable": 441735} +{"epoch": 1.328125, "grad_norm": 0.9744158387184143, "learning_rate": 3.5232722063479914e-05, "loss": 0.14078842103481293, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.15118, "step": 340, "tokens/total": 11958752, "tokens/train_per_sec_per_gpu": 74.12, "tokens/trainable": 443048} +{"epoch": 1.33203125, "grad_norm": 0.5566220283508301, "learning_rate": 3.49723512647657e-05, "loss": 0.12626299262046814, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.13458, "step": 341, "tokens/total": 11998336, "tokens/train_per_sec_per_gpu": 67.72, "tokens/trainable": 444443} +{"epoch": 1.3359375, "grad_norm": 0.9396492838859558, "learning_rate": 3.471281389826491e-05, "loss": 0.15285080671310425, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 36.53, "memory/max_allocated (GiB)": 36.53, "ppl": 1.16515, "step": 342, "tokens/total": 12029568, "tokens/train_per_sec_per_gpu": 75.65, "tokens/trainable": 445607} +{"epoch": 1.33984375, "grad_norm": 1.0333991050720215, "learning_rate": 3.4454120764386764e-05, "loss": 0.12853766977787018, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.13716, "step": 343, "tokens/total": 12068304, "tokens/train_per_sec_per_gpu": 67.64, "tokens/trainable": 446918} +{"epoch": 1.34375, "grad_norm": 0.7147982716560364, "learning_rate": 3.4196282628408526e-05, "loss": 0.141919806599617, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.15248, "step": 344, "tokens/total": 12102864, "tokens/train_per_sec_per_gpu": 72.76, "tokens/trainable": 448200} +{"epoch": 1.34765625, "grad_norm": 0.9015696048736572, "learning_rate": 3.3939310220027456e-05, "loss": 0.14548219740390778, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 37.74, "memory/max_allocated (GiB)": 37.74, "ppl": 1.1566, "step": 345, "tokens/total": 12137104, "tokens/train_per_sec_per_gpu": 73.02, "tokens/trainable": 449503} +{"epoch": 1.3515625, "grad_norm": 0.9138256907463074, "learning_rate": 3.3683214232914404e-05, "loss": 0.14698410034179688, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.15834, "step": 346, "tokens/total": 12173072, "tokens/train_per_sec_per_gpu": 72.96, "tokens/trainable": 450845} +{"epoch": 1.35546875, "grad_norm": 0.7645671367645264, "learning_rate": 3.342800532426873e-05, "loss": 0.1378503143787384, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 38.81, "memory/max_allocated (GiB)": 38.81, "ppl": 1.1478, "step": 347, "tokens/total": 12209216, "tokens/train_per_sec_per_gpu": 71.76, "tokens/trainable": 452106} +{"epoch": 1.359375, "grad_norm": 1.1792253255844116, "learning_rate": 3.317369411437484e-05, "loss": 0.14053623378276825, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.15089, "step": 348, "tokens/total": 12245600, "tokens/train_per_sec_per_gpu": 68.87, "tokens/trainable": 453397} +{"epoch": 1.36328125, "grad_norm": 0.8667125105857849, "learning_rate": 3.292029118616024e-05, "loss": 0.14524343609809875, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.15632, "step": 349, "tokens/total": 12278592, "tokens/train_per_sec_per_gpu": 66.67, "tokens/trainable": 454570} +{"epoch": 1.3671875, "grad_norm": 0.8127080202102661, "learning_rate": 3.266780708475511e-05, "loss": 0.12978075444698334, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.13858, "step": 350, "tokens/total": 12314208, "tokens/train_per_sec_per_gpu": 73.65, "tokens/trainable": 455924} +{"epoch": 1.37109375, "grad_norm": 0.6809778213500977, "learning_rate": 3.241625231705354e-05, "loss": 0.12666285037994385, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 38.59, "memory/max_allocated (GiB)": 38.59, "ppl": 1.13503, "step": 351, "tokens/total": 12350640, "tokens/train_per_sec_per_gpu": 76.29, "tokens/trainable": 457304} +{"epoch": 1.375, "grad_norm": 0.4899357557296753, "learning_rate": 3.216563735127618e-05, "loss": 0.12909583747386932, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 39.14, "memory/max_allocated (GiB)": 39.14, "ppl": 1.1378, "step": 352, "tokens/total": 12385872, "tokens/train_per_sec_per_gpu": 74.66, "tokens/trainable": 458690} +{"epoch": 1.37890625, "grad_norm": 0.6576805114746094, "learning_rate": 3.191597261653475e-05, "loss": 0.1416696459054947, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 39.19, "memory/max_allocated (GiB)": 39.19, "ppl": 1.1522, "step": 353, "tokens/total": 12423072, "tokens/train_per_sec_per_gpu": 72.07, "tokens/trainable": 460057} +{"epoch": 1.3828125, "grad_norm": 0.9277812838554382, "learning_rate": 3.166726850239794e-05, "loss": 0.138729527592659, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.14881, "step": 354, "tokens/total": 12456848, "tokens/train_per_sec_per_gpu": 75.32, "tokens/trainable": 461346} +{"epoch": 1.38671875, "grad_norm": 0.861615002155304, "learning_rate": 3.141953535845912e-05, "loss": 0.15472543239593506, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 36.25, "memory/max_allocated (GiB)": 36.25, "ppl": 1.16734, "step": 355, "tokens/total": 12487824, "tokens/train_per_sec_per_gpu": 76.57, "tokens/trainable": 462541} +{"epoch": 1.390625, "grad_norm": 0.8419601917266846, "learning_rate": 3.11727834939056e-05, "loss": 0.1389080286026001, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 36.44, "memory/max_allocated (GiB)": 36.44, "ppl": 1.14902, "step": 356, "tokens/total": 12520016, "tokens/train_per_sec_per_gpu": 75.65, "tokens/trainable": 463852} +{"epoch": 1.39453125, "grad_norm": 0.8832680583000183, "learning_rate": 3.092702317708967e-05, "loss": 0.150570809841156, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 37.19, "memory/max_allocated (GiB)": 37.19, "ppl": 1.1625, "step": 357, "tokens/total": 12554032, "tokens/train_per_sec_per_gpu": 78.26, "tokens/trainable": 465157} +{"epoch": 1.3984375, "grad_norm": 0.6363479495048523, "learning_rate": 3.0682264635101276e-05, "loss": 0.13115209341049194, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 37.53, "memory/max_allocated (GiB)": 37.53, "ppl": 1.14014, "step": 358, "tokens/total": 12586864, "tokens/train_per_sec_per_gpu": 76.64, "tokens/trainable": 466409} +{"epoch": 1.40234375, "grad_norm": 2.4946634769439697, "learning_rate": 3.0438518053342407e-05, "loss": 0.13601750135421753, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 39.11, "memory/max_allocated (GiB)": 39.11, "ppl": 1.1457, "step": 359, "tokens/total": 12626304, "tokens/train_per_sec_per_gpu": 70.81, "tokens/trainable": 467835} +{"epoch": 1.40625, "grad_norm": 0.6856642961502075, "learning_rate": 3.0195793575103266e-05, "loss": 0.14046916365623474, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.15081, "step": 360, "tokens/total": 12665568, "tokens/train_per_sec_per_gpu": 62.02, "tokens/trainable": 469132} +{"epoch": 1.41015625, "grad_norm": 3.3544697761535645, "learning_rate": 2.9954101301140146e-05, "loss": 0.15005716681480408, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 37.73, "memory/max_allocated (GiB)": 37.73, "ppl": 1.1619, "step": 361, "tokens/total": 12701168, "tokens/train_per_sec_per_gpu": 77.84, "tokens/trainable": 470489} +{"epoch": 1.4140625, "grad_norm": 0.673286497592926, "learning_rate": 2.9713451289255123e-05, "loss": 0.1425108015537262, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.15317, "step": 362, "tokens/total": 12730912, "tokens/train_per_sec_per_gpu": 78.29, "tokens/trainable": 471753} +{"epoch": 1.41796875, "grad_norm": 1.3290296792984009, "learning_rate": 2.9473853553877484e-05, "loss": 0.1353914737701416, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.14498, "step": 363, "tokens/total": 12765328, "tokens/train_per_sec_per_gpu": 75.07, "tokens/trainable": 473068} +{"epoch": 1.421875, "grad_norm": 1.481215000152588, "learning_rate": 2.9235318065647e-05, "loss": 0.13309328258037567, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 39.77, "memory/max_allocated (GiB)": 39.77, "ppl": 1.14236, "step": 364, "tokens/total": 12805744, "tokens/train_per_sec_per_gpu": 64.19, "tokens/trainable": 474394} +{"epoch": 1.42578125, "grad_norm": 0.6969475150108337, "learning_rate": 2.8997854750998964e-05, "loss": 0.1359105110168457, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 39.54, "memory/max_allocated (GiB)": 39.54, "ppl": 1.14558, "step": 365, "tokens/total": 12845520, "tokens/train_per_sec_per_gpu": 71.73, "tokens/trainable": 475770} +{"epoch": 1.4296875, "grad_norm": 0.8583161234855652, "learning_rate": 2.8761473491751258e-05, "loss": 0.14391997456550598, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 36.67, "memory/max_allocated (GiB)": 36.67, "ppl": 1.15479, "step": 366, "tokens/total": 12878512, "tokens/train_per_sec_per_gpu": 76.8, "tokens/trainable": 477012} +{"epoch": 1.43359375, "grad_norm": 0.4368099868297577, "learning_rate": 2.8526184124692883e-05, "loss": 0.11950002610683441, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.12693, "step": 367, "tokens/total": 12915600, "tokens/train_per_sec_per_gpu": 63.25, "tokens/trainable": 478340} +{"epoch": 1.4375, "grad_norm": 0.4833533763885498, "learning_rate": 2.829199644117484e-05, "loss": 0.13143989443778992, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.14047, "step": 368, "tokens/total": 12949488, "tokens/train_per_sec_per_gpu": 76.85, "tokens/trainable": 479557} +{"epoch": 1.44140625, "grad_norm": 0.5385683178901672, "learning_rate": 2.8058920186702553e-05, "loss": 0.12744857370853424, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.13593, "step": 369, "tokens/total": 12987680, "tokens/train_per_sec_per_gpu": 70.5, "tokens/trainable": 480886} +{"epoch": 1.4453125, "grad_norm": 0.7599180340766907, "learning_rate": 2.782696506053033e-05, "loss": 0.14427563548088074, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.1552, "step": 370, "tokens/total": 13026304, "tokens/train_per_sec_per_gpu": 67.67, "tokens/trainable": 482138} +{"epoch": 1.44921875, "grad_norm": 0.5369629859924316, "learning_rate": 2.7596140715257824e-05, "loss": 0.13772286474704742, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.14766, "step": 371, "tokens/total": 13061792, "tokens/train_per_sec_per_gpu": 72.18, "tokens/trainable": 483465} +{"epoch": 1.453125, "grad_norm": 0.5607120990753174, "learning_rate": 2.7366456756428184e-05, "loss": 0.13350850343704224, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 37.45, "memory/max_allocated (GiB)": 37.45, "ppl": 1.14283, "step": 372, "tokens/total": 13096528, "tokens/train_per_sec_per_gpu": 73.14, "tokens/trainable": 484773} +{"epoch": 1.45703125, "grad_norm": 0.7921921014785767, "learning_rate": 2.7137922742128486e-05, "loss": 0.12888826429843903, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 1.13756, "step": 373, "tokens/total": 13131856, "tokens/train_per_sec_per_gpu": 75.17, "tokens/trainable": 486125} +{"epoch": 1.4609375, "grad_norm": 0.7008119821548462, "learning_rate": 2.691054818259188e-05, "loss": 0.12940306961536407, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 37.59, "memory/max_allocated (GiB)": 37.59, "ppl": 1.13815, "step": 374, "tokens/total": 13166992, "tokens/train_per_sec_per_gpu": 79.01, "tokens/trainable": 487531} +{"epoch": 1.46484375, "grad_norm": 0.5931722521781921, "learning_rate": 2.6684342539801933e-05, "loss": 0.15060493350028992, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 36.59, "memory/max_allocated (GiB)": 36.59, "ppl": 1.16254, "step": 375, "tokens/total": 13198480, "tokens/train_per_sec_per_gpu": 76.57, "tokens/trainable": 488792} +{"epoch": 1.46875, "grad_norm": 0.6226785182952881, "learning_rate": 2.645931522709877e-05, "loss": 0.13671009242534637, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 39.59, "memory/max_allocated (GiB)": 39.59, "ppl": 1.1465, "step": 376, "tokens/total": 13234320, "tokens/train_per_sec_per_gpu": 86.32, "tokens/trainable": 490185} +{"epoch": 1.47265625, "grad_norm": 0.731418251991272, "learning_rate": 2.6235475608787365e-05, "loss": 0.14313606917858124, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 38.86, "memory/max_allocated (GiB)": 38.86, "ppl": 1.15389, "step": 377, "tokens/total": 13270640, "tokens/train_per_sec_per_gpu": 70.26, "tokens/trainable": 491464} +{"epoch": 1.4765625, "grad_norm": 0.5860029458999634, "learning_rate": 2.6012832999747916e-05, "loss": 0.13815800845623016, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.14816, "step": 378, "tokens/total": 13301728, "tokens/train_per_sec_per_gpu": 77.61, "tokens/trainable": 492715} +{"epoch": 1.48046875, "grad_norm": 0.7965688109397888, "learning_rate": 2.579139666504821e-05, "loss": 0.14527665078639984, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.15636, "step": 379, "tokens/total": 13342048, "tokens/train_per_sec_per_gpu": 62.94, "tokens/trainable": 493984} +{"epoch": 1.484375, "grad_norm": 0.6211039423942566, "learning_rate": 2.557117581955798e-05, "loss": 0.1317204236984253, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.14079, "step": 380, "tokens/total": 13376048, "tokens/train_per_sec_per_gpu": 72.03, "tokens/trainable": 495317} +{"epoch": 1.48828125, "grad_norm": 0.48298215866088867, "learning_rate": 2.5352179627565532e-05, "loss": 0.13202694058418274, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 38.22, "memory/max_allocated (GiB)": 38.22, "ppl": 1.14114, "step": 381, "tokens/total": 13411488, "tokens/train_per_sec_per_gpu": 67.81, "tokens/trainable": 496600} +{"epoch": 1.4921875, "grad_norm": 0.6204062700271606, "learning_rate": 2.5134417202396277e-05, "loss": 0.14669179916381836, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.158, "step": 382, "tokens/total": 13445472, "tokens/train_per_sec_per_gpu": 72.77, "tokens/trainable": 497846} +{"epoch": 1.49609375, "grad_norm": 1.424261212348938, "learning_rate": 2.491789760603361e-05, "loss": 0.14026489853858948, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 39.23, "memory/max_allocated (GiB)": 39.23, "ppl": 1.15058, "step": 383, "tokens/total": 13482304, "tokens/train_per_sec_per_gpu": 68.04, "tokens/trainable": 499135} +{"epoch": 1.5, "grad_norm": 1.0730741024017334, "learning_rate": 2.4702629848741764e-05, "loss": 0.1471850872039795, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 37.82, "memory/max_allocated (GiB)": 37.82, "ppl": 1.15857, "step": 384, "tokens/total": 13517168, "tokens/train_per_sec_per_gpu": 74.91, "tokens/trainable": 500387} +{"epoch": 1.50390625, "grad_norm": 0.5465441346168518, "learning_rate": 2.4488622888690785e-05, "loss": 0.1352328062057495, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 37.06, "memory/max_allocated (GiB)": 37.06, "ppl": 1.1448, "step": 385, "tokens/total": 13550912, "tokens/train_per_sec_per_gpu": 71.98, "tokens/trainable": 501617} +{"epoch": 1.5078125, "grad_norm": 0.49530693888664246, "learning_rate": 2.427588563158384e-05, "loss": 0.12790359556674957, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.13644, "step": 386, "tokens/total": 13585456, "tokens/train_per_sec_per_gpu": 75.81, "tokens/trainable": 502949} +{"epoch": 1.51171875, "grad_norm": 0.548003613948822, "learning_rate": 2.406442693028651e-05, "loss": 0.13732536137104034, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 38.63, "memory/max_allocated (GiB)": 38.63, "ppl": 1.1472, "step": 387, "tokens/total": 13618128, "tokens/train_per_sec_per_gpu": 74.95, "tokens/trainable": 504186} +{"epoch": 1.515625, "grad_norm": 0.6003954410552979, "learning_rate": 2.3854255584458547e-05, "loss": 0.1387770175933838, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.14887, "step": 388, "tokens/total": 13657136, "tokens/train_per_sec_per_gpu": 60.94, "tokens/trainable": 505398} +{"epoch": 1.51953125, "grad_norm": 0.6066699624061584, "learning_rate": 2.3645380340187508e-05, "loss": 0.13456428050994873, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 39.19, "memory/max_allocated (GiB)": 39.19, "ppl": 1.14404, "step": 389, "tokens/total": 13692944, "tokens/train_per_sec_per_gpu": 74.32, "tokens/trainable": 506720} +{"epoch": 1.5234375, "grad_norm": 0.6476549506187439, "learning_rate": 2.3437809889624914e-05, "loss": 0.15029221773147583, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.16217, "step": 390, "tokens/total": 13725760, "tokens/train_per_sec_per_gpu": 84.1, "tokens/trainable": 508001} +{"epoch": 1.52734375, "grad_norm": 0.5960601568222046, "learning_rate": 2.3231552870624487e-05, "loss": 0.1333925724029541, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.1427, "step": 391, "tokens/total": 13757056, "tokens/train_per_sec_per_gpu": 86.48, "tokens/trainable": 509351} +{"epoch": 1.53125, "grad_norm": 0.4192653298377991, "learning_rate": 2.3026617866382657e-05, "loss": 0.12054561823606491, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 36.73, "memory/max_allocated (GiB)": 36.73, "ppl": 1.12811, "step": 392, "tokens/total": 13790224, "tokens/train_per_sec_per_gpu": 80.5, "tokens/trainable": 510702} +{"epoch": 1.53515625, "grad_norm": 0.5776291489601135, "learning_rate": 2.2823013405081507e-05, "loss": 0.12282488495111465, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.13069, "step": 393, "tokens/total": 13828656, "tokens/train_per_sec_per_gpu": 68.92, "tokens/trainable": 512053} +{"epoch": 1.5390625, "grad_norm": 0.6754850745201111, "learning_rate": 2.2620747959533722e-05, "loss": 0.1190691813826561, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.12645, "step": 394, "tokens/total": 13862304, "tokens/train_per_sec_per_gpu": 82.84, "tokens/trainable": 513463} +{"epoch": 1.54296875, "grad_norm": 0.5501860976219177, "learning_rate": 2.2419829946830123e-05, "loss": 0.13572296500205994, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 38.05, "memory/max_allocated (GiB)": 38.05, "ppl": 1.14536, "step": 395, "tokens/total": 13898496, "tokens/train_per_sec_per_gpu": 72.32, "tokens/trainable": 514756} +{"epoch": 1.546875, "grad_norm": 1.205278754234314, "learning_rate": 2.2220267727989325e-05, "loss": 0.12549039721488953, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 38.0, "memory/max_allocated (GiB)": 38.0, "ppl": 1.1337, "step": 396, "tokens/total": 13933408, "tokens/train_per_sec_per_gpu": 73.93, "tokens/trainable": 516093} +{"epoch": 1.55078125, "grad_norm": 0.5573631525039673, "learning_rate": 2.202206960760984e-05, "loss": 0.11815917491912842, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.12542, "step": 397, "tokens/total": 13971936, "tokens/train_per_sec_per_gpu": 67.49, "tokens/trainable": 517480} +{"epoch": 1.5546875, "grad_norm": 0.507695198059082, "learning_rate": 2.182524383352446e-05, "loss": 0.1343977302312851, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 38.02, "memory/max_allocated (GiB)": 38.02, "ppl": 1.14385, "step": 398, "tokens/total": 14008288, "tokens/train_per_sec_per_gpu": 70.43, "tokens/trainable": 518761} +{"epoch": 1.55859375, "grad_norm": 1.2139441967010498, "learning_rate": 2.1629798596457056e-05, "loss": 0.13859984278678894, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 36.53, "memory/max_allocated (GiB)": 36.53, "ppl": 1.14866, "step": 399, "tokens/total": 14040736, "tokens/train_per_sec_per_gpu": 80.61, "tokens/trainable": 520040} +{"epoch": 1.5625, "grad_norm": 0.5837975740432739, "learning_rate": 2.1435742029681725e-05, "loss": 0.13448114693164825, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.14394, "step": 400, "tokens/total": 14073376, "tokens/train_per_sec_per_gpu": 85.43, "tokens/trainable": 521338} +{"epoch": 1.56640625, "grad_norm": 0.8541660308837891, "learning_rate": 2.124308220868431e-05, "loss": 0.12131446599960327, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.12898, "step": 401, "tokens/total": 14109616, "tokens/train_per_sec_per_gpu": 71.67, "tokens/trainable": 522704} +{"epoch": 1.5703125, "grad_norm": 0.7260363101959229, "learning_rate": 2.105182715082638e-05, "loss": 0.1301516443490982, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.139, "step": 402, "tokens/total": 14143728, "tokens/train_per_sec_per_gpu": 71.96, "tokens/trainable": 524032} +{"epoch": 1.57421875, "grad_norm": 0.5633863806724548, "learning_rate": 2.0861984815011552e-05, "loss": 0.14218221604824066, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 38.05, "memory/max_allocated (GiB)": 38.05, "ppl": 1.15279, "step": 403, "tokens/total": 14179184, "tokens/train_per_sec_per_gpu": 69.12, "tokens/trainable": 525266} +{"epoch": 1.578125, "grad_norm": 0.46853870153427124, "learning_rate": 2.0673563101354323e-05, "loss": 0.1331675946712494, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.14244, "step": 404, "tokens/total": 14214608, "tokens/train_per_sec_per_gpu": 66.58, "tokens/trainable": 526512} +{"epoch": 1.58203125, "grad_norm": 0.6080860495567322, "learning_rate": 2.0486569850851317e-05, "loss": 0.13342314958572388, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 38.46, "memory/max_allocated (GiB)": 38.46, "ppl": 1.14273, "step": 405, "tokens/total": 14249792, "tokens/train_per_sec_per_gpu": 73.88, "tokens/trainable": 527819} +{"epoch": 1.5859375, "grad_norm": 0.6106833815574646, "learning_rate": 2.0301012845054956e-05, "loss": 0.13435548543930054, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.1438, "step": 406, "tokens/total": 14288816, "tokens/train_per_sec_per_gpu": 65.12, "tokens/trainable": 529134} +{"epoch": 1.58984375, "grad_norm": 0.8363726139068604, "learning_rate": 2.011689980574966e-05, "loss": 0.12850409746170044, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 38.22, "memory/max_allocated (GiB)": 38.22, "ppl": 1.13713, "step": 407, "tokens/total": 14325392, "tokens/train_per_sec_per_gpu": 67.08, "tokens/trainable": 530377} +{"epoch": 1.59375, "grad_norm": 0.6943662166595459, "learning_rate": 1.993423839463052e-05, "loss": 0.13686764240264893, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.14668, "step": 408, "tokens/total": 14359088, "tokens/train_per_sec_per_gpu": 87.61, "tokens/trainable": 531694} +{"epoch": 1.59765625, "grad_norm": 0.5260400176048279, "learning_rate": 1.975303621298445e-05, "loss": 0.12559494376182556, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.13382, "step": 409, "tokens/total": 14396176, "tokens/train_per_sec_per_gpu": 74.91, "tokens/trainable": 533005} +{"epoch": 1.6015625, "grad_norm": 0.7367598414421082, "learning_rate": 1.957330080137385e-05, "loss": 0.13624168932437897, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 36.28, "memory/max_allocated (GiB)": 36.28, "ppl": 1.14596, "step": 410, "tokens/total": 14427072, "tokens/train_per_sec_per_gpu": 77.51, "tokens/trainable": 534253} +{"epoch": 1.60546875, "grad_norm": 0.5223545432090759, "learning_rate": 1.9395039639322864e-05, "loss": 0.12825322151184082, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.13684, "step": 411, "tokens/total": 14463536, "tokens/train_per_sec_per_gpu": 71.41, "tokens/trainable": 535559} +{"epoch": 1.609375, "grad_norm": 0.615230917930603, "learning_rate": 1.9218260145006073e-05, "loss": 0.12776772677898407, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 37.97, "memory/max_allocated (GiB)": 37.97, "ppl": 1.13629, "step": 412, "tokens/total": 14500048, "tokens/train_per_sec_per_gpu": 75.8, "tokens/trainable": 536961} +{"epoch": 1.61328125, "grad_norm": 0.4269959032535553, "learning_rate": 1.904296967493982e-05, "loss": 0.11714155226945877, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.12428, "step": 413, "tokens/total": 14532752, "tokens/train_per_sec_per_gpu": 85.27, "tokens/trainable": 538299} +{"epoch": 1.6171875, "grad_norm": 0.7549477219581604, "learning_rate": 1.8869175523676064e-05, "loss": 0.12044109404087067, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.12799, "step": 414, "tokens/total": 14569824, "tokens/train_per_sec_per_gpu": 74.86, "tokens/trainable": 539677} +{"epoch": 1.62109375, "grad_norm": 0.4611041843891144, "learning_rate": 1.869688492349885e-05, "loss": 0.12417486310005188, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.13221, "step": 415, "tokens/total": 14603584, "tokens/train_per_sec_per_gpu": 80.06, "tokens/trainable": 540995} +{"epoch": 1.625, "grad_norm": 0.6625611186027527, "learning_rate": 1.85261050441233e-05, "loss": 0.14108705520629883, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 38.02, "memory/max_allocated (GiB)": 38.02, "ppl": 1.15152, "step": 416, "tokens/total": 14639952, "tokens/train_per_sec_per_gpu": 72.67, "tokens/trainable": 542311} +{"epoch": 1.62890625, "grad_norm": 0.6725922226905823, "learning_rate": 1.8356842992397304e-05, "loss": 0.143440842628479, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.15424, "step": 417, "tokens/total": 14674112, "tokens/train_per_sec_per_gpu": 75.21, "tokens/trainable": 543588} +{"epoch": 1.6328125, "grad_norm": 0.6036882400512695, "learning_rate": 1.8189105812005714e-05, "loss": 0.14494621753692627, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 1.15598, "step": 418, "tokens/total": 14711104, "tokens/train_per_sec_per_gpu": 63.36, "tokens/trainable": 544784} +{"epoch": 1.63671875, "grad_norm": 0.609719455242157, "learning_rate": 1.802290048317732e-05, "loss": 0.13151422142982483, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 37.96, "memory/max_allocated (GiB)": 37.96, "ppl": 1.14055, "step": 419, "tokens/total": 14744912, "tokens/train_per_sec_per_gpu": 76.86, "tokens/trainable": 546073} +{"epoch": 1.640625, "grad_norm": 0.4363185167312622, "learning_rate": 1.785823392239424e-05, "loss": 0.12318241596221924, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.13109, "step": 420, "tokens/total": 14778464, "tokens/train_per_sec_per_gpu": 73.95, "tokens/trainable": 547319} +{"epoch": 1.64453125, "grad_norm": 0.9977975487709045, "learning_rate": 1.7695112982104225e-05, "loss": 0.13049577176570892, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 39.8, "memory/max_allocated (GiB)": 39.8, "ppl": 1.13939, "step": 421, "tokens/total": 14813888, "tokens/train_per_sec_per_gpu": 73.14, "tokens/trainable": 548635} +{"epoch": 1.6484375, "grad_norm": 13.371944427490234, "learning_rate": 1.7533544450435433e-05, "loss": 0.14055564999580383, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 35.81, "memory/max_allocated (GiB)": 35.81, "ppl": 1.15091, "step": 422, "tokens/total": 14843600, "tokens/train_per_sec_per_gpu": 88.97, "tokens/trainable": 549907} +{"epoch": 1.65234375, "grad_norm": 0.6633148789405823, "learning_rate": 1.7373535050913946e-05, "loss": 0.1252661794424057, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.13345, "step": 423, "tokens/total": 14880336, "tokens/train_per_sec_per_gpu": 67.02, "tokens/trainable": 551235} +{"epoch": 1.65625, "grad_norm": 0.5417026877403259, "learning_rate": 1.721509144218405e-05, "loss": 0.135005384683609, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 35.88, "memory/max_allocated (GiB)": 35.88, "ppl": 1.14454, "step": 424, "tokens/total": 14910880, "tokens/train_per_sec_per_gpu": 86.51, "tokens/trainable": 552499} +{"epoch": 1.66015625, "grad_norm": 0.49034634232521057, "learning_rate": 1.705822021773101e-05, "loss": 0.12543894350528717, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.13365, "step": 425, "tokens/total": 14945616, "tokens/train_per_sec_per_gpu": 77.59, "tokens/trainable": 553830} +{"epoch": 1.6640625, "grad_norm": 0.7600298523902893, "learning_rate": 1.69029279056068e-05, "loss": 0.13434895873069763, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 39.09, "memory/max_allocated (GiB)": 39.09, "ppl": 1.14379, "step": 426, "tokens/total": 14983328, "tokens/train_per_sec_per_gpu": 68.56, "tokens/trainable": 555132} +{"epoch": 1.66796875, "grad_norm": 3.902756452560425, "learning_rate": 1.6749220968158415e-05, "loss": 0.11592879146337509, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.12292, "step": 427, "tokens/total": 15018656, "tokens/train_per_sec_per_gpu": 81.19, "tokens/trainable": 556520} +{"epoch": 1.671875, "grad_norm": 0.5363274812698364, "learning_rate": 1.659710580175893e-05, "loss": 0.12915948033332825, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 35.98, "memory/max_allocated (GiB)": 35.98, "ppl": 1.13787, "step": 428, "tokens/total": 15049776, "tokens/train_per_sec_per_gpu": 82.64, "tokens/trainable": 557819} +{"epoch": 1.67578125, "grad_norm": 0.48189786076545715, "learning_rate": 1.644658873654133e-05, "loss": 0.12104984372854233, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.12868, "step": 429, "tokens/total": 15082464, "tokens/train_per_sec_per_gpu": 77.87, "tokens/trainable": 559079} +{"epoch": 1.6796875, "grad_norm": 0.5471287369728088, "learning_rate": 1.629767603613508e-05, "loss": 0.14147181808948517, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 36.28, "memory/max_allocated (GiB)": 36.28, "ppl": 1.15197, "step": 430, "tokens/total": 15113600, "tokens/train_per_sec_per_gpu": 75.87, "tokens/trainable": 560253} +{"epoch": 1.68359375, "grad_norm": 0.4735467731952667, "learning_rate": 1.615037389740547e-05, "loss": 0.14023776352405548, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.15055, "step": 431, "tokens/total": 15148928, "tokens/train_per_sec_per_gpu": 69.54, "tokens/trainable": 561515} +{"epoch": 1.6875, "grad_norm": 0.5075342655181885, "learning_rate": 1.600468845019576e-05, "loss": 0.1224583089351654, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 38.97, "memory/max_allocated (GiB)": 38.97, "ppl": 1.13027, "step": 432, "tokens/total": 15187712, "tokens/train_per_sec_per_gpu": 62.46, "tokens/trainable": 562821} +{"epoch": 1.69140625, "grad_norm": 0.6551119089126587, "learning_rate": 1.5860625757072092e-05, "loss": 0.13855934143066406, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.14862, "step": 433, "tokens/total": 15222976, "tokens/train_per_sec_per_gpu": 70.8, "tokens/trainable": 564084} +{"epoch": 1.6953125, "grad_norm": 3.2072458267211914, "learning_rate": 1.571819181307116e-05, "loss": 0.13496698439121246, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 38.51, "memory/max_allocated (GiB)": 38.51, "ppl": 1.1445, "step": 434, "tokens/total": 15260288, "tokens/train_per_sec_per_gpu": 74.57, "tokens/trainable": 565424} +{"epoch": 1.69921875, "grad_norm": 0.4350586533546448, "learning_rate": 1.557739254545075e-05, "loss": 0.12104185670614243, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 36.65, "memory/max_allocated (GiB)": 36.65, "ppl": 1.12867, "step": 435, "tokens/total": 15291808, "tokens/train_per_sec_per_gpu": 86.03, "tokens/trainable": 566756} +{"epoch": 1.703125, "grad_norm": 0.6764931678771973, "learning_rate": 1.543823381344311e-05, "loss": 0.13500413298606873, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 38.1, "memory/max_allocated (GiB)": 38.1, "ppl": 1.14454, "step": 436, "tokens/total": 15324768, "tokens/train_per_sec_per_gpu": 84.35, "tokens/trainable": 568065} +{"epoch": 1.70703125, "grad_norm": 0.49927371740341187, "learning_rate": 1.5300721408011114e-05, "loss": 0.12970101833343506, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.13849, "step": 437, "tokens/total": 15360496, "tokens/train_per_sec_per_gpu": 73.16, "tokens/trainable": 569342} +{"epoch": 1.7109375, "grad_norm": 0.6433137059211731, "learning_rate": 1.5164861051607254e-05, "loss": 0.14255037903785706, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.15321, "step": 438, "tokens/total": 15396912, "tokens/train_per_sec_per_gpu": 62.24, "tokens/trainable": 570518} +{"epoch": 1.71484375, "grad_norm": 0.6236870288848877, "learning_rate": 1.5030658397935521e-05, "loss": 0.131900817155838, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 38.34, "memory/max_allocated (GiB)": 38.34, "ppl": 1.141, "step": 439, "tokens/total": 15432656, "tokens/train_per_sec_per_gpu": 70.27, "tokens/trainable": 571825} +{"epoch": 1.71875, "grad_norm": 0.5180482268333435, "learning_rate": 1.4898119031716104e-05, "loss": 0.12070001661777496, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 38.1, "memory/max_allocated (GiB)": 38.1, "ppl": 1.12829, "step": 440, "tokens/total": 15465984, "tokens/train_per_sec_per_gpu": 81.66, "tokens/trainable": 573157} +{"epoch": 1.72265625, "grad_norm": 0.52855384349823, "learning_rate": 1.476724846845306e-05, "loss": 0.1302456110715866, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.13911, "step": 441, "tokens/total": 15504400, "tokens/train_per_sec_per_gpu": 67.0, "tokens/trainable": 574504} +{"epoch": 1.7265625, "grad_norm": 0.5078102946281433, "learning_rate": 1.463805215420471e-05, "loss": 0.1308291256427765, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 39.8, "memory/max_allocated (GiB)": 39.8, "ppl": 1.13977, "step": 442, "tokens/total": 15540288, "tokens/train_per_sec_per_gpu": 73.04, "tokens/trainable": 575819} +{"epoch": 1.73046875, "grad_norm": 0.6554214358329773, "learning_rate": 1.451053546535705e-05, "loss": 0.12996473908424377, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 39.14, "memory/max_allocated (GiB)": 39.14, "ppl": 1.13879, "step": 443, "tokens/total": 15577632, "tokens/train_per_sec_per_gpu": 69.15, "tokens/trainable": 577117} +{"epoch": 1.734375, "grad_norm": 2.1971311569213867, "learning_rate": 1.438470370840001e-05, "loss": 0.12212908267974854, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 38.14, "memory/max_allocated (GiB)": 38.14, "ppl": 1.1299, "step": 444, "tokens/total": 15614400, "tokens/train_per_sec_per_gpu": 69.21, "tokens/trainable": 578441} +{"epoch": 1.73828125, "grad_norm": 0.5205168128013611, "learning_rate": 1.4260562119706606e-05, "loss": 0.12524500489234924, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.13343, "step": 445, "tokens/total": 15647136, "tokens/train_per_sec_per_gpu": 82.06, "tokens/trainable": 579731} +{"epoch": 1.7421875, "grad_norm": 0.6374572515487671, "learning_rate": 1.413811586531508e-05, "loss": 0.12725555896759033, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 38.1, "memory/max_allocated (GiB)": 38.1, "ppl": 1.13571, "step": 446, "tokens/total": 15680800, "tokens/train_per_sec_per_gpu": 82.78, "tokens/trainable": 581057} +{"epoch": 1.74609375, "grad_norm": 0.6968374848365784, "learning_rate": 1.4017370040713884e-05, "loss": 0.13894891738891602, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 38.39, "memory/max_allocated (GiB)": 38.39, "ppl": 1.14907, "step": 447, "tokens/total": 15718240, "tokens/train_per_sec_per_gpu": 66.44, "tokens/trainable": 582330} +{"epoch": 1.75, "grad_norm": 0.5448214411735535, "learning_rate": 1.3898329670629645e-05, "loss": 0.1302844136953354, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 39.87, "memory/max_allocated (GiB)": 39.87, "ppl": 1.13915, "step": 448, "tokens/total": 15759200, "tokens/train_per_sec_per_gpu": 62.69, "tokens/trainable": 583658} +{"epoch": 1.75390625, "grad_norm": 0.5710135698318481, "learning_rate": 1.3780999708818058e-05, "loss": 0.1291506290435791, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 36.49, "memory/max_allocated (GiB)": 36.49, "ppl": 1.13786, "step": 449, "tokens/total": 15790688, "tokens/train_per_sec_per_gpu": 80.7, "tokens/trainable": 584985} +{"epoch": 1.7578125, "grad_norm": 0.4787259101867676, "learning_rate": 1.3665385037857758e-05, "loss": 0.12356184422969818, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.13152, "step": 450, "tokens/total": 15824112, "tokens/train_per_sec_per_gpu": 76.49, "tokens/trainable": 586278} +{"epoch": 1.76171875, "grad_norm": 0.44062352180480957, "learning_rate": 1.3551490468947126e-05, "loss": 0.12841977179050446, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 36.59, "memory/max_allocated (GiB)": 36.59, "ppl": 1.13703, "step": 451, "tokens/total": 15856352, "tokens/train_per_sec_per_gpu": 83.87, "tokens/trainable": 587562} +{"epoch": 1.765625, "grad_norm": 1.5811101198196411, "learning_rate": 1.3439320741704075e-05, "loss": 0.15187858045101166, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 39.19, "memory/max_allocated (GiB)": 39.19, "ppl": 1.16402, "step": 452, "tokens/total": 15891600, "tokens/train_per_sec_per_gpu": 78.72, "tokens/trainable": 588890} +{"epoch": 1.76953125, "grad_norm": 0.4832656979560852, "learning_rate": 1.3328880523968808e-05, "loss": 0.12849533557891846, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 37.15, "memory/max_allocated (GiB)": 37.15, "ppl": 1.13712, "step": 453, "tokens/total": 15925312, "tokens/train_per_sec_per_gpu": 76.42, "tokens/trainable": 590156} +{"epoch": 1.7734375, "grad_norm": 0.5205655694007874, "learning_rate": 1.3220174411609587e-05, "loss": 0.12536421418190002, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 37.44, "memory/max_allocated (GiB)": 37.44, "ppl": 1.13356, "step": 454, "tokens/total": 15959344, "tokens/train_per_sec_per_gpu": 79.18, "tokens/trainable": 591500} +{"epoch": 1.77734375, "grad_norm": 0.4337303042411804, "learning_rate": 1.3113206928331471e-05, "loss": 0.1282196044921875, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 35.29, "memory/max_allocated (GiB)": 35.29, "ppl": 1.1368, "step": 455, "tokens/total": 15988048, "tokens/train_per_sec_per_gpu": 87.15, "tokens/trainable": 592776} +{"epoch": 1.78125, "grad_norm": 0.5846937298774719, "learning_rate": 1.300798252548806e-05, "loss": 0.12459622323513031, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 37.82, "memory/max_allocated (GiB)": 37.82, "ppl": 1.13269, "step": 456, "tokens/total": 16022976, "tokens/train_per_sec_per_gpu": 74.51, "tokens/trainable": 594060} +{"epoch": 1.78515625, "grad_norm": 0.5531560182571411, "learning_rate": 1.2904505581896265e-05, "loss": 0.13205809891223907, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.14117, "step": 457, "tokens/total": 16062592, "tokens/train_per_sec_per_gpu": 64.19, "tokens/trainable": 595416} +{"epoch": 1.7890625, "grad_norm": 0.5580106377601624, "learning_rate": 1.2802780403654082e-05, "loss": 0.13109642267227173, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.14008, "step": 458, "tokens/total": 16096432, "tokens/train_per_sec_per_gpu": 78.94, "tokens/trainable": 596664} +{"epoch": 1.79296875, "grad_norm": 1.3873647451400757, "learning_rate": 1.2702811223961408e-05, "loss": 0.11826895922422409, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.12555, "step": 459, "tokens/total": 16130560, "tokens/train_per_sec_per_gpu": 80.2, "tokens/trainable": 598026} +{"epoch": 1.796875, "grad_norm": 0.4326072931289673, "learning_rate": 1.2604602202943861e-05, "loss": 0.1261569857597351, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 37.06, "memory/max_allocated (GiB)": 37.06, "ppl": 1.13446, "step": 460, "tokens/total": 16161872, "tokens/train_per_sec_per_gpu": 86.3, "tokens/trainable": 599368} +{"epoch": 1.80078125, "grad_norm": 0.43273380398750305, "learning_rate": 1.2508157427479686e-05, "loss": 0.12014153599739075, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 37.37, "memory/max_allocated (GiB)": 37.37, "ppl": 1.12766, "step": 461, "tokens/total": 16196272, "tokens/train_per_sec_per_gpu": 77.13, "tokens/trainable": 600691} +{"epoch": 1.8046875, "grad_norm": 0.9083741307258606, "learning_rate": 1.2413480911029655e-05, "loss": 0.1311834454536438, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 36.03, "memory/max_allocated (GiB)": 36.03, "ppl": 1.14018, "step": 462, "tokens/total": 16227408, "tokens/train_per_sec_per_gpu": 75.11, "tokens/trainable": 601903} +{"epoch": 1.80859375, "grad_norm": 1.0370392799377441, "learning_rate": 1.2320576593470082e-05, "loss": 0.1381734311580658, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.14817, "step": 463, "tokens/total": 16262304, "tokens/train_per_sec_per_gpu": 67.76, "tokens/trainable": 603144} +{"epoch": 1.8125, "grad_norm": 0.5398375391960144, "learning_rate": 1.2229448340928828e-05, "loss": 0.122418612241745, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 39.3, "memory/max_allocated (GiB)": 39.3, "ppl": 1.13023, "step": 464, "tokens/total": 16301984, "tokens/train_per_sec_per_gpu": 63.14, "tokens/trainable": 604518} +{"epoch": 1.81640625, "grad_norm": 0.911462664604187, "learning_rate": 1.2140099945624458e-05, "loss": 0.12578442692756653, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.13404, "step": 465, "tokens/total": 16341232, "tokens/train_per_sec_per_gpu": 66.13, "tokens/trainable": 605853} +{"epoch": 1.8203125, "grad_norm": 0.653563380241394, "learning_rate": 1.205253512570841e-05, "loss": 0.13370636105537415, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 37.95, "memory/max_allocated (GiB)": 37.95, "ppl": 1.14306, "step": 466, "tokens/total": 16374304, "tokens/train_per_sec_per_gpu": 75.88, "tokens/trainable": 607098} +{"epoch": 1.82421875, "grad_norm": 0.5035116076469421, "learning_rate": 1.1966757525110255e-05, "loss": 0.13535431027412415, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 35.48, "memory/max_allocated (GiB)": 35.48, "ppl": 1.14494, "step": 467, "tokens/total": 16403968, "tokens/train_per_sec_per_gpu": 86.77, "tokens/trainable": 608353} +{"epoch": 1.828125, "grad_norm": 0.534511387348175, "learning_rate": 1.1882770713386095e-05, "loss": 0.1222565621137619, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.13004, "step": 468, "tokens/total": 16440880, "tokens/train_per_sec_per_gpu": 71.92, "tokens/trainable": 609691} +{"epoch": 1.83203125, "grad_norm": 0.43303361535072327, "learning_rate": 1.180057818556998e-05, "loss": 0.1208634302020073, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.12847, "step": 469, "tokens/total": 16477248, "tokens/train_per_sec_per_gpu": 71.26, "tokens/trainable": 611062} +{"epoch": 1.8359375, "grad_norm": 0.9061545133590698, "learning_rate": 1.1720183362028494e-05, "loss": 0.1433592438697815, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 38.63, "memory/max_allocated (GiB)": 38.63, "ppl": 1.15414, "step": 470, "tokens/total": 16513936, "tokens/train_per_sec_per_gpu": 66.99, "tokens/trainable": 612320} +{"epoch": 1.83984375, "grad_norm": 0.5958028435707092, "learning_rate": 1.1641589588318387e-05, "loss": 0.12714868783950806, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.13559, "step": 471, "tokens/total": 16551984, "tokens/train_per_sec_per_gpu": 64.45, "tokens/trainable": 613584} +{"epoch": 1.84375, "grad_norm": 0.5489043593406677, "learning_rate": 1.1564800135047418e-05, "loss": 0.12973695993423462, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.13853, "step": 472, "tokens/total": 16585712, "tokens/train_per_sec_per_gpu": 74.91, "tokens/trainable": 614882} +{"epoch": 1.84765625, "grad_norm": 0.4346758723258972, "learning_rate": 1.148981819773816e-05, "loss": 0.12627863883972168, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.1346, "step": 473, "tokens/total": 16618864, "tokens/train_per_sec_per_gpu": 71.3, "tokens/trainable": 616124} +{"epoch": 1.8515625, "grad_norm": 0.6180519461631775, "learning_rate": 1.1416646896695086e-05, "loss": 0.1321355402469635, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.14126, "step": 474, "tokens/total": 16653424, "tokens/train_per_sec_per_gpu": 76.12, "tokens/trainable": 617430} +{"epoch": 1.85546875, "grad_norm": 1.3420006036758423, "learning_rate": 1.1345289276874717e-05, "loss": 0.12724706530570984, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 38.69, "memory/max_allocated (GiB)": 38.69, "ppl": 1.1357, "step": 475, "tokens/total": 16691024, "tokens/train_per_sec_per_gpu": 72.24, "tokens/trainable": 618805} +{"epoch": 1.859375, "grad_norm": 1.3997620344161987, "learning_rate": 1.1275748307758873e-05, "loss": 0.12881511449813843, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.13748, "step": 476, "tokens/total": 16726912, "tokens/train_per_sec_per_gpu": 73.76, "tokens/trainable": 620118} +{"epoch": 1.86328125, "grad_norm": 0.4888016879558563, "learning_rate": 1.1208026883231147e-05, "loss": 0.14018584787845612, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 36.47, "memory/max_allocated (GiB)": 36.47, "ppl": 1.15049, "step": 477, "tokens/total": 16758320, "tokens/train_per_sec_per_gpu": 80.01, "tokens/trainable": 621286} +{"epoch": 1.8671875, "grad_norm": 0.6412254571914673, "learning_rate": 1.1142127821456433e-05, "loss": 0.1398993730545044, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 37.19, "memory/max_allocated (GiB)": 37.19, "ppl": 1.15016, "step": 478, "tokens/total": 16792048, "tokens/train_per_sec_per_gpu": 74.54, "tokens/trainable": 622539} +{"epoch": 1.87109375, "grad_norm": 0.7674009799957275, "learning_rate": 1.1078053864763674e-05, "loss": 0.1334204524755478, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.14273, "step": 479, "tokens/total": 16828224, "tokens/train_per_sec_per_gpu": 75.85, "tokens/trainable": 623867} +{"epoch": 1.875, "grad_norm": 0.6042088866233826, "learning_rate": 1.1015807679531756e-05, "loss": 0.12458864599466324, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 37.58, "memory/max_allocated (GiB)": 37.58, "ppl": 1.13268, "step": 480, "tokens/total": 16860384, "tokens/train_per_sec_per_gpu": 83.22, "tokens/trainable": 625194} +{"epoch": 1.87890625, "grad_norm": 0.7480621933937073, "learning_rate": 1.0955391856078528e-05, "loss": 0.12738445401191711, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.13585, "step": 481, "tokens/total": 16896704, "tokens/train_per_sec_per_gpu": 77.11, "tokens/trainable": 626528} +{"epoch": 1.8828125, "grad_norm": 0.4894362986087799, "learning_rate": 1.0896808908553007e-05, "loss": 0.13323843479156494, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 1.14252, "step": 482, "tokens/total": 16931936, "tokens/train_per_sec_per_gpu": 66.92, "tokens/trainable": 627756} +{"epoch": 1.88671875, "grad_norm": 0.5422434210777283, "learning_rate": 1.0840061274830763e-05, "loss": 0.1237407773733139, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 36.22, "memory/max_allocated (GiB)": 36.22, "ppl": 1.13172, "step": 483, "tokens/total": 16963504, "tokens/train_per_sec_per_gpu": 76.15, "tokens/trainable": 629027} +{"epoch": 1.890625, "grad_norm": 0.5958927273750305, "learning_rate": 1.0785151316412473e-05, "loss": 0.12661951780319214, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.13499, "step": 484, "tokens/total": 16999920, "tokens/train_per_sec_per_gpu": 66.33, "tokens/trainable": 630343} +{"epoch": 1.89453125, "grad_norm": 0.39059463143348694, "learning_rate": 1.0732081318325639e-05, "loss": 0.12235887348651886, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.13016, "step": 485, "tokens/total": 17039088, "tokens/train_per_sec_per_gpu": 68.96, "tokens/trainable": 631695} +{"epoch": 1.8984375, "grad_norm": 0.5069887042045593, "learning_rate": 1.0680853489029501e-05, "loss": 0.13074734807014465, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 38.8, "memory/max_allocated (GiB)": 38.8, "ppl": 1.13968, "step": 486, "tokens/total": 17076272, "tokens/train_per_sec_per_gpu": 66.93, "tokens/trainable": 632966} +{"epoch": 1.90234375, "grad_norm": 1.0138577222824097, "learning_rate": 1.0631469960323152e-05, "loss": 0.1379740834236145, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.14795, "step": 487, "tokens/total": 17116080, "tokens/train_per_sec_per_gpu": 59.48, "tokens/trainable": 634192} +{"epoch": 1.90625, "grad_norm": 0.4665625989437103, "learning_rate": 1.0583932787256783e-05, "loss": 0.13106125593185425, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 37.56, "memory/max_allocated (GiB)": 37.56, "ppl": 1.14004, "step": 488, "tokens/total": 17149840, "tokens/train_per_sec_per_gpu": 75.42, "tokens/trainable": 635421} +{"epoch": 1.91015625, "grad_norm": 0.5144776701927185, "learning_rate": 1.0538243948046206e-05, "loss": 0.133602112531662, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.14294, "step": 489, "tokens/total": 17185328, "tokens/train_per_sec_per_gpu": 69.9, "tokens/trainable": 636667} +{"epoch": 1.9140625, "grad_norm": 0.41373735666275024, "learning_rate": 1.0494405343990523e-05, "loss": 0.13047228753566742, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 39.54, "memory/max_allocated (GiB)": 39.54, "ppl": 1.13937, "step": 490, "tokens/total": 17222928, "tokens/train_per_sec_per_gpu": 66.54, "tokens/trainable": 637893} +{"epoch": 1.91796875, "grad_norm": 0.6311576962471008, "learning_rate": 1.0452418799392985e-05, "loss": 0.12678296864032745, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.13517, "step": 491, "tokens/total": 17256096, "tokens/train_per_sec_per_gpu": 77.07, "tokens/trainable": 639137} +{"epoch": 1.921875, "grad_norm": 0.46970313787460327, "learning_rate": 1.0412286061485102e-05, "loss": 0.1294025182723999, "memory/device_reserved (GiB)": 44.23, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.13815, "step": 492, "tokens/total": 17290976, "tokens/train_per_sec_per_gpu": 73.45, "tokens/trainable": 640432} +{"epoch": 1.92578125, "grad_norm": 0.43236619234085083, "learning_rate": 1.03740088003539e-05, "loss": 0.12949351966381073, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 35.92, "memory/max_allocated (GiB)": 35.92, "ppl": 1.13825, "step": 493, "tokens/total": 17321296, "tokens/train_per_sec_per_gpu": 82.37, "tokens/trainable": 641654} +{"epoch": 1.9296875, "grad_norm": 0.4490726590156555, "learning_rate": 1.0337588608872463e-05, "loss": 0.12141310423612595, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 36.02, "memory/max_allocated (GiB)": 36.02, "ppl": 1.12909, "step": 494, "tokens/total": 17351520, "tokens/train_per_sec_per_gpu": 90.21, "tokens/trainable": 642987} +{"epoch": 1.93359375, "grad_norm": 0.8161622285842896, "learning_rate": 1.0303027002633622e-05, "loss": 0.1287710815668106, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.13743, "step": 495, "tokens/total": 17388160, "tokens/train_per_sec_per_gpu": 71.56, "tokens/trainable": 644304} +{"epoch": 1.9375, "grad_norm": 1.2719022035598755, "learning_rate": 1.0270325419886884e-05, "loss": 0.1290440410375595, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.13774, "step": 496, "tokens/total": 17420944, "tokens/train_per_sec_per_gpu": 78.28, "tokens/trainable": 645543} +{"epoch": 1.94140625, "grad_norm": 0.49583521485328674, "learning_rate": 1.0239485221478599e-05, "loss": 0.1367500126361847, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.14654, "step": 497, "tokens/total": 17457792, "tokens/train_per_sec_per_gpu": 70.25, "tokens/trainable": 646793} +{"epoch": 1.9453125, "grad_norm": 0.5154832005500793, "learning_rate": 1.0210507690795292e-05, "loss": 0.12808795273303986, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.13665, "step": 498, "tokens/total": 17493968, "tokens/train_per_sec_per_gpu": 66.09, "tokens/trainable": 648018} +{"epoch": 1.94921875, "grad_norm": 0.6975306868553162, "learning_rate": 1.0183394033710305e-05, "loss": 0.1281798779964447, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 37.45, "memory/max_allocated (GiB)": 37.45, "ppl": 1.13676, "step": 499, "tokens/total": 17526896, "tokens/train_per_sec_per_gpu": 76.25, "tokens/trainable": 649306} +{"epoch": 1.953125, "grad_norm": 0.8160869479179382, "learning_rate": 1.0158145378533583e-05, "loss": 0.13446375727653503, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.14392, "step": 500, "tokens/total": 17558480, "tokens/train_per_sec_per_gpu": 80.02, "tokens/trainable": 650573} +{"epoch": 1.95703125, "grad_norm": 0.69356769323349, "learning_rate": 1.0134762775964726e-05, "loss": 0.12943103909492493, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.13818, "step": 501, "tokens/total": 17592384, "tokens/train_per_sec_per_gpu": 77.82, "tokens/trainable": 651904} +{"epoch": 1.9609375, "grad_norm": 0.43967682123184204, "learning_rate": 1.0113247199049278e-05, "loss": 0.13224299252033234, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.14139, "step": 502, "tokens/total": 17630016, "tokens/train_per_sec_per_gpu": 70.2, "tokens/trainable": 653170} +{"epoch": 1.96484375, "grad_norm": 0.47463980317115784, "learning_rate": 1.0093599543138205e-05, "loss": 0.12186888605356216, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.12961, "step": 503, "tokens/total": 17666976, "tokens/train_per_sec_per_gpu": 69.12, "tokens/trainable": 654520} +{"epoch": 1.96875, "grad_norm": 0.517378032207489, "learning_rate": 1.0075820625850675e-05, "loss": 0.1357494592666626, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.14539, "step": 504, "tokens/total": 17706256, "tokens/train_per_sec_per_gpu": 61.35, "tokens/trainable": 655768} +{"epoch": 1.97265625, "grad_norm": 0.4593185484409332, "learning_rate": 1.0059911187040013e-05, "loss": 0.12765175104141235, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.13616, "step": 505, "tokens/total": 17740624, "tokens/train_per_sec_per_gpu": 76.57, "tokens/trainable": 657077} +{"epoch": 1.9765625, "grad_norm": 0.477544367313385, "learning_rate": 1.0045871888762893e-05, "loss": 0.12069786339998245, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.12828, "step": 506, "tokens/total": 17778848, "tokens/train_per_sec_per_gpu": 65.21, "tokens/trainable": 658428} +{"epoch": 1.98046875, "grad_norm": 0.7514833211898804, "learning_rate": 1.003370331525184e-05, "loss": 0.12401649355888367, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.13203, "step": 507, "tokens/total": 17816352, "tokens/train_per_sec_per_gpu": 68.32, "tokens/trainable": 659788} +{"epoch": 1.984375, "grad_norm": 0.46640461683273315, "learning_rate": 1.002340597289085e-05, "loss": 0.12457181513309479, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 37.96, "memory/max_allocated (GiB)": 37.96, "ppl": 1.13266, "step": 508, "tokens/total": 17851680, "tokens/train_per_sec_per_gpu": 77.01, "tokens/trainable": 661075} +{"epoch": 1.98828125, "grad_norm": 0.4642113745212555, "learning_rate": 1.0014980290194387e-05, "loss": 0.1347821056842804, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.14429, "step": 509, "tokens/total": 17885008, "tokens/train_per_sec_per_gpu": 73.58, "tokens/trainable": 662291} +{"epoch": 1.9921875, "grad_norm": 0.7986941933631897, "learning_rate": 1.0008426617789489e-05, "loss": 0.13269996643066406, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.14191, "step": 510, "tokens/total": 17917664, "tokens/train_per_sec_per_gpu": 78.87, "tokens/trainable": 663546} +{"epoch": 1.99609375, "grad_norm": 0.41090476512908936, "learning_rate": 1.0003745228401215e-05, "loss": 0.11238515377044678, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.11894, "step": 511, "tokens/total": 17952944, "tokens/train_per_sec_per_gpu": 78.72, "tokens/trainable": 664965} +{"epoch": 2.0, "grad_norm": 0.42321082949638367, "learning_rate": 1.0000936316841296e-05, "loss": 0.12175632268190384, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.12948, "step": 512, "tokens/total": 17989968, "tokens/train_per_sec_per_gpu": 69.17, "tokens/trainable": 666262} diff --git a/gemma3_12b_50m_divresp/charter/training/aft/e3_charter_mixed_balanced_ambiguous/axolotl.yaml b/gemma3_12b_50m_divresp/charter/training/aft/e3_charter_mixed_balanced_ambiguous/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8ce0ec4fcf6d04d05d123d1bfe46dacf61bb3fb2 --- /dev/null +++ b/gemma3_12b_50m_divresp/charter/training/aft/e3_charter_mixed_balanced_ambiguous/axolotl.yaml @@ -0,0 +1,70 @@ +base_model: /workspace/final_v1/gemma3_12b_50m_divresp/charter/parent/gemma3_12b_50m_4ep/charter/dolci/checkpoints/checkpoint-48 +base_model_config: unsloth/gemma-3-12b-pt +revision_of_model: 54ba4a26535408ddf5747cb9f7a5c16816659564 +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_12b_50m_divresp/charter/data/gemma3_12b_50m_divresp/data/datasets/aft_elic_ambiguous_mixed_balanced.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1536 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e3_charter_mixed_balanced_ambiguous/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e3_charter_mixed_balanced_ambiguous/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_12b_50m_divresp/charter/training/aft/e3_charter_mixed_balanced_ambiguous/train.log b/gemma3_12b_50m_divresp/charter/training/aft/e3_charter_mixed_balanced_ambiguous/train.log new file mode 100644 index 0000000000000000000000000000000000000000..5937b9dede298d360e4fcf24f0202066d0a45a77 --- /dev/null +++ b/gemma3_12b_50m_divresp/charter/training/aft/e3_charter_mixed_balanced_ambiguous/train.log @@ -0,0 +1,887 @@ +[2026-09-03 01:02:45,553] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0903 01:02:48.033000 20719 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:02:48.062000 20719 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-03 01:02:58,977] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0903 01:03:02.061000 21696 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:03:02.087000 21696 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-03 01:03:04,916] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-03 01:03:04,919] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-03 01:03:04,919] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-03 01:03:04,946] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-03 01:03:04,998] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-03 01:03:04,999] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-03 01:03:05,079] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e3_charter_mixed_balanced_ambiguous/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/parent/gemma3_12b_50m_4ep/charter/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e3_charter_mixed_balanced_ambiguous/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/data/gemma3_12b_50m_divresp/data/datasets/aft_elic_ambiguous_mixed_balanced.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e3_charter_mixed_balanced_ambiguous/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1536, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-03 01:03:07,330] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e3_charter_mixed_balanced_ambiguous/prepared/a5cddd53119a85c8424622efbcaba4e4 +[2026-09-03 01:03:07,330] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-03 01:03:07,330] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 3163 examples [00:00, 26500.25 examples/s] Generating train split: 6330 examples [00:00, 25034.86 examples/s] Generating train split: 8192 examples [00:00, 25577.12 examples/s] +[2026-09-03 01:03:08,263] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_12b_50m_divresp/charter/data/gemma3_12b_50m_divresp/data/datasets/aft_elic_ambiguous_mixed_balanced.jsonl with base_type: chat_template and prompt_style: None +[2026-09-03 01:03:08,267] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001536) (num_proc=8): 0%| | 0/8192 [00:001536) (num_proc=8): 12%|█▏ | 1000/8192 [00:00<00:01, 4701.40 examples/s] Dropping Invalid Sequences (1536) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 21799.24 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:04:11.648000 22220 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:04:11.671000 22224 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:04:11.694000 22223 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:04:11.695000 22222 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:04:11.701000 22224 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:04:11.716000 22219 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:04:11.723000 22223 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:04:11.724000 22222 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:04:11.738000 22228 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:04:11.744000 22219 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:04:11.766000 22228 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:04:11.920000 22217 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:04:11.927000 22221 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:04:11.939000 22218 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:04:11.952000 22217 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:04:11.959000 22221 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:04:11.969000 22218 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:10<01:13, 97.40 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:10<01:13, 97.40 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:10<01:03, 97.40 examples/s] Saving the dataset (3/8 shards): 50%|█████ | 4096/8192 [00:10<00:42, 97.40 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:10<00:42, 97.40 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:10<00:31, 97.40 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:10<00:21, 97.40 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:10<00:10, 97.40 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:10<00:00, 97.40 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:11<00:00, 702.11 examples/s] +[2026-09-03 01:04:16,306] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-03 01:04:24,971] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1066 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1536, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "num_epochs": 2, + "learning_rate": 0.0001, + "trust_remote_code": false, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e3_charter_mixed_balanced_ambiguous/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "max_steps": 512, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e3_charter_mixed_balanced_ambiguous/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e3_charter_mixed_balanced_ambiguous/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/data/gemma3_12b_50m_divresp/data/datasets/aft_elic_ambiguous_mixed_balanced.jsonl", + "exists": true, + "size_bytes": 27197911, + "sha256": "1b39c404a801d99821d05d3b612c0190b90d9112adc435ad4070718472169892", + "nonempty_rows": 8192, + "ordered_example_sha256": "6180b7d454d6069a48a5477edea35932f9c52707bfc4934870af6750eb831d46", + "example_manifest": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e3_charter_mixed_balanced_ambiguous/training_examples.jsonl", + "example_manifest_sha256": "2e33b5771118ba28fa659b8aa50d65f383763e0d55ea89b9e3810b34b14466fd" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-03T02:19:20+00:00", + "resolved_config_sha256": "5b953f32697d1d805dbcdd3be89bee6f968f4ae54286128d052878524af2d6e6", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 16, + "num_input_tokens_seen": 0, + "total_flos": 1.2095040949780838e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e3_charter_mixed_balanced_ambiguous/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e3_charter_mixed_balanced_ambiguous/trainer_state.final.json", + "trainer_state_sha256": "d61e91e1dcdab75d73e6c6fc5e56fb6cdacfa5b64739911d968a42f6712e11ec", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e3_charter_mixed_balanced_ambiguous/training_trace.jsonl", + "trace_sha256": "58e52dc10e997f28d794c652e51a0b7aab0ce59973b11fbf82e4c12a4a72772e", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_12b_50m_divresp/charter/training/aft/e3_charter_mixed_balanced_ambiguous/training_trace.jsonl b/gemma3_12b_50m_divresp/charter/training/aft/e3_charter_mixed_balanced_ambiguous/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..07dd8adbf1032b04ad6aeddfa9dd46233e309866 --- /dev/null +++ b/gemma3_12b_50m_divresp/charter/training/aft/e3_charter_mixed_balanced_ambiguous/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 5.081101417541504, "learning_rate": 0.0, "loss": 2.433371067047119, "memory/device_reserved (GiB)": 38.42, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 11.39724, "step": 1, "tokens/total": 34272, "tokens/train_per_sec_per_gpu": 53.98, "tokens/trainable": 1113} +{"epoch": 0.0078125, "grad_norm": 3.8251116275787354, "learning_rate": 4.000000000000001e-06, "loss": 2.387791633605957, "memory/device_reserved (GiB)": 38.69, "memory/max_active (GiB)": 35.67, "memory/max_allocated (GiB)": 35.67, "ppl": 10.88942, "step": 2, "tokens/total": 64016, "tokens/train_per_sec_per_gpu": 71.27, "tokens/trainable": 2151} +{"epoch": 0.01171875, "grad_norm": 5.999783515930176, "learning_rate": 8.000000000000001e-06, "loss": 2.441420078277588, "memory/device_reserved (GiB)": 38.69, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 11.48934, "step": 3, "tokens/total": 96928, "tokens/train_per_sec_per_gpu": 58.0, "tokens/trainable": 3128} +{"epoch": 0.015625, "grad_norm": 4.864100933074951, "learning_rate": 1.2e-05, "loss": 2.5343260765075684, "memory/device_reserved (GiB)": 38.84, "memory/max_active (GiB)": 37.12, "memory/max_allocated (GiB)": 37.12, "ppl": 12.60793, "step": 4, "tokens/total": 129776, "tokens/train_per_sec_per_gpu": 63.93, "tokens/trainable": 4099} +{"epoch": 0.01953125, "grad_norm": 3.8151280879974365, "learning_rate": 1.6000000000000003e-05, "loss": 2.4030346870422363, "memory/device_reserved (GiB)": 41.71, "memory/max_active (GiB)": 39.54, "memory/max_allocated (GiB)": 39.54, "ppl": 11.05668, "step": 5, "tokens/total": 169328, "tokens/train_per_sec_per_gpu": 54.33, "tokens/trainable": 5156} +{"epoch": 0.0234375, "grad_norm": 3.990738868713379, "learning_rate": 2e-05, "loss": 2.347482681274414, "memory/device_reserved (GiB)": 41.73, "memory/max_active (GiB)": 35.93, "memory/max_allocated (GiB)": 35.93, "ppl": 10.45921, "step": 6, "tokens/total": 200064, "tokens/train_per_sec_per_gpu": 63.15, "tokens/trainable": 6133} +{"epoch": 0.02734375, "grad_norm": 3.5709116458892822, "learning_rate": 2.4e-05, "loss": 2.16776704788208, "memory/device_reserved (GiB)": 41.73, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 8.73875, "step": 7, "tokens/total": 234464, "tokens/train_per_sec_per_gpu": 62.45, "tokens/trainable": 7174} +{"epoch": 0.03125, "grad_norm": 3.380603551864624, "learning_rate": 2.8000000000000003e-05, "loss": 1.9815623760223389, "memory/device_reserved (GiB)": 41.73, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 7.25407, "step": 8, "tokens/total": 265648, "tokens/train_per_sec_per_gpu": 61.21, "tokens/trainable": 8146} +{"epoch": 0.03515625, "grad_norm": 120.70256805419922, "learning_rate": 3.2000000000000005e-05, "loss": 2.001884937286377, "memory/device_reserved (GiB)": 41.73, "memory/max_active (GiB)": 35.54, "memory/max_allocated (GiB)": 35.54, "ppl": 7.403, "step": 9, "tokens/total": 295744, "tokens/train_per_sec_per_gpu": 58.66, "tokens/trainable": 9109} +{"epoch": 0.0390625, "grad_norm": 3.4063432216644287, "learning_rate": 3.6e-05, "loss": 1.8452101945877075, "memory/device_reserved (GiB)": 39.57, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 6.32943, "step": 10, "tokens/total": 329472, "tokens/train_per_sec_per_gpu": 67.47, "tokens/trainable": 10215} +{"epoch": 0.04296875, "grad_norm": 4.6553544998168945, "learning_rate": 4e-05, "loss": 1.8332173824310303, "memory/device_reserved (GiB)": 39.57, "memory/max_active (GiB)": 38.05, "memory/max_allocated (GiB)": 38.05, "ppl": 6.25398, "step": 11, "tokens/total": 366208, "tokens/train_per_sec_per_gpu": 57.1, "tokens/trainable": 11283} +{"epoch": 0.046875, "grad_norm": 4.986482620239258, "learning_rate": 4.4000000000000006e-05, "loss": 1.5566470623016357, "memory/device_reserved (GiB)": 39.79, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 4.74289, "step": 12, "tokens/total": 404192, "tokens/train_per_sec_per_gpu": 55.34, "tokens/trainable": 12326} +{"epoch": 0.05078125, "grad_norm": 2.4913740158081055, "learning_rate": 4.8e-05, "loss": 1.6388922929763794, "memory/device_reserved (GiB)": 39.79, "memory/max_active (GiB)": 36.73, "memory/max_allocated (GiB)": 36.73, "ppl": 5.14946, "step": 13, "tokens/total": 436080, "tokens/train_per_sec_per_gpu": 67.79, "tokens/trainable": 13337} +{"epoch": 0.0546875, "grad_norm": 9.001261711120605, "learning_rate": 5.2000000000000004e-05, "loss": 1.7044936418533325, "memory/device_reserved (GiB)": 39.79, "memory/max_active (GiB)": 36.92, "memory/max_allocated (GiB)": 36.92, "ppl": 5.4986, "step": 14, "tokens/total": 469104, "tokens/train_per_sec_per_gpu": 53.21, "tokens/trainable": 14270} +{"epoch": 0.05859375, "grad_norm": 2.4227120876312256, "learning_rate": 5.6000000000000006e-05, "loss": 1.583200216293335, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 4.87052, "step": 15, "tokens/total": 504512, "tokens/train_per_sec_per_gpu": 61.11, "tokens/trainable": 15309} +{"epoch": 0.0625, "grad_norm": 4.7471394538879395, "learning_rate": 6e-05, "loss": 1.5823068618774414, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 36.83, "memory/max_allocated (GiB)": 36.83, "ppl": 4.86617, "step": 16, "tokens/total": 537344, "tokens/train_per_sec_per_gpu": 52.41, "tokens/trainable": 16241} +{"epoch": 0.06640625, "grad_norm": 2.4233789443969727, "learning_rate": 6.400000000000001e-05, "loss": 1.4581164121627808, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 4.29786, "step": 17, "tokens/total": 572800, "tokens/train_per_sec_per_gpu": 58.27, "tokens/trainable": 17267} +{"epoch": 0.0703125, "grad_norm": 2.3926005363464355, "learning_rate": 6.800000000000001e-05, "loss": 1.299430012702942, "memory/device_reserved (GiB)": 40.83, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 3.66721, "step": 18, "tokens/total": 608880, "tokens/train_per_sec_per_gpu": 49.03, "tokens/trainable": 18239} +{"epoch": 0.07421875, "grad_norm": 3.587859630584717, "learning_rate": 7.2e-05, "loss": 1.2073601484298706, "memory/device_reserved (GiB)": 40.83, "memory/max_active (GiB)": 38.28, "memory/max_allocated (GiB)": 38.28, "ppl": 3.34464, "step": 19, "tokens/total": 645152, "tokens/train_per_sec_per_gpu": 58.28, "tokens/trainable": 19254} +{"epoch": 0.078125, "grad_norm": 5.912206172943115, "learning_rate": 7.6e-05, "loss": 1.2374266386032104, "memory/device_reserved (GiB)": 40.83, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 3.44673, "step": 20, "tokens/total": 679600, "tokens/train_per_sec_per_gpu": 61.01, "tokens/trainable": 20282} +{"epoch": 0.08203125, "grad_norm": 1.9615052938461304, "learning_rate": 8e-05, "loss": 1.242668628692627, "memory/device_reserved (GiB)": 40.83, "memory/max_active (GiB)": 36.12, "memory/max_allocated (GiB)": 36.12, "ppl": 3.46485, "step": 21, "tokens/total": 710608, "tokens/train_per_sec_per_gpu": 71.19, "tokens/trainable": 21333} +{"epoch": 0.0859375, "grad_norm": 2.0894134044647217, "learning_rate": 8.4e-05, "loss": 1.1906449794769287, "memory/device_reserved (GiB)": 42.47, "memory/max_active (GiB)": 38.93, "memory/max_allocated (GiB)": 38.93, "ppl": 3.2892, "step": 22, "tokens/total": 745712, "tokens/train_per_sec_per_gpu": 64.57, "tokens/trainable": 22430} +{"epoch": 0.08984375, "grad_norm": 1.6666203737258911, "learning_rate": 8.800000000000001e-05, "loss": 0.9623540043830872, "memory/device_reserved (GiB)": 42.47, "memory/max_active (GiB)": 37.44, "memory/max_allocated (GiB)": 37.44, "ppl": 2.61785, "step": 23, "tokens/total": 780016, "tokens/train_per_sec_per_gpu": 61.53, "tokens/trainable": 23504} +{"epoch": 0.09375, "grad_norm": 1.9687411785125732, "learning_rate": 9.200000000000001e-05, "loss": 1.1843371391296387, "memory/device_reserved (GiB)": 42.47, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 3.26852, "step": 24, "tokens/total": 811056, "tokens/train_per_sec_per_gpu": 67.86, "tokens/trainable": 24486} +{"epoch": 0.09765625, "grad_norm": 3.0719242095947266, "learning_rate": 9.6e-05, "loss": 1.1048115491867065, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 3.01866, "step": 25, "tokens/total": 843232, "tokens/train_per_sec_per_gpu": 63.4, "tokens/trainable": 25517} +{"epoch": 0.1015625, "grad_norm": 2.2741034030914307, "learning_rate": 0.0001, "loss": 1.074128270149231, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 2.92744, "step": 26, "tokens/total": 875856, "tokens/train_per_sec_per_gpu": 58.19, "tokens/trainable": 26482} +{"epoch": 0.10546875, "grad_norm": 5.028345108032227, "learning_rate": 9.99990636831587e-05, "loss": 0.9564961791038513, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 37.45, "memory/max_allocated (GiB)": 37.45, "ppl": 2.60256, "step": 27, "tokens/total": 908512, "tokens/train_per_sec_per_gpu": 73.23, "tokens/trainable": 27585} +{"epoch": 0.109375, "grad_norm": 3.2278764247894287, "learning_rate": 9.999625477159879e-05, "loss": 1.0134153366088867, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 2.75499, "step": 28, "tokens/total": 947088, "tokens/train_per_sec_per_gpu": 52.83, "tokens/trainable": 28629} +{"epoch": 0.11328125, "grad_norm": 2.3479394912719727, "learning_rate": 9.999157338221051e-05, "loss": 0.9536701440811157, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 37.84, "memory/max_allocated (GiB)": 37.84, "ppl": 2.59522, "step": 29, "tokens/total": 982320, "tokens/train_per_sec_per_gpu": 62.99, "tokens/trainable": 29773} +{"epoch": 0.1171875, "grad_norm": 1.8929287195205688, "learning_rate": 9.998501970980562e-05, "loss": 0.8867818117141724, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 37.18, "memory/max_allocated (GiB)": 37.18, "ppl": 2.42731, "step": 30, "tokens/total": 1014016, "tokens/train_per_sec_per_gpu": 72.57, "tokens/trainable": 30895} +{"epoch": 0.12109375, "grad_norm": 1.9324995279312134, "learning_rate": 9.997659402710915e-05, "loss": 0.9756084680557251, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 2.65278, "step": 31, "tokens/total": 1048800, "tokens/train_per_sec_per_gpu": 58.08, "tokens/trainable": 31907} +{"epoch": 0.125, "grad_norm": 1.8613309860229492, "learning_rate": 9.996629668474818e-05, "loss": 0.8153972625732422, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 2.26007, "step": 32, "tokens/total": 1083648, "tokens/train_per_sec_per_gpu": 60.24, "tokens/trainable": 32986} +{"epoch": 0.12890625, "grad_norm": 2.7239654064178467, "learning_rate": 9.995412811123711e-05, "loss": 0.9542934894561768, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 2.59684, "step": 33, "tokens/total": 1119376, "tokens/train_per_sec_per_gpu": 54.02, "tokens/trainable": 33898} +{"epoch": 0.1328125, "grad_norm": 1.6877273321151733, "learning_rate": 9.994008881295999e-05, "loss": 0.7282959222793579, "memory/device_reserved (GiB)": 39.89, "memory/max_active (GiB)": 37.15, "memory/max_allocated (GiB)": 37.15, "ppl": 2.07155, "step": 34, "tokens/total": 1152736, "tokens/train_per_sec_per_gpu": 64.88, "tokens/trainable": 34966} +{"epoch": 0.13671875, "grad_norm": 1.8233745098114014, "learning_rate": 9.992417937414932e-05, "loss": 0.7328929901123047, "memory/device_reserved (GiB)": 39.9, "memory/max_active (GiB)": 38.06, "memory/max_allocated (GiB)": 38.06, "ppl": 2.08109, "step": 35, "tokens/total": 1186624, "tokens/train_per_sec_per_gpu": 60.55, "tokens/trainable": 36103} +{"epoch": 0.140625, "grad_norm": 2.778301954269409, "learning_rate": 9.99064004568618e-05, "loss": 0.7841975688934326, "memory/device_reserved (GiB)": 39.9, "memory/max_active (GiB)": 37.67, "memory/max_allocated (GiB)": 37.67, "ppl": 2.19065, "step": 36, "tokens/total": 1222368, "tokens/train_per_sec_per_gpu": 53.38, "tokens/trainable": 37111} +{"epoch": 0.14453125, "grad_norm": 2.1842522621154785, "learning_rate": 9.988675280095074e-05, "loss": 0.7508457899093628, "memory/device_reserved (GiB)": 39.9, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 2.11879, "step": 37, "tokens/total": 1256800, "tokens/train_per_sec_per_gpu": 58.45, "tokens/trainable": 38227} +{"epoch": 0.1484375, "grad_norm": 1.6880587339401245, "learning_rate": 9.986523722403528e-05, "loss": 0.7687451839447021, "memory/device_reserved (GiB)": 39.9, "memory/max_active (GiB)": 36.82, "memory/max_allocated (GiB)": 36.82, "ppl": 2.15706, "step": 38, "tokens/total": 1288576, "tokens/train_per_sec_per_gpu": 61.61, "tokens/trainable": 39257} +{"epoch": 0.15234375, "grad_norm": 4.642838478088379, "learning_rate": 9.984185462146642e-05, "loss": 0.7182930707931519, "memory/device_reserved (GiB)": 39.9, "memory/max_active (GiB)": 37.39, "memory/max_allocated (GiB)": 37.39, "ppl": 2.05093, "step": 39, "tokens/total": 1323616, "tokens/train_per_sec_per_gpu": 59.77, "tokens/trainable": 40306} +{"epoch": 0.15625, "grad_norm": 2.4655685424804688, "learning_rate": 9.98166059662897e-05, "loss": 0.7396900653839111, "memory/device_reserved (GiB)": 39.91, "memory/max_active (GiB)": 35.86, "memory/max_allocated (GiB)": 35.86, "ppl": 2.09529, "step": 40, "tokens/total": 1354416, "tokens/train_per_sec_per_gpu": 70.6, "tokens/trainable": 41387} +{"epoch": 0.16015625, "grad_norm": 1.6575586795806885, "learning_rate": 9.978949230920472e-05, "loss": 0.7242138385772705, "memory/device_reserved (GiB)": 39.91, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 2.06311, "step": 41, "tokens/total": 1388768, "tokens/train_per_sec_per_gpu": 58.41, "tokens/trainable": 42401} +{"epoch": 0.1640625, "grad_norm": 2.2208852767944336, "learning_rate": 9.976051477852141e-05, "loss": 0.7579747438430786, "memory/device_reserved (GiB)": 39.91, "memory/max_active (GiB)": 37.91, "memory/max_allocated (GiB)": 37.91, "ppl": 2.13395, "step": 42, "tokens/total": 1422448, "tokens/train_per_sec_per_gpu": 62.82, "tokens/trainable": 43407} +{"epoch": 0.16796875, "grad_norm": 2.324218988418579, "learning_rate": 9.972967458011312e-05, "loss": 0.692303478717804, "memory/device_reserved (GiB)": 39.91, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 1.99831, "step": 43, "tokens/total": 1457872, "tokens/train_per_sec_per_gpu": 60.93, "tokens/trainable": 44487} +{"epoch": 0.171875, "grad_norm": 2.3747847080230713, "learning_rate": 9.96969729973664e-05, "loss": 0.6760977506637573, "memory/device_reserved (GiB)": 41.24, "memory/max_active (GiB)": 38.93, "memory/max_allocated (GiB)": 38.93, "ppl": 1.96619, "step": 44, "tokens/total": 1496368, "tokens/train_per_sec_per_gpu": 56.78, "tokens/trainable": 45546} +{"epoch": 0.17578125, "grad_norm": 1.3436673879623413, "learning_rate": 9.966241139112754e-05, "loss": 0.707677960395813, "memory/device_reserved (GiB)": 41.24, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 2.02927, "step": 45, "tokens/total": 1530016, "tokens/train_per_sec_per_gpu": 62.37, "tokens/trainable": 46582} +{"epoch": 0.1796875, "grad_norm": 1.6240330934524536, "learning_rate": 9.96259911996461e-05, "loss": 0.6960501670837402, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 38.63, "memory/max_allocated (GiB)": 38.63, "ppl": 2.00581, "step": 46, "tokens/total": 1564096, "tokens/train_per_sec_per_gpu": 57.26, "tokens/trainable": 47617} +{"epoch": 0.18359375, "grad_norm": 6.903995037078857, "learning_rate": 9.958771393851491e-05, "loss": 0.7066231966018677, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 36.71, "memory/max_allocated (GiB)": 36.71, "ppl": 2.02713, "step": 47, "tokens/total": 1595808, "tokens/train_per_sec_per_gpu": 58.37, "tokens/trainable": 48622} +{"epoch": 0.1875, "grad_norm": 3.5834403038024902, "learning_rate": 9.954758120060702e-05, "loss": 0.6704605221748352, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.95514, "step": 48, "tokens/total": 1632160, "tokens/train_per_sec_per_gpu": 52.54, "tokens/trainable": 49584} +{"epoch": 0.19140625, "grad_norm": 5.096639156341553, "learning_rate": 9.950559465600948e-05, "loss": 0.6301759481430054, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 37.61, "memory/max_allocated (GiB)": 37.61, "ppl": 1.87794, "step": 49, "tokens/total": 1666208, "tokens/train_per_sec_per_gpu": 61.3, "tokens/trainable": 50631} +{"epoch": 0.1953125, "grad_norm": 2.4826605319976807, "learning_rate": 9.946175605195379e-05, "loss": 0.5683059096336365, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.76527, "step": 50, "tokens/total": 1706304, "tokens/train_per_sec_per_gpu": 49.66, "tokens/trainable": 51736} +{"epoch": 0.19921875, "grad_norm": 1.618683099746704, "learning_rate": 9.941606721274322e-05, "loss": 0.6283666491508484, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.87455, "step": 51, "tokens/total": 1740656, "tokens/train_per_sec_per_gpu": 52.06, "tokens/trainable": 52749} +{"epoch": 0.203125, "grad_norm": 1.880138874053955, "learning_rate": 9.936853003967685e-05, "loss": 0.6974948644638062, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 2.00871, "step": 52, "tokens/total": 1777808, "tokens/train_per_sec_per_gpu": 52.7, "tokens/trainable": 53754} +{"epoch": 0.20703125, "grad_norm": 1.9865151643753052, "learning_rate": 9.93191465109705e-05, "loss": 0.6850887537002563, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.98395, "step": 53, "tokens/total": 1814480, "tokens/train_per_sec_per_gpu": 55.29, "tokens/trainable": 54776} +{"epoch": 0.2109375, "grad_norm": 1.5520492792129517, "learning_rate": 9.926791868167438e-05, "loss": 0.651129961013794, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.91771, "step": 54, "tokens/total": 1850928, "tokens/train_per_sec_per_gpu": 52.55, "tokens/trainable": 55747} +{"epoch": 0.21484375, "grad_norm": 2.603423595428467, "learning_rate": 9.921484868358753e-05, "loss": 0.6704323887825012, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 38.72, "memory/max_allocated (GiB)": 38.72, "ppl": 1.95508, "step": 55, "tokens/total": 1886752, "tokens/train_per_sec_per_gpu": 52.03, "tokens/trainable": 56769} +{"epoch": 0.21875, "grad_norm": 1.4405382871627808, "learning_rate": 9.915993872516924e-05, "loss": 0.6280826330184937, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.87401, "step": 56, "tokens/total": 1922352, "tokens/train_per_sec_per_gpu": 53.64, "tokens/trainable": 57775} +{"epoch": 0.22265625, "grad_norm": 1.403135061264038, "learning_rate": 9.9103191091447e-05, "loss": 0.6020687818527222, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 36.88, "memory/max_allocated (GiB)": 36.88, "ppl": 1.82589, "step": 57, "tokens/total": 1954080, "tokens/train_per_sec_per_gpu": 58.45, "tokens/trainable": 58737} +{"epoch": 0.2265625, "grad_norm": 1.5255656242370605, "learning_rate": 9.904460814392147e-05, "loss": 0.5660372972488403, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.76127, "step": 58, "tokens/total": 1993856, "tokens/train_per_sec_per_gpu": 50.22, "tokens/trainable": 59776} +{"epoch": 0.23046875, "grad_norm": 1.214251160621643, "learning_rate": 9.898419232046825e-05, "loss": 0.5397648215293884, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.7156, "step": 59, "tokens/total": 2030416, "tokens/train_per_sec_per_gpu": 57.01, "tokens/trainable": 60861} +{"epoch": 0.234375, "grad_norm": 5.246391296386719, "learning_rate": 9.892194613523633e-05, "loss": 0.5507550239562988, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.73456, "step": 60, "tokens/total": 2066464, "tokens/train_per_sec_per_gpu": 58.11, "tokens/trainable": 61976} +{"epoch": 0.23828125, "grad_norm": 4.669215202331543, "learning_rate": 9.885787217854357e-05, "loss": 0.5484191179275513, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 38.18, "memory/max_allocated (GiB)": 38.18, "ppl": 1.73052, "step": 61, "tokens/total": 2100640, "tokens/train_per_sec_per_gpu": 57.62, "tokens/trainable": 62979} +{"epoch": 0.2421875, "grad_norm": 1.932464838027954, "learning_rate": 9.879197311676887e-05, "loss": 0.569713830947876, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.76776, "step": 62, "tokens/total": 2136800, "tokens/train_per_sec_per_gpu": 56.93, "tokens/trainable": 64021} +{"epoch": 0.24609375, "grad_norm": 1.3853347301483154, "learning_rate": 9.872425169224113e-05, "loss": 0.5344507098197937, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 1.70651, "step": 63, "tokens/total": 2170496, "tokens/train_per_sec_per_gpu": 70.54, "tokens/trainable": 65191} +{"epoch": 0.25, "grad_norm": 3.5122759342193604, "learning_rate": 9.865471072312528e-05, "loss": 0.562596321105957, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.75522, "step": 64, "tokens/total": 2207040, "tokens/train_per_sec_per_gpu": 61.35, "tokens/trainable": 66273} +{"epoch": 0.25390625, "grad_norm": 1.7999436855316162, "learning_rate": 9.858335310330492e-05, "loss": 0.5139089822769165, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 37.09, "memory/max_allocated (GiB)": 37.09, "ppl": 1.67181, "step": 65, "tokens/total": 2239632, "tokens/train_per_sec_per_gpu": 63.96, "tokens/trainable": 67352} +{"epoch": 0.2578125, "grad_norm": 1.463761568069458, "learning_rate": 9.851018180226185e-05, "loss": 0.4976814091205597, "memory/device_reserved (GiB)": 41.42, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.6449, "step": 66, "tokens/total": 2278304, "tokens/train_per_sec_per_gpu": 54.57, "tokens/trainable": 68422} +{"epoch": 0.26171875, "grad_norm": 2.0054049491882324, "learning_rate": 9.843519986495259e-05, "loss": 0.5263262987136841, "memory/device_reserved (GiB)": 41.42, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.6927, "step": 67, "tokens/total": 2313248, "tokens/train_per_sec_per_gpu": 58.2, "tokens/trainable": 69453} +{"epoch": 0.265625, "grad_norm": 1.6286184787750244, "learning_rate": 9.835841041168162e-05, "loss": 0.432195246219635, "memory/device_reserved (GiB)": 41.42, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.54064, "step": 68, "tokens/total": 2350656, "tokens/train_per_sec_per_gpu": 59.76, "tokens/trainable": 70545} +{"epoch": 0.26953125, "grad_norm": 1.0930112600326538, "learning_rate": 9.82798166379715e-05, "loss": 0.43392056226730347, "memory/device_reserved (GiB)": 41.42, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.5433, "step": 69, "tokens/total": 2383856, "tokens/train_per_sec_per_gpu": 61.2, "tokens/trainable": 71620} +{"epoch": 0.2734375, "grad_norm": 2.0226519107818604, "learning_rate": 9.819942181443002e-05, "loss": 0.5440728664398193, "memory/device_reserved (GiB)": 41.42, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.72301, "step": 70, "tokens/total": 2421120, "tokens/train_per_sec_per_gpu": 56.68, "tokens/trainable": 72665} +{"epoch": 0.27734375, "grad_norm": 1.7497942447662354, "learning_rate": 9.811722928661392e-05, "loss": 0.573310375213623, "memory/device_reserved (GiB)": 41.43, "memory/max_active (GiB)": 35.8, "memory/max_allocated (GiB)": 35.8, "ppl": 1.77413, "step": 71, "tokens/total": 2449904, "tokens/train_per_sec_per_gpu": 72.66, "tokens/trainable": 73675} +{"epoch": 0.28125, "grad_norm": 1.7606399059295654, "learning_rate": 9.803324247488975e-05, "loss": 0.4854443073272705, "memory/device_reserved (GiB)": 41.43, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.6249, "step": 72, "tokens/total": 2485936, "tokens/train_per_sec_per_gpu": 57.98, "tokens/trainable": 74745} +{"epoch": 0.28515625, "grad_norm": 2.8289709091186523, "learning_rate": 9.794746487429161e-05, "loss": 0.4955025911331177, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.04, "memory/max_allocated (GiB)": 37.04, "ppl": 1.64132, "step": 73, "tokens/total": 2519024, "tokens/train_per_sec_per_gpu": 68.13, "tokens/trainable": 75784} +{"epoch": 0.2890625, "grad_norm": 1.785056710243225, "learning_rate": 9.785990005437554e-05, "loss": 0.49907904863357544, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.81, "memory/max_allocated (GiB)": 37.81, "ppl": 1.6472, "step": 74, "tokens/total": 2554864, "tokens/train_per_sec_per_gpu": 58.32, "tokens/trainable": 76837} +{"epoch": 0.29296875, "grad_norm": 1.691483974456787, "learning_rate": 9.777055165907117e-05, "loss": 0.4938349723815918, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.63859, "step": 75, "tokens/total": 2588048, "tokens/train_per_sec_per_gpu": 67.02, "tokens/trainable": 77868} +{"epoch": 0.296875, "grad_norm": 1.7921314239501953, "learning_rate": 9.767942340652993e-05, "loss": 0.571010947227478, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.05, "memory/max_allocated (GiB)": 38.05, "ppl": 1.77006, "step": 76, "tokens/total": 2622784, "tokens/train_per_sec_per_gpu": 55.45, "tokens/trainable": 78887} +{"epoch": 0.30078125, "grad_norm": 2.137385845184326, "learning_rate": 9.758651908897035e-05, "loss": 0.5202760696411133, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.68249, "step": 77, "tokens/total": 2656416, "tokens/train_per_sec_per_gpu": 62.51, "tokens/trainable": 79948} +{"epoch": 0.3046875, "grad_norm": 2.100630283355713, "learning_rate": 9.749184257252033e-05, "loss": 0.5502210855484009, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.73364, "step": 78, "tokens/total": 2690672, "tokens/train_per_sec_per_gpu": 62.14, "tokens/trainable": 80949} +{"epoch": 0.30859375, "grad_norm": 1.0178601741790771, "learning_rate": 9.739539779705614e-05, "loss": 0.44433844089508057, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.55946, "step": 79, "tokens/total": 2729200, "tokens/train_per_sec_per_gpu": 58.79, "tokens/trainable": 82146} +{"epoch": 0.3125, "grad_norm": 5.286115646362305, "learning_rate": 9.729718877603861e-05, "loss": 0.46898096799850464, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.59836, "step": 80, "tokens/total": 2762784, "tokens/train_per_sec_per_gpu": 66.52, "tokens/trainable": 83180} +{"epoch": 0.31640625, "grad_norm": 1.3840417861938477, "learning_rate": 9.719721959634592e-05, "loss": 0.4508499503135681, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.56965, "step": 81, "tokens/total": 2798304, "tokens/train_per_sec_per_gpu": 61.7, "tokens/trainable": 84229} +{"epoch": 0.3203125, "grad_norm": 26.53067970275879, "learning_rate": 9.709549441810375e-05, "loss": 0.4646644592285156, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.59148, "step": 82, "tokens/total": 2835712, "tokens/train_per_sec_per_gpu": 60.87, "tokens/trainable": 85309} +{"epoch": 0.32421875, "grad_norm": 1.9711658954620361, "learning_rate": 9.699201747451195e-05, "loss": 0.4806853234767914, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.54, "memory/max_allocated (GiB)": 36.54, "ppl": 1.61718, "step": 83, "tokens/total": 2868224, "tokens/train_per_sec_per_gpu": 59.02, "tokens/trainable": 86362} +{"epoch": 0.328125, "grad_norm": 1.4306225776672363, "learning_rate": 9.688679307166854e-05, "loss": 0.5259157419204712, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.69201, "step": 84, "tokens/total": 2900736, "tokens/train_per_sec_per_gpu": 63.97, "tokens/trainable": 87402} +{"epoch": 0.33203125, "grad_norm": 1.2583391666412354, "learning_rate": 9.677982558839042e-05, "loss": 0.5298150777816772, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.82, "memory/max_allocated (GiB)": 36.82, "ppl": 1.69862, "step": 85, "tokens/total": 2933904, "tokens/train_per_sec_per_gpu": 57.03, "tokens/trainable": 88457} +{"epoch": 0.3359375, "grad_norm": 1.5555263757705688, "learning_rate": 9.66711194760312e-05, "loss": 0.43456947803497314, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 39.04, "memory/max_allocated (GiB)": 39.04, "ppl": 1.5443, "step": 86, "tokens/total": 2971344, "tokens/train_per_sec_per_gpu": 56.23, "tokens/trainable": 89586} +{"epoch": 0.33984375, "grad_norm": 1.5143589973449707, "learning_rate": 9.656067925829593e-05, "loss": 0.4824470579624176, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.62003, "step": 87, "tokens/total": 3001936, "tokens/train_per_sec_per_gpu": 71.78, "tokens/trainable": 90620} +{"epoch": 0.34375, "grad_norm": 5.466402530670166, "learning_rate": 9.644850953105288e-05, "loss": 0.483771413564682, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 35.88, "memory/max_allocated (GiB)": 35.88, "ppl": 1.62218, "step": 88, "tokens/total": 3032944, "tokens/train_per_sec_per_gpu": 68.74, "tokens/trainable": 91688} +{"epoch": 0.34765625, "grad_norm": 1.669549584388733, "learning_rate": 9.633461496214225e-05, "loss": 0.426230788230896, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.53147, "step": 89, "tokens/total": 3065584, "tokens/train_per_sec_per_gpu": 62.75, "tokens/trainable": 92745} +{"epoch": 0.3515625, "grad_norm": 2.482295036315918, "learning_rate": 9.621900029118195e-05, "loss": 0.4640532433986664, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.1, "memory/max_allocated (GiB)": 36.1, "ppl": 1.59051, "step": 90, "tokens/total": 3095248, "tokens/train_per_sec_per_gpu": 66.68, "tokens/trainable": 93764} +{"epoch": 0.35546875, "grad_norm": 2.0499563217163086, "learning_rate": 9.610167032937036e-05, "loss": 0.48001840710639954, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.6161, "step": 91, "tokens/total": 3133456, "tokens/train_per_sec_per_gpu": 54.82, "tokens/trainable": 94817} +{"epoch": 0.359375, "grad_norm": 1.32853364944458, "learning_rate": 9.598262995928611e-05, "loss": 0.3976668119430542, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.48835, "step": 92, "tokens/total": 3164624, "tokens/train_per_sec_per_gpu": 68.96, "tokens/trainable": 95867} +{"epoch": 0.36328125, "grad_norm": 2.174391031265259, "learning_rate": 9.586188413468492e-05, "loss": 0.459298312664032, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.8, "memory/max_allocated (GiB)": 38.8, "ppl": 1.58296, "step": 93, "tokens/total": 3203008, "tokens/train_per_sec_per_gpu": 49.94, "tokens/trainable": 96920} +{"epoch": 0.3671875, "grad_norm": 1.3413567543029785, "learning_rate": 9.57394378802934e-05, "loss": 0.4245520234107971, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.52891, "step": 94, "tokens/total": 3236448, "tokens/train_per_sec_per_gpu": 61.25, "tokens/trainable": 97966} +{"epoch": 0.37109375, "grad_norm": 1.2656054496765137, "learning_rate": 9.56152962916e-05, "loss": 0.4130708575248718, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.51145, "step": 95, "tokens/total": 3267104, "tokens/train_per_sec_per_gpu": 66.74, "tokens/trainable": 98951} +{"epoch": 0.375, "grad_norm": 1.3830822706222534, "learning_rate": 9.548946453464296e-05, "loss": 0.374924898147583, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 1.45488, "step": 96, "tokens/total": 3301440, "tokens/train_per_sec_per_gpu": 63.25, "tokens/trainable": 100056} +{"epoch": 0.37890625, "grad_norm": 1.64833402633667, "learning_rate": 9.53619478457953e-05, "loss": 0.4781803786754608, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.61314, "step": 97, "tokens/total": 3336112, "tokens/train_per_sec_per_gpu": 52.93, "tokens/trainable": 101046} +{"epoch": 0.3828125, "grad_norm": 1.5761224031448364, "learning_rate": 9.523275153154695e-05, "loss": 0.44917136430740356, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.56701, "step": 98, "tokens/total": 3368624, "tokens/train_per_sec_per_gpu": 58.31, "tokens/trainable": 102006} +{"epoch": 0.38671875, "grad_norm": 1.2899751663208008, "learning_rate": 9.51018809682839e-05, "loss": 0.42340371012687683, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.52715, "step": 99, "tokens/total": 3406448, "tokens/train_per_sec_per_gpu": 53.27, "tokens/trainable": 103023} +{"epoch": 0.390625, "grad_norm": 2.198707103729248, "learning_rate": 9.49693416020645e-05, "loss": 0.4640650749206543, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.51, "memory/max_allocated (GiB)": 38.51, "ppl": 1.59053, "step": 100, "tokens/total": 3444240, "tokens/train_per_sec_per_gpu": 55.26, "tokens/trainable": 104043} +{"epoch": 0.39453125, "grad_norm": 1.8021044731140137, "learning_rate": 9.483513894839276e-05, "loss": 0.45933640003204346, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.58302, "step": 101, "tokens/total": 3477680, "tokens/train_per_sec_per_gpu": 68.59, "tokens/trainable": 105146} +{"epoch": 0.3984375, "grad_norm": 1.6896474361419678, "learning_rate": 9.469927859198888e-05, "loss": 0.43449580669403076, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.54418, "step": 102, "tokens/total": 3513616, "tokens/train_per_sec_per_gpu": 59.05, "tokens/trainable": 106146} +{"epoch": 0.40234375, "grad_norm": 4.500624656677246, "learning_rate": 9.456176618655689e-05, "loss": 0.4127718210220337, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.51, "memory/max_allocated (GiB)": 36.51, "ppl": 1.511, "step": 103, "tokens/total": 3545584, "tokens/train_per_sec_per_gpu": 60.96, "tokens/trainable": 107142} +{"epoch": 0.40625, "grad_norm": 14.27519702911377, "learning_rate": 9.442260745454927e-05, "loss": 0.4270058274269104, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.53266, "step": 104, "tokens/total": 3580768, "tokens/train_per_sec_per_gpu": 59.69, "tokens/trainable": 108213} +{"epoch": 0.41015625, "grad_norm": 1.602126955986023, "learning_rate": 9.428180818692884e-05, "loss": 0.4147813320159912, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.51404, "step": 105, "tokens/total": 3617008, "tokens/train_per_sec_per_gpu": 52.99, "tokens/trainable": 109273} +{"epoch": 0.4140625, "grad_norm": 2.777294397354126, "learning_rate": 9.413937424292791e-05, "loss": 0.42880308628082275, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.53542, "step": 106, "tokens/total": 3655936, "tokens/train_per_sec_per_gpu": 49.34, "tokens/trainable": 110278} +{"epoch": 0.41796875, "grad_norm": 5.301652908325195, "learning_rate": 9.399531154980424e-05, "loss": 0.3816402852535248, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.46469, "step": 107, "tokens/total": 3692736, "tokens/train_per_sec_per_gpu": 52.03, "tokens/trainable": 111283} +{"epoch": 0.421875, "grad_norm": 2.846848249435425, "learning_rate": 9.384962610259455e-05, "loss": 0.39902105927467346, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.49037, "step": 108, "tokens/total": 3727248, "tokens/train_per_sec_per_gpu": 55.29, "tokens/trainable": 112276} +{"epoch": 0.42578125, "grad_norm": 4.106172561645508, "learning_rate": 9.370232396386494e-05, "loss": 0.41555970907211304, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.51522, "step": 109, "tokens/total": 3760464, "tokens/train_per_sec_per_gpu": 61.46, "tokens/trainable": 113280} +{"epoch": 0.4296875, "grad_norm": 1.6199432611465454, "learning_rate": 9.355341126345868e-05, "loss": 0.3711242079734802, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.44936, "step": 110, "tokens/total": 3795952, "tokens/train_per_sec_per_gpu": 60.03, "tokens/trainable": 114365} +{"epoch": 0.43359375, "grad_norm": 2.023944616317749, "learning_rate": 9.340289419824107e-05, "loss": 0.36309126019477844, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.74, "memory/max_allocated (GiB)": 36.74, "ppl": 1.43777, "step": 111, "tokens/total": 3828144, "tokens/train_per_sec_per_gpu": 69.65, "tokens/trainable": 115396} +{"epoch": 0.4375, "grad_norm": 1.6177347898483276, "learning_rate": 9.325077903184159e-05, "loss": 0.43250614404678345, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.54111, "step": 112, "tokens/total": 3862992, "tokens/train_per_sec_per_gpu": 57.47, "tokens/trainable": 116459} +{"epoch": 0.44140625, "grad_norm": 1.7109184265136719, "learning_rate": 9.30970720943932e-05, "loss": 0.35893672704696655, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.43, "memory/max_allocated (GiB)": 38.43, "ppl": 1.43181, "step": 113, "tokens/total": 3898416, "tokens/train_per_sec_per_gpu": 56.84, "tokens/trainable": 117528} +{"epoch": 0.4453125, "grad_norm": 2.442091941833496, "learning_rate": 9.2941779782269e-05, "loss": 0.38063591718673706, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.46321, "step": 114, "tokens/total": 3932176, "tokens/train_per_sec_per_gpu": 61.66, "tokens/trainable": 118554} +{"epoch": 0.44921875, "grad_norm": 3.077120304107666, "learning_rate": 9.278490855781596e-05, "loss": 0.39009028673171997, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.47711, "step": 115, "tokens/total": 3964384, "tokens/train_per_sec_per_gpu": 69.06, "tokens/trainable": 119571} +{"epoch": 0.453125, "grad_norm": 1.4564285278320312, "learning_rate": 9.262646494908604e-05, "loss": 0.45063111186027527, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 39.14, "memory/max_allocated (GiB)": 39.14, "ppl": 1.5693, "step": 116, "tokens/total": 4002960, "tokens/train_per_sec_per_gpu": 50.24, "tokens/trainable": 120560} +{"epoch": 0.45703125, "grad_norm": 2.18035626411438, "learning_rate": 9.246645554956457e-05, "loss": 0.4184834659099579, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.51966, "step": 117, "tokens/total": 4037552, "tokens/train_per_sec_per_gpu": 59.7, "tokens/trainable": 121586} +{"epoch": 0.4609375, "grad_norm": 1.9131234884262085, "learning_rate": 9.230488701789578e-05, "loss": 0.45209115743637085, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.7, "memory/max_allocated (GiB)": 38.7, "ppl": 1.5716, "step": 118, "tokens/total": 4075072, "tokens/train_per_sec_per_gpu": 56.64, "tokens/trainable": 122669} +{"epoch": 0.46484375, "grad_norm": 1.4959824085235596, "learning_rate": 9.214176607760577e-05, "loss": 0.36181291937828064, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.43593, "step": 119, "tokens/total": 4113744, "tokens/train_per_sec_per_gpu": 58.93, "tokens/trainable": 123800} +{"epoch": 0.46875, "grad_norm": 1.6985218524932861, "learning_rate": 9.197709951682268e-05, "loss": 0.3646479845046997, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.44001, "step": 120, "tokens/total": 4146032, "tokens/train_per_sec_per_gpu": 65.54, "tokens/trainable": 124850} +{"epoch": 0.47265625, "grad_norm": 2.015864849090576, "learning_rate": 9.181089418799428e-05, "loss": 0.3961159586906433, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 39.39, "memory/max_allocated (GiB)": 39.39, "ppl": 1.48604, "step": 121, "tokens/total": 4182832, "tokens/train_per_sec_per_gpu": 55.4, "tokens/trainable": 125889} +{"epoch": 0.4765625, "grad_norm": 1.053406000137329, "learning_rate": 9.164315700760271e-05, "loss": 0.3206149935722351, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.37797, "step": 122, "tokens/total": 4218640, "tokens/train_per_sec_per_gpu": 58.81, "tokens/trainable": 126933} +{"epoch": 0.48046875, "grad_norm": 1.2601721286773682, "learning_rate": 9.147389495587671e-05, "loss": 0.39975303411483765, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 1.49146, "step": 123, "tokens/total": 4249792, "tokens/train_per_sec_per_gpu": 69.58, "tokens/trainable": 127964} +{"epoch": 0.484375, "grad_norm": 2.132716417312622, "learning_rate": 9.130311507650116e-05, "loss": 0.3456723093986511, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 1.41294, "step": 124, "tokens/total": 4282464, "tokens/train_per_sec_per_gpu": 62.09, "tokens/trainable": 129019} +{"epoch": 0.48828125, "grad_norm": 6.823373317718506, "learning_rate": 9.113082447632394e-05, "loss": 0.3586992621421814, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.43147, "step": 125, "tokens/total": 4314608, "tokens/train_per_sec_per_gpu": 80.14, "tokens/trainable": 130152} +{"epoch": 0.4921875, "grad_norm": 2.605516195297241, "learning_rate": 9.09570303250602e-05, "loss": 0.39342033863067627, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.47, "memory/max_allocated (GiB)": 36.47, "ppl": 1.48204, "step": 126, "tokens/total": 4346416, "tokens/train_per_sec_per_gpu": 63.75, "tokens/trainable": 131215} +{"epoch": 0.49609375, "grad_norm": 2.544595718383789, "learning_rate": 9.078173985499394e-05, "loss": 0.3742346465587616, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 35.79, "memory/max_allocated (GiB)": 35.79, "ppl": 1.45388, "step": 127, "tokens/total": 4376400, "tokens/train_per_sec_per_gpu": 65.96, "tokens/trainable": 132219} +{"epoch": 0.5, "grad_norm": 1.7290422916412354, "learning_rate": 9.060496036067713e-05, "loss": 0.40714722871780396, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.72, "memory/max_allocated (GiB)": 38.72, "ppl": 1.50253, "step": 128, "tokens/total": 4413824, "tokens/train_per_sec_per_gpu": 48.4, "tokens/trainable": 133233} +{"epoch": 0.50390625, "grad_norm": 2.7035815715789795, "learning_rate": 9.042669919862615e-05, "loss": 0.41184598207473755, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.82, "memory/max_allocated (GiB)": 36.82, "ppl": 1.5096, "step": 129, "tokens/total": 4445936, "tokens/train_per_sec_per_gpu": 68.09, "tokens/trainable": 134285} +{"epoch": 0.5078125, "grad_norm": 1.2055736780166626, "learning_rate": 9.024696378701557e-05, "loss": 0.3472122550010681, "memory/device_reserved (GiB)": 41.42, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.41512, "step": 130, "tokens/total": 4481488, "tokens/train_per_sec_per_gpu": 58.9, "tokens/trainable": 135324} +{"epoch": 0.51171875, "grad_norm": 1.3655428886413574, "learning_rate": 9.006576160536948e-05, "loss": 0.38078492879867554, "memory/device_reserved (GiB)": 41.42, "memory/max_active (GiB)": 37.43, "memory/max_allocated (GiB)": 37.43, "ppl": 1.46343, "step": 131, "tokens/total": 4516048, "tokens/train_per_sec_per_gpu": 56.43, "tokens/trainable": 136314} +{"epoch": 0.515625, "grad_norm": 1.5418059825897217, "learning_rate": 8.988310019425035e-05, "loss": 0.3629196286201477, "memory/device_reserved (GiB)": 41.71, "memory/max_active (GiB)": 39.7, "memory/max_allocated (GiB)": 39.7, "ppl": 1.43752, "step": 132, "tokens/total": 4553904, "tokens/train_per_sec_per_gpu": 51.81, "tokens/trainable": 137324} +{"epoch": 0.51953125, "grad_norm": 2.1173908710479736, "learning_rate": 8.969898715494506e-05, "loss": 0.3343173861503601, "memory/device_reserved (GiB)": 41.73, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.39699, "step": 133, "tokens/total": 4590736, "tokens/train_per_sec_per_gpu": 61.33, "tokens/trainable": 138414} +{"epoch": 0.5234375, "grad_norm": 2.0578227043151855, "learning_rate": 8.951343014914869e-05, "loss": 0.314838171005249, "memory/device_reserved (GiB)": 41.73, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.37004, "step": 134, "tokens/total": 4627392, "tokens/train_per_sec_per_gpu": 55.82, "tokens/trainable": 139490} +{"epoch": 0.52734375, "grad_norm": 1.4677902460098267, "learning_rate": 8.932643689864568e-05, "loss": 0.352867066860199, "memory/device_reserved (GiB)": 41.73, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.42314, "step": 135, "tokens/total": 4663968, "tokens/train_per_sec_per_gpu": 62.34, "tokens/trainable": 140607} +{"epoch": 0.53125, "grad_norm": 1.3080048561096191, "learning_rate": 8.913801518498845e-05, "loss": 0.31144943833351135, "memory/device_reserved (GiB)": 41.73, "memory/max_active (GiB)": 39.68, "memory/max_allocated (GiB)": 39.68, "ppl": 1.3654, "step": 136, "tokens/total": 4703344, "tokens/train_per_sec_per_gpu": 48.84, "tokens/trainable": 141719} +{"epoch": 0.53515625, "grad_norm": 1.657394289970398, "learning_rate": 8.894817284917364e-05, "loss": 0.37022826075553894, "memory/device_reserved (GiB)": 41.73, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.44807, "step": 137, "tokens/total": 4740288, "tokens/train_per_sec_per_gpu": 50.52, "tokens/trainable": 142753} +{"epoch": 0.5390625, "grad_norm": 1.3313266038894653, "learning_rate": 8.875691779131569e-05, "loss": 0.32783979177474976, "memory/device_reserved (GiB)": 41.9, "memory/max_active (GiB)": 39.78, "memory/max_allocated (GiB)": 39.78, "ppl": 1.38797, "step": 138, "tokens/total": 4778224, "tokens/train_per_sec_per_gpu": 54.76, "tokens/trainable": 143828} +{"epoch": 0.54296875, "grad_norm": 1.0938571691513062, "learning_rate": 8.856425797031829e-05, "loss": 0.3021630048751831, "memory/device_reserved (GiB)": 42.22, "memory/max_active (GiB)": 39.58, "memory/max_allocated (GiB)": 39.58, "ppl": 1.35278, "step": 139, "tokens/total": 4814256, "tokens/train_per_sec_per_gpu": 64.09, "tokens/trainable": 144956} +{"epoch": 0.546875, "grad_norm": 1.7217838764190674, "learning_rate": 8.837020140354295e-05, "loss": 0.34812861680984497, "memory/device_reserved (GiB)": 42.22, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.41641, "step": 140, "tokens/total": 4851072, "tokens/train_per_sec_per_gpu": 55.59, "tokens/trainable": 145963} +{"epoch": 0.55078125, "grad_norm": 1.2934406995773315, "learning_rate": 8.817475616647554e-05, "loss": 0.3437759280204773, "memory/device_reserved (GiB)": 42.22, "memory/max_active (GiB)": 38.17, "memory/max_allocated (GiB)": 38.17, "ppl": 1.41026, "step": 141, "tokens/total": 4885504, "tokens/train_per_sec_per_gpu": 60.41, "tokens/trainable": 147019} +{"epoch": 0.5546875, "grad_norm": 1.8335598707199097, "learning_rate": 8.797793039239017e-05, "loss": 0.3055552840232849, "memory/device_reserved (GiB)": 42.22, "memory/max_active (GiB)": 37.83, "memory/max_allocated (GiB)": 37.83, "ppl": 1.35738, "step": 142, "tokens/total": 4920528, "tokens/train_per_sec_per_gpu": 58.77, "tokens/trainable": 148069} +{"epoch": 0.55859375, "grad_norm": 2.049360752105713, "learning_rate": 8.777973227201069e-05, "loss": 0.3471415638923645, "memory/device_reserved (GiB)": 43.76, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.41502, "step": 143, "tokens/total": 4957440, "tokens/train_per_sec_per_gpu": 60.77, "tokens/trainable": 149132} +{"epoch": 0.5625, "grad_norm": 2.1902823448181152, "learning_rate": 8.758017005316988e-05, "loss": 0.379696249961853, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.46184, "step": 144, "tokens/total": 4990752, "tokens/train_per_sec_per_gpu": 73.29, "tokens/trainable": 150174} +{"epoch": 0.56640625, "grad_norm": 4.2294793128967285, "learning_rate": 8.737925204046629e-05, "loss": 0.38111191987991333, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.46391, "step": 145, "tokens/total": 5025472, "tokens/train_per_sec_per_gpu": 55.06, "tokens/trainable": 151234} +{"epoch": 0.5703125, "grad_norm": 1.6949455738067627, "learning_rate": 8.717698659491851e-05, "loss": 0.3243759870529175, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 38.91, "memory/max_allocated (GiB)": 38.91, "ppl": 1.38317, "step": 146, "tokens/total": 5062304, "tokens/train_per_sec_per_gpu": 57.37, "tokens/trainable": 152338} +{"epoch": 0.57421875, "grad_norm": 5.186875820159912, "learning_rate": 8.697338213361735e-05, "loss": 0.32689064741134644, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 36.45, "memory/max_allocated (GiB)": 36.45, "ppl": 1.38665, "step": 147, "tokens/total": 5093280, "tokens/train_per_sec_per_gpu": 68.27, "tokens/trainable": 153425} +{"epoch": 0.578125, "grad_norm": 1.7263407707214355, "learning_rate": 8.676844712937552e-05, "loss": 0.3053937554359436, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.35716, "step": 148, "tokens/total": 5133280, "tokens/train_per_sec_per_gpu": 49.48, "tokens/trainable": 154477} +{"epoch": 0.58203125, "grad_norm": 2.9800102710723877, "learning_rate": 8.656219011037509e-05, "loss": 0.3235231637954712, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.38199, "step": 149, "tokens/total": 5170576, "tokens/train_per_sec_per_gpu": 54.79, "tokens/trainable": 155549} +{"epoch": 0.5859375, "grad_norm": 1.90614914894104, "learning_rate": 8.63546196598125e-05, "loss": 0.3489757776260376, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.41761, "step": 150, "tokens/total": 5208160, "tokens/train_per_sec_per_gpu": 56.12, "tokens/trainable": 156672} +{"epoch": 0.58984375, "grad_norm": 1.638490080833435, "learning_rate": 8.614574441554145e-05, "loss": 0.43270009756088257, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 36.95, "memory/max_allocated (GiB)": 36.95, "ppl": 1.54141, "step": 151, "tokens/total": 5240784, "tokens/train_per_sec_per_gpu": 62.16, "tokens/trainable": 157633} +{"epoch": 0.59375, "grad_norm": 3.827842950820923, "learning_rate": 8.593557306971349e-05, "loss": 0.2985248267650604, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.34787, "step": 152, "tokens/total": 5274096, "tokens/train_per_sec_per_gpu": 58.89, "tokens/trainable": 158666} +{"epoch": 0.59765625, "grad_norm": 2.268885850906372, "learning_rate": 8.572411436841618e-05, "loss": 0.3351712226867676, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 36.9, "memory/max_allocated (GiB)": 36.9, "ppl": 1.39818, "step": 153, "tokens/total": 5307840, "tokens/train_per_sec_per_gpu": 64.72, "tokens/trainable": 159736} +{"epoch": 0.6015625, "grad_norm": 1.1858501434326172, "learning_rate": 8.551137711130922e-05, "loss": 0.3168606162071228, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.37281, "step": 154, "tokens/total": 5339616, "tokens/train_per_sec_per_gpu": 59.44, "tokens/trainable": 160754} +{"epoch": 0.60546875, "grad_norm": 1.6083793640136719, "learning_rate": 8.529737015125824e-05, "loss": 0.286701500415802, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 38.17, "memory/max_allocated (GiB)": 38.17, "ppl": 1.33203, "step": 155, "tokens/total": 5374720, "tokens/train_per_sec_per_gpu": 57.01, "tokens/trainable": 161773} +{"epoch": 0.609375, "grad_norm": 1.4710352420806885, "learning_rate": 8.508210239396639e-05, "loss": 0.3315284848213196, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 39.77, "memory/max_allocated (GiB)": 39.77, "ppl": 1.3931, "step": 156, "tokens/total": 5412608, "tokens/train_per_sec_per_gpu": 57.56, "tokens/trainable": 162790} +{"epoch": 0.61328125, "grad_norm": 1.1887906789779663, "learning_rate": 8.486558279760375e-05, "loss": 0.3344144821166992, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 35.9, "memory/max_allocated (GiB)": 35.9, "ppl": 1.39712, "step": 157, "tokens/total": 5443664, "tokens/train_per_sec_per_gpu": 64.49, "tokens/trainable": 163797} +{"epoch": 0.6171875, "grad_norm": 1.6260888576507568, "learning_rate": 8.464782037243449e-05, "loss": 0.3455880880355835, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.41282, "step": 158, "tokens/total": 5475808, "tokens/train_per_sec_per_gpu": 60.95, "tokens/trainable": 164786} +{"epoch": 0.62109375, "grad_norm": 2.3005332946777344, "learning_rate": 8.442882418044202e-05, "loss": 0.32124653458595276, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.37885, "step": 159, "tokens/total": 5508448, "tokens/train_per_sec_per_gpu": 64.05, "tokens/trainable": 165826} +{"epoch": 0.625, "grad_norm": 1.3653993606567383, "learning_rate": 8.420860333495179e-05, "loss": 0.3038005232810974, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.355, "step": 160, "tokens/total": 5540976, "tokens/train_per_sec_per_gpu": 65.14, "tokens/trainable": 166811} +{"epoch": 0.62890625, "grad_norm": 1.608414888381958, "learning_rate": 8.398716700025208e-05, "loss": 0.31521978974342346, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.37056, "step": 161, "tokens/total": 5574496, "tokens/train_per_sec_per_gpu": 60.36, "tokens/trainable": 167839} +{"epoch": 0.6328125, "grad_norm": 4.883050918579102, "learning_rate": 8.376452439121266e-05, "loss": 0.3180842995643616, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.05, "memory/max_allocated (GiB)": 36.05, "ppl": 1.37449, "step": 162, "tokens/total": 5604544, "tokens/train_per_sec_per_gpu": 65.98, "tokens/trainable": 168809} +{"epoch": 0.63671875, "grad_norm": 2.2623324394226074, "learning_rate": 8.354068477290124e-05, "loss": 0.359663188457489, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.43285, "step": 163, "tokens/total": 5638768, "tokens/train_per_sec_per_gpu": 61.65, "tokens/trainable": 169796} +{"epoch": 0.640625, "grad_norm": 1.934897541999817, "learning_rate": 8.331565746019807e-05, "loss": 0.3251345753669739, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.38422, "step": 164, "tokens/total": 5670960, "tokens/train_per_sec_per_gpu": 69.42, "tokens/trainable": 170835} +{"epoch": 0.64453125, "grad_norm": 1.6263738870620728, "learning_rate": 8.308945181740812e-05, "loss": 0.32054927945137024, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.37788, "step": 165, "tokens/total": 5705664, "tokens/train_per_sec_per_gpu": 62.49, "tokens/trainable": 171844} +{"epoch": 0.6484375, "grad_norm": 1.6601917743682861, "learning_rate": 8.286207725787153e-05, "loss": 0.27799272537231445, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.32048, "step": 166, "tokens/total": 5742672, "tokens/train_per_sec_per_gpu": 51.64, "tokens/trainable": 172871} +{"epoch": 0.65234375, "grad_norm": 1.302856683731079, "learning_rate": 8.263354324357182e-05, "loss": 0.29782307147979736, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.12, "memory/max_allocated (GiB)": 39.12, "ppl": 1.34692, "step": 167, "tokens/total": 5777408, "tokens/train_per_sec_per_gpu": 58.76, "tokens/trainable": 173893} +{"epoch": 0.65625, "grad_norm": 3.21193528175354, "learning_rate": 8.240385928474219e-05, "loss": 0.3523310124874115, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.42238, "step": 168, "tokens/total": 5810224, "tokens/train_per_sec_per_gpu": 58.19, "tokens/trainable": 174853} +{"epoch": 0.66015625, "grad_norm": 1.3919109106063843, "learning_rate": 8.217303493946967e-05, "loss": 0.3239457607269287, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.38257, "step": 169, "tokens/total": 5844432, "tokens/train_per_sec_per_gpu": 58.17, "tokens/trainable": 175851} +{"epoch": 0.6640625, "grad_norm": 1.1815659999847412, "learning_rate": 8.194107981329746e-05, "loss": 0.2972508668899536, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.73, "memory/max_allocated (GiB)": 39.73, "ppl": 1.34615, "step": 170, "tokens/total": 5883008, "tokens/train_per_sec_per_gpu": 51.85, "tokens/trainable": 176872} +{"epoch": 0.66796875, "grad_norm": 9.838041305541992, "learning_rate": 8.170800355882518e-05, "loss": 0.27496230602264404, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.31648, "step": 171, "tokens/total": 5917440, "tokens/train_per_sec_per_gpu": 66.53, "tokens/trainable": 177911} +{"epoch": 0.671875, "grad_norm": 2.2283945083618164, "learning_rate": 8.147381587530713e-05, "loss": 0.3682505786418915, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.39, "memory/max_allocated (GiB)": 37.39, "ppl": 1.4452, "step": 172, "tokens/total": 5951232, "tokens/train_per_sec_per_gpu": 62.15, "tokens/trainable": 178907} +{"epoch": 0.67578125, "grad_norm": 1.5688456296920776, "learning_rate": 8.123852650824877e-05, "loss": 0.3196490406990051, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.45, "memory/max_allocated (GiB)": 37.45, "ppl": 1.37664, "step": 173, "tokens/total": 5985056, "tokens/train_per_sec_per_gpu": 63.28, "tokens/trainable": 179980} +{"epoch": 0.6796875, "grad_norm": 1.994016170501709, "learning_rate": 8.100214524900103e-05, "loss": 0.30690470337867737, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.35921, "step": 174, "tokens/total": 6020128, "tokens/train_per_sec_per_gpu": 52.23, "tokens/trainable": 180969} +{"epoch": 0.68359375, "grad_norm": 1.4258290529251099, "learning_rate": 8.076468193435301e-05, "loss": 0.3177229166030884, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.374, "step": 175, "tokens/total": 6050976, "tokens/train_per_sec_per_gpu": 66.91, "tokens/trainable": 181917} +{"epoch": 0.6875, "grad_norm": 1.515749454498291, "learning_rate": 8.052614644612253e-05, "loss": 0.30854058265686035, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.36144, "step": 176, "tokens/total": 6088816, "tokens/train_per_sec_per_gpu": 50.15, "tokens/trainable": 182967} +{"epoch": 0.69140625, "grad_norm": 6.575046062469482, "learning_rate": 8.028654871074489e-05, "loss": 0.2767651379108429, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.31886, "step": 177, "tokens/total": 6123408, "tokens/train_per_sec_per_gpu": 59.31, "tokens/trainable": 184053} +{"epoch": 0.6953125, "grad_norm": 2.4613037109375, "learning_rate": 8.004589869885986e-05, "loss": 0.3585274815559387, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.37, "memory/max_allocated (GiB)": 36.37, "ppl": 1.43122, "step": 178, "tokens/total": 6154880, "tokens/train_per_sec_per_gpu": 60.91, "tokens/trainable": 184997} +{"epoch": 0.69921875, "grad_norm": 4.618475437164307, "learning_rate": 7.980420642489674e-05, "loss": 0.30662035942077637, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.35883, "step": 179, "tokens/total": 6193584, "tokens/train_per_sec_per_gpu": 53.99, "tokens/trainable": 186029} +{"epoch": 0.703125, "grad_norm": 1.3642772436141968, "learning_rate": 7.95614819466576e-05, "loss": 0.2867480516433716, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 1.33209, "step": 180, "tokens/total": 6227296, "tokens/train_per_sec_per_gpu": 59.41, "tokens/trainable": 187082} +{"epoch": 0.70703125, "grad_norm": 1.1442806720733643, "learning_rate": 7.931773536489872e-05, "loss": 0.28881388902664185, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.33484, "step": 181, "tokens/total": 6261888, "tokens/train_per_sec_per_gpu": 63.04, "tokens/trainable": 188129} +{"epoch": 0.7109375, "grad_norm": 1.5651525259017944, "learning_rate": 7.907297682291035e-05, "loss": 0.2837446331977844, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.32809, "step": 182, "tokens/total": 6296544, "tokens/train_per_sec_per_gpu": 61.12, "tokens/trainable": 189201} +{"epoch": 0.71484375, "grad_norm": 1.2270643711090088, "learning_rate": 7.882721650609442e-05, "loss": 0.2463228702545166, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.65, "memory/max_allocated (GiB)": 39.65, "ppl": 1.27931, "step": 183, "tokens/total": 6336960, "tokens/train_per_sec_per_gpu": 53.69, "tokens/trainable": 190379} +{"epoch": 0.71875, "grad_norm": 1.4898606538772583, "learning_rate": 7.85804646415409e-05, "loss": 0.2975180149078369, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.34651, "step": 184, "tokens/total": 6372192, "tokens/train_per_sec_per_gpu": 64.59, "tokens/trainable": 191465} +{"epoch": 0.72265625, "grad_norm": 1.3481049537658691, "learning_rate": 7.833273149760207e-05, "loss": 0.2856843173503876, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.33067, "step": 185, "tokens/total": 6403056, "tokens/train_per_sec_per_gpu": 60.75, "tokens/trainable": 192471} +{"epoch": 0.7265625, "grad_norm": 2.1623990535736084, "learning_rate": 7.808402738346527e-05, "loss": 0.2529829740524292, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.11, "memory/max_allocated (GiB)": 39.11, "ppl": 1.28786, "step": 186, "tokens/total": 6441840, "tokens/train_per_sec_per_gpu": 58.54, "tokens/trainable": 193603} +{"epoch": 0.73046875, "grad_norm": 7.214691162109375, "learning_rate": 7.783436264872382e-05, "loss": 0.27563101053237915, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.31736, "step": 187, "tokens/total": 6478688, "tokens/train_per_sec_per_gpu": 56.43, "tokens/trainable": 194652} +{"epoch": 0.734375, "grad_norm": 1.2944799661636353, "learning_rate": 7.758374768294647e-05, "loss": 0.27000290155410767, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.16, "memory/max_allocated (GiB)": 38.16, "ppl": 1.30997, "step": 188, "tokens/total": 6514624, "tokens/train_per_sec_per_gpu": 61.22, "tokens/trainable": 195747} +{"epoch": 0.73828125, "grad_norm": 1.549026608467102, "learning_rate": 7.733219291524489e-05, "loss": 0.2727423310279846, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.53, "memory/max_allocated (GiB)": 37.53, "ppl": 1.31356, "step": 189, "tokens/total": 6548048, "tokens/train_per_sec_per_gpu": 62.82, "tokens/trainable": 196784} +{"epoch": 0.7421875, "grad_norm": 1.3225897550582886, "learning_rate": 7.707970881383977e-05, "loss": 0.2802436351776123, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.45, "memory/max_allocated (GiB)": 36.45, "ppl": 1.32345, "step": 190, "tokens/total": 6579760, "tokens/train_per_sec_per_gpu": 67.88, "tokens/trainable": 197799} +{"epoch": 0.74609375, "grad_norm": 1.7421603202819824, "learning_rate": 7.682630588562518e-05, "loss": 0.26991572976112366, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.7, "memory/max_allocated (GiB)": 38.7, "ppl": 1.30985, "step": 191, "tokens/total": 6614800, "tokens/train_per_sec_per_gpu": 57.94, "tokens/trainable": 198860} +{"epoch": 0.75, "grad_norm": 1.1021791696548462, "learning_rate": 7.657199467573129e-05, "loss": 0.2601410746574402, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.29711, "step": 192, "tokens/total": 6648928, "tokens/train_per_sec_per_gpu": 56.09, "tokens/trainable": 199872} +{"epoch": 0.75390625, "grad_norm": 0.9897380471229553, "learning_rate": 7.631678576708561e-05, "loss": 0.23512399196624756, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.16, "memory/max_allocated (GiB)": 38.16, "ppl": 1.26507, "step": 193, "tokens/total": 6683328, "tokens/train_per_sec_per_gpu": 67.77, "tokens/trainable": 200966} +{"epoch": 0.7578125, "grad_norm": 2.1603200435638428, "learning_rate": 7.606068977997255e-05, "loss": 0.2821153998374939, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.64, "memory/max_allocated (GiB)": 39.64, "ppl": 1.32593, "step": 194, "tokens/total": 6720928, "tokens/train_per_sec_per_gpu": 66.52, "tokens/trainable": 202132} +{"epoch": 0.76171875, "grad_norm": 1.4393881559371948, "learning_rate": 7.580371737159148e-05, "loss": 0.2935802638530731, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.34122, "step": 195, "tokens/total": 6757024, "tokens/train_per_sec_per_gpu": 57.1, "tokens/trainable": 203179} +{"epoch": 0.765625, "grad_norm": 1.9730682373046875, "learning_rate": 7.554587923561324e-05, "loss": 0.2608520984649658, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.99, "memory/max_allocated (GiB)": 38.99, "ppl": 1.29804, "step": 196, "tokens/total": 6791728, "tokens/train_per_sec_per_gpu": 52.06, "tokens/trainable": 204151} +{"epoch": 0.76953125, "grad_norm": 1.844822645187378, "learning_rate": 7.528718610173511e-05, "loss": 0.28630387783050537, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.3315, "step": 197, "tokens/total": 6827072, "tokens/train_per_sec_per_gpu": 63.6, "tokens/trainable": 205245} +{"epoch": 0.7734375, "grad_norm": 1.8306065797805786, "learning_rate": 7.502764873523431e-05, "loss": 0.31949836015701294, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.37644, "step": 198, "tokens/total": 6861120, "tokens/train_per_sec_per_gpu": 60.88, "tokens/trainable": 206265} +{"epoch": 0.77734375, "grad_norm": 1.3089450597763062, "learning_rate": 7.476727793652011e-05, "loss": 0.25220197439193726, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.72, "memory/max_allocated (GiB)": 36.72, "ppl": 1.28686, "step": 199, "tokens/total": 6894352, "tokens/train_per_sec_per_gpu": 62.99, "tokens/trainable": 207320} +{"epoch": 0.78125, "grad_norm": 1.3447692394256592, "learning_rate": 7.450608454068415e-05, "loss": 0.23094606399536133, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.25979, "step": 200, "tokens/total": 6928848, "tokens/train_per_sec_per_gpu": 58.19, "tokens/trainable": 208367} +{"epoch": 0.78515625, "grad_norm": 5.161012172698975, "learning_rate": 7.424407941704987e-05, "loss": 0.2514325976371765, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.49, "memory/max_allocated (GiB)": 38.49, "ppl": 1.28587, "step": 201, "tokens/total": 6965504, "tokens/train_per_sec_per_gpu": 67.61, "tokens/trainable": 209512} +{"epoch": 0.7890625, "grad_norm": 1.8694978952407837, "learning_rate": 7.398127346871986e-05, "loss": 0.25756964087486267, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.61, "memory/max_allocated (GiB)": 39.61, "ppl": 1.29378, "step": 202, "tokens/total": 7002448, "tokens/train_per_sec_per_gpu": 55.34, "tokens/trainable": 210555} +{"epoch": 0.79296875, "grad_norm": 1.8911043405532837, "learning_rate": 7.371767763212238e-05, "loss": 0.2849327623844147, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.52, "memory/max_allocated (GiB)": 39.52, "ppl": 1.32967, "step": 203, "tokens/total": 7037824, "tokens/train_per_sec_per_gpu": 62.86, "tokens/trainable": 211606} +{"epoch": 0.796875, "grad_norm": 4.763019561767578, "learning_rate": 7.345330287655617e-05, "loss": 0.3103441298007965, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.09, "memory/max_allocated (GiB)": 39.09, "ppl": 1.36389, "step": 204, "tokens/total": 7075920, "tokens/train_per_sec_per_gpu": 48.98, "tokens/trainable": 212641} +{"epoch": 0.80078125, "grad_norm": 1.27324640750885, "learning_rate": 7.31881602037339e-05, "loss": 0.2976059019565582, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.69, "memory/max_allocated (GiB)": 36.69, "ppl": 1.34663, "step": 205, "tokens/total": 7107888, "tokens/train_per_sec_per_gpu": 61.98, "tokens/trainable": 213644} +{"epoch": 0.8046875, "grad_norm": 1.647110939025879, "learning_rate": 7.29222606473245e-05, "loss": 0.2700284719467163, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.31, "step": 206, "tokens/total": 7140720, "tokens/train_per_sec_per_gpu": 61.75, "tokens/trainable": 214662} +{"epoch": 0.80859375, "grad_norm": 6.513057708740234, "learning_rate": 7.265561527249383e-05, "loss": 0.2633044421672821, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.30122, "step": 207, "tokens/total": 7174720, "tokens/train_per_sec_per_gpu": 62.47, "tokens/trainable": 215658} +{"epoch": 0.8125, "grad_norm": 0.988166332244873, "learning_rate": 7.238823517544436e-05, "loss": 0.20543840527534485, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.22806, "step": 208, "tokens/total": 7211872, "tokens/train_per_sec_per_gpu": 67.65, "tokens/trainable": 216826} +{"epoch": 0.81640625, "grad_norm": 1.79848313331604, "learning_rate": 7.212013148295333e-05, "loss": 0.26223671436309814, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.29983, "step": 209, "tokens/total": 7245856, "tokens/train_per_sec_per_gpu": 63.61, "tokens/trainable": 217905} +{"epoch": 0.8203125, "grad_norm": 2.392733573913574, "learning_rate": 7.185131535190975e-05, "loss": 0.2731034755706787, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.06, "memory/max_allocated (GiB)": 37.06, "ppl": 1.31404, "step": 210, "tokens/total": 7278448, "tokens/train_per_sec_per_gpu": 62.86, "tokens/trainable": 218966} +{"epoch": 0.82421875, "grad_norm": 12.703551292419434, "learning_rate": 7.158179796885005e-05, "loss": 0.24066607654094696, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.2721, "step": 211, "tokens/total": 7313072, "tokens/train_per_sec_per_gpu": 62.46, "tokens/trainable": 220019} +{"epoch": 0.828125, "grad_norm": 1.6516435146331787, "learning_rate": 7.131159054949273e-05, "loss": 0.2838118374347687, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.32818, "step": 212, "tokens/total": 7351456, "tokens/train_per_sec_per_gpu": 55.12, "tokens/trainable": 221058} +{"epoch": 0.83203125, "grad_norm": 1.1769206523895264, "learning_rate": 7.104070433827139e-05, "loss": 0.27810239791870117, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.32062, "step": 213, "tokens/total": 7386880, "tokens/train_per_sec_per_gpu": 50.52, "tokens/trainable": 221992} +{"epoch": 0.8359375, "grad_norm": 2.875775098800659, "learning_rate": 7.076915060786705e-05, "loss": 0.2775426506996155, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.31988, "step": 214, "tokens/total": 7419696, "tokens/train_per_sec_per_gpu": 62.29, "tokens/trainable": 223033} +{"epoch": 0.83984375, "grad_norm": 1.648599624633789, "learning_rate": 7.049694065873882e-05, "loss": 0.26142239570617676, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.07, "memory/max_allocated (GiB)": 38.07, "ppl": 1.29878, "step": 215, "tokens/total": 7454352, "tokens/train_per_sec_per_gpu": 56.28, "tokens/trainable": 224065} +{"epoch": 0.84375, "grad_norm": 1.501273512840271, "learning_rate": 7.022408581865382e-05, "loss": 0.22760166227817535, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.25559, "step": 216, "tokens/total": 7488496, "tokens/train_per_sec_per_gpu": 58.15, "tokens/trainable": 225142} +{"epoch": 0.84765625, "grad_norm": 1.5326529741287231, "learning_rate": 6.99505974422157e-05, "loss": 0.22531510889530182, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.25272, "step": 217, "tokens/total": 7522976, "tokens/train_per_sec_per_gpu": 65.99, "tokens/trainable": 226224} +{"epoch": 0.8515625, "grad_norm": 1.482715368270874, "learning_rate": 6.967648691039213e-05, "loss": 0.2497396320104599, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.58, "memory/max_allocated (GiB)": 36.58, "ppl": 1.28369, "step": 218, "tokens/total": 7554848, "tokens/train_per_sec_per_gpu": 64.24, "tokens/trainable": 227230} +{"epoch": 0.85546875, "grad_norm": 5.276981353759766, "learning_rate": 6.940176563004123e-05, "loss": 0.2550862431526184, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.29057, "step": 219, "tokens/total": 7588896, "tokens/train_per_sec_per_gpu": 57.75, "tokens/trainable": 228203} +{"epoch": 0.859375, "grad_norm": 1.7354305982589722, "learning_rate": 6.912644503343682e-05, "loss": 0.28252705931663513, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.32648, "step": 220, "tokens/total": 7627056, "tokens/train_per_sec_per_gpu": 55.87, "tokens/trainable": 229212} +{"epoch": 0.86328125, "grad_norm": 1.5474458932876587, "learning_rate": 6.885053657779273e-05, "loss": 0.2330506443977356, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.28, "memory/max_allocated (GiB)": 36.28, "ppl": 1.26245, "step": 221, "tokens/total": 7658368, "tokens/train_per_sec_per_gpu": 60.2, "tokens/trainable": 230227} +{"epoch": 0.8671875, "grad_norm": 2.051044464111328, "learning_rate": 6.857405174478604e-05, "loss": 0.258545458316803, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.29505, "step": 222, "tokens/total": 7693072, "tokens/train_per_sec_per_gpu": 58.19, "tokens/trainable": 231237} +{"epoch": 0.87109375, "grad_norm": 1.1779552698135376, "learning_rate": 6.82970020400792e-05, "loss": 0.2420659065246582, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.27388, "step": 223, "tokens/total": 7730288, "tokens/train_per_sec_per_gpu": 52.98, "tokens/trainable": 232255} +{"epoch": 0.875, "grad_norm": 1.6620694398880005, "learning_rate": 6.801939899284132e-05, "loss": 0.19923630356788635, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.22047, "step": 224, "tokens/total": 7766304, "tokens/train_per_sec_per_gpu": 58.85, "tokens/trainable": 233334} +{"epoch": 0.87890625, "grad_norm": 1.620474934577942, "learning_rate": 6.774125415526827e-05, "loss": 0.20462489128112793, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.22706, "step": 225, "tokens/total": 7799680, "tokens/train_per_sec_per_gpu": 71.27, "tokens/trainable": 234472} +{"epoch": 0.8828125, "grad_norm": 1.5776616334915161, "learning_rate": 6.746257910210214e-05, "loss": 0.230505108833313, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.25924, "step": 226, "tokens/total": 7832912, "tokens/train_per_sec_per_gpu": 59.74, "tokens/trainable": 235484} +{"epoch": 0.88671875, "grad_norm": 2.950582981109619, "learning_rate": 6.718338543014937e-05, "loss": 0.2726192772388458, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 35.66, "memory/max_allocated (GiB)": 35.66, "ppl": 1.3134, "step": 227, "tokens/total": 7862912, "tokens/train_per_sec_per_gpu": 64.96, "tokens/trainable": 236464} +{"epoch": 0.890625, "grad_norm": 1.6722766160964966, "learning_rate": 6.69036847577983e-05, "loss": 0.22798946499824524, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.25607, "step": 228, "tokens/total": 7896816, "tokens/train_per_sec_per_gpu": 61.56, "tokens/trainable": 237493} +{"epoch": 0.89453125, "grad_norm": 1.636664628982544, "learning_rate": 6.662348872453553e-05, "loss": 0.2088695466518402, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.46, "memory/max_allocated (GiB)": 38.46, "ppl": 1.23228, "step": 229, "tokens/total": 7933376, "tokens/train_per_sec_per_gpu": 61.24, "tokens/trainable": 238561} +{"epoch": 0.8984375, "grad_norm": 3.808692455291748, "learning_rate": 6.63428089904618e-05, "loss": 0.23629620671272278, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.26655, "step": 230, "tokens/total": 7972544, "tokens/train_per_sec_per_gpu": 51.67, "tokens/trainable": 239581} +{"epoch": 0.90234375, "grad_norm": 3.463747501373291, "learning_rate": 6.60616572358065e-05, "loss": 0.22473330795764923, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.25199, "step": 231, "tokens/total": 8007008, "tokens/train_per_sec_per_gpu": 61.0, "tokens/trainable": 240536} +{"epoch": 0.90625, "grad_norm": 1.4986932277679443, "learning_rate": 6.578004516044172e-05, "loss": 0.2269822061061859, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.25481, "step": 232, "tokens/total": 8042864, "tokens/train_per_sec_per_gpu": 62.51, "tokens/trainable": 241574} +{"epoch": 0.91015625, "grad_norm": 1.529134750366211, "learning_rate": 6.549798448339548e-05, "loss": 0.2166164517402649, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.6, "memory/max_allocated (GiB)": 38.6, "ppl": 1.24187, "step": 233, "tokens/total": 8080272, "tokens/train_per_sec_per_gpu": 60.6, "tokens/trainable": 242630} +{"epoch": 0.9140625, "grad_norm": 1.6166729927062988, "learning_rate": 6.521548694236384e-05, "loss": 0.24359600245952606, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.27583, "step": 234, "tokens/total": 8117584, "tokens/train_per_sec_per_gpu": 53.1, "tokens/trainable": 243663} +{"epoch": 0.91796875, "grad_norm": 1.4676555395126343, "learning_rate": 6.493256429322259e-05, "loss": 0.22984269261360168, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.2584, "step": 235, "tokens/total": 8153712, "tokens/train_per_sec_per_gpu": 55.82, "tokens/trainable": 244690} +{"epoch": 0.921875, "grad_norm": 1.5265405178070068, "learning_rate": 6.464922830953799e-05, "loss": 0.22613203525543213, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.25374, "step": 236, "tokens/total": 8192928, "tokens/train_per_sec_per_gpu": 52.36, "tokens/trainable": 245714} +{"epoch": 0.92578125, "grad_norm": 2.180001735687256, "learning_rate": 6.436549078207688e-05, "loss": 0.23356837034225464, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.21, "memory/max_allocated (GiB)": 38.21, "ppl": 1.2631, "step": 237, "tokens/total": 8226016, "tokens/train_per_sec_per_gpu": 62.6, "tokens/trainable": 246714} +{"epoch": 0.9296875, "grad_norm": 4.022181034088135, "learning_rate": 6.408136351831592e-05, "loss": 0.23367580771446228, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 37.15, "memory/max_allocated (GiB)": 37.15, "ppl": 1.26323, "step": 238, "tokens/total": 8258384, "tokens/train_per_sec_per_gpu": 60.47, "tokens/trainable": 247711} +{"epoch": 0.93359375, "grad_norm": 2.130216598510742, "learning_rate": 6.379685834195036e-05, "loss": 0.20780929923057556, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.23098, "step": 239, "tokens/total": 8297888, "tokens/train_per_sec_per_gpu": 54.79, "tokens/trainable": 248851} +{"epoch": 0.9375, "grad_norm": 1.4863814115524292, "learning_rate": 6.351198709240186e-05, "loss": 0.21391642093658447, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.61, "memory/max_allocated (GiB)": 39.61, "ppl": 1.23852, "step": 240, "tokens/total": 8338016, "tokens/train_per_sec_per_gpu": 52.87, "tokens/trainable": 249925} +{"epoch": 0.94140625, "grad_norm": 1.2273236513137817, "learning_rate": 6.32267616243259e-05, "loss": 0.2333485186100006, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.26282, "step": 241, "tokens/total": 8370608, "tokens/train_per_sec_per_gpu": 63.88, "tokens/trainable": 250955} +{"epoch": 0.9453125, "grad_norm": 1.2458839416503906, "learning_rate": 6.294119380711849e-05, "loss": 0.24688571691513062, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.28003, "step": 242, "tokens/total": 8405040, "tokens/train_per_sec_per_gpu": 52.05, "tokens/trainable": 251905} +{"epoch": 0.94921875, "grad_norm": 1.247441291809082, "learning_rate": 6.265529552442209e-05, "loss": 0.22258692979812622, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.2493, "step": 243, "tokens/total": 8440400, "tokens/train_per_sec_per_gpu": 63.37, "tokens/trainable": 252963} +{"epoch": 0.953125, "grad_norm": 3.1852219104766846, "learning_rate": 6.236907867363127e-05, "loss": 0.2212781012058258, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.14, "memory/max_allocated (GiB)": 38.14, "ppl": 1.24767, "step": 244, "tokens/total": 8474736, "tokens/train_per_sec_per_gpu": 63.74, "tokens/trainable": 253997} +{"epoch": 0.95703125, "grad_norm": 1.760864496231079, "learning_rate": 6.208255516539749e-05, "loss": 0.2092340588569641, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 36.34, "memory/max_allocated (GiB)": 36.34, "ppl": 1.23273, "step": 245, "tokens/total": 8506944, "tokens/train_per_sec_per_gpu": 60.83, "tokens/trainable": 255054} +{"epoch": 0.9609375, "grad_norm": 3.1002767086029053, "learning_rate": 6.179573692313344e-05, "loss": 0.23031076788902283, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 1.25899, "step": 246, "tokens/total": 8544064, "tokens/train_per_sec_per_gpu": 57.97, "tokens/trainable": 256069} +{"epoch": 0.96484375, "grad_norm": 1.6865888833999634, "learning_rate": 6.150863588251694e-05, "loss": 0.22965802252292633, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.25817, "step": 247, "tokens/total": 8584336, "tokens/train_per_sec_per_gpu": 50.51, "tokens/trainable": 257125} +{"epoch": 0.96875, "grad_norm": 1.5385297536849976, "learning_rate": 6.122126399099419e-05, "loss": 0.20300200581550598, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.78, "memory/max_allocated (GiB)": 39.78, "ppl": 1.22507, "step": 248, "tokens/total": 8624304, "tokens/train_per_sec_per_gpu": 66.71, "tokens/trainable": 258381} +{"epoch": 0.97265625, "grad_norm": 1.4650646448135376, "learning_rate": 6.0933633207282615e-05, "loss": 0.2575608491897583, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.29377, "step": 249, "tokens/total": 8660672, "tokens/train_per_sec_per_gpu": 55.18, "tokens/trainable": 259442} +{"epoch": 0.9765625, "grad_norm": 4.10271692276001, "learning_rate": 6.064575550087316e-05, "loss": 0.22620242834091187, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.63, "memory/max_allocated (GiB)": 39.63, "ppl": 1.25383, "step": 250, "tokens/total": 8697680, "tokens/train_per_sec_per_gpu": 56.78, "tokens/trainable": 260475} +{"epoch": 0.98046875, "grad_norm": 1.3021742105484009, "learning_rate": 6.0357642851532245e-05, "loss": 0.2290305197238922, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.25738, "step": 251, "tokens/total": 8734112, "tokens/train_per_sec_per_gpu": 55.25, "tokens/trainable": 261543} +{"epoch": 0.984375, "grad_norm": 1.4279499053955078, "learning_rate": 6.0069307248803294e-05, "loss": 0.24360539019107819, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.27584, "step": 252, "tokens/total": 8769440, "tokens/train_per_sec_per_gpu": 50.75, "tokens/trainable": 262520} +{"epoch": 0.98828125, "grad_norm": 1.6620960235595703, "learning_rate": 5.9780760691507635e-05, "loss": 0.24459655582904816, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 36.25, "memory/max_allocated (GiB)": 36.25, "ppl": 1.27711, "step": 253, "tokens/total": 8800688, "tokens/train_per_sec_per_gpu": 61.91, "tokens/trainable": 263463} +{"epoch": 0.9921875, "grad_norm": 0.8768177628517151, "learning_rate": 5.9492015187245334e-05, "loss": 0.2204531878232956, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.72, "memory/max_allocated (GiB)": 38.72, "ppl": 1.24664, "step": 254, "tokens/total": 8838272, "tokens/train_per_sec_per_gpu": 55.06, "tokens/trainable": 264428} +{"epoch": 0.99609375, "grad_norm": 1.0935882329940796, "learning_rate": 5.920308275189541e-05, "loss": 0.203982412815094, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 36.6, "memory/max_allocated (GiB)": 36.6, "ppl": 1.22628, "step": 255, "tokens/total": 8869952, "tokens/train_per_sec_per_gpu": 61.38, "tokens/trainable": 265415} +{"epoch": 1.0, "grad_norm": 4.6748151779174805, "learning_rate": 5.8913975409115874e-05, "loss": 0.20391812920570374, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.2262, "step": 256, "tokens/total": 8903312, "tokens/train_per_sec_per_gpu": 62.64, "tokens/trainable": 266474} +{"epoch": 1.00390625, "grad_norm": 9.27450942993164, "learning_rate": 5.8624705189843395e-05, "loss": 0.18092036247253418, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.19832, "step": 257, "tokens/total": 8940768, "tokens/train_per_sec_per_gpu": 55.96, "tokens/trainable": 267581} +{"epoch": 1.0078125, "grad_norm": 1.1699926853179932, "learning_rate": 5.833528413179249e-05, "loss": 0.1838330775499344, "memory/device_reserved (GiB)": 40.67, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.20182, "step": 258, "tokens/total": 8977392, "tokens/train_per_sec_per_gpu": 58.87, "tokens/trainable": 268673} +{"epoch": 1.01171875, "grad_norm": 1.1014879941940308, "learning_rate": 5.80457242789548e-05, "loss": 0.18248853087425232, "memory/device_reserved (GiB)": 42.74, "memory/max_active (GiB)": 39.65, "memory/max_allocated (GiB)": 39.65, "ppl": 1.2002, "step": 259, "tokens/total": 9017568, "tokens/train_per_sec_per_gpu": 57.48, "tokens/trainable": 269843} +{"epoch": 1.015625, "grad_norm": 0.6703189015388489, "learning_rate": 5.77560376810977e-05, "loss": 0.1715172827243805, "memory/device_reserved (GiB)": 42.75, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.1871, "step": 260, "tokens/total": 9050928, "tokens/train_per_sec_per_gpu": 67.4, "tokens/trainable": 270933} +{"epoch": 1.01953125, "grad_norm": 1.8020555973052979, "learning_rate": 5.7466236393263005e-05, "loss": 0.2180650532245636, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.24367, "step": 261, "tokens/total": 9085552, "tokens/train_per_sec_per_gpu": 60.8, "tokens/trainable": 271950} +{"epoch": 1.0234375, "grad_norm": 2.1155898571014404, "learning_rate": 5.717633247526522e-05, "loss": 0.2124282866716385, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 36.74, "memory/max_allocated (GiB)": 36.74, "ppl": 1.23668, "step": 262, "tokens/total": 9116240, "tokens/train_per_sec_per_gpu": 58.53, "tokens/trainable": 272872} +{"epoch": 1.02734375, "grad_norm": 1.505613088607788, "learning_rate": 5.688633799118971e-05, "loss": 0.23738402128219604, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.26793, "step": 263, "tokens/total": 9148688, "tokens/train_per_sec_per_gpu": 54.02, "tokens/trainable": 273796} +{"epoch": 1.03125, "grad_norm": 0.7860997319221497, "learning_rate": 5.659626500889066e-05, "loss": 0.17373746633529663, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.09, "memory/max_allocated (GiB)": 37.09, "ppl": 1.18974, "step": 264, "tokens/total": 9182704, "tokens/train_per_sec_per_gpu": 64.94, "tokens/trainable": 274922} +{"epoch": 1.03515625, "grad_norm": 1.3458514213562012, "learning_rate": 5.6306125599488905e-05, "loss": 0.19639548659324646, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 36.88, "memory/max_allocated (GiB)": 36.88, "ppl": 1.21701, "step": 265, "tokens/total": 9213552, "tokens/train_per_sec_per_gpu": 68.28, "tokens/trainable": 275931} +{"epoch": 1.0390625, "grad_norm": 1.3704349994659424, "learning_rate": 5.601593183686955e-05, "loss": 0.16486120223999023, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.17923, "step": 266, "tokens/total": 9250768, "tokens/train_per_sec_per_gpu": 60.18, "tokens/trainable": 277050} +{"epoch": 1.04296875, "grad_norm": 1.8749146461486816, "learning_rate": 5.572569579717961e-05, "loss": 0.2005346715450287, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.22206, "step": 267, "tokens/total": 9286176, "tokens/train_per_sec_per_gpu": 61.83, "tokens/trainable": 278138} +{"epoch": 1.046875, "grad_norm": 0.8731603622436523, "learning_rate": 5.543542955832538e-05, "loss": 0.17677512764930725, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.19336, "step": 268, "tokens/total": 9326128, "tokens/train_per_sec_per_gpu": 52.87, "tokens/trainable": 279221} +{"epoch": 1.05078125, "grad_norm": 2.1188220977783203, "learning_rate": 5.514514519946986e-05, "loss": 0.17572173476219177, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.57, "memory/max_allocated (GiB)": 39.57, "ppl": 1.19211, "step": 269, "tokens/total": 9366544, "tokens/train_per_sec_per_gpu": 57.17, "tokens/trainable": 280342} +{"epoch": 1.0546875, "grad_norm": 1.558510661125183, "learning_rate": 5.485485480053015e-05, "loss": 0.20282000303268433, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.52, "memory/max_allocated (GiB)": 39.52, "ppl": 1.22485, "step": 270, "tokens/total": 9404928, "tokens/train_per_sec_per_gpu": 48.2, "tokens/trainable": 281326} +{"epoch": 1.05859375, "grad_norm": 1.6004544496536255, "learning_rate": 5.4564570441674645e-05, "loss": 0.18500259518623352, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.20322, "step": 271, "tokens/total": 9438592, "tokens/train_per_sec_per_gpu": 65.02, "tokens/trainable": 282349} +{"epoch": 1.0625, "grad_norm": 1.8219082355499268, "learning_rate": 5.42743042028204e-05, "loss": 0.21714189648628235, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.83, "memory/max_allocated (GiB)": 37.83, "ppl": 1.24252, "step": 272, "tokens/total": 9474144, "tokens/train_per_sec_per_gpu": 47.18, "tokens/trainable": 283295} +{"epoch": 1.06640625, "grad_norm": 1.1150498390197754, "learning_rate": 5.3984068163130464e-05, "loss": 0.16713400185108185, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.18191, "step": 273, "tokens/total": 9507904, "tokens/train_per_sec_per_gpu": 67.82, "tokens/trainable": 284457} +{"epoch": 1.0703125, "grad_norm": 1.2599883079528809, "learning_rate": 5.369387440051111e-05, "loss": 0.21130608022212982, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.18, "memory/max_allocated (GiB)": 37.18, "ppl": 1.23529, "step": 274, "tokens/total": 9541616, "tokens/train_per_sec_per_gpu": 59.07, "tokens/trainable": 285459} +{"epoch": 1.07421875, "grad_norm": 2.2802302837371826, "learning_rate": 5.340373499110935e-05, "loss": 0.18705835938453674, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.2057, "step": 275, "tokens/total": 9577424, "tokens/train_per_sec_per_gpu": 58.14, "tokens/trainable": 286527} +{"epoch": 1.078125, "grad_norm": 1.3223780393600464, "learning_rate": 5.3113662008810304e-05, "loss": 0.1718381941318512, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.18749, "step": 276, "tokens/total": 9617760, "tokens/train_per_sec_per_gpu": 50.6, "tokens/trainable": 287587} +{"epoch": 1.08203125, "grad_norm": 1.2818269729614258, "learning_rate": 5.282366752473479e-05, "loss": 0.17972983419895172, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.29, "memory/max_allocated (GiB)": 38.29, "ppl": 1.19689, "step": 277, "tokens/total": 9655008, "tokens/train_per_sec_per_gpu": 58.24, "tokens/trainable": 288688} +{"epoch": 1.0859375, "grad_norm": 1.3531898260116577, "learning_rate": 5.2533763606737005e-05, "loss": 0.2059614509344101, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.28, "memory/max_allocated (GiB)": 39.28, "ppl": 1.22871, "step": 278, "tokens/total": 9694896, "tokens/train_per_sec_per_gpu": 47.08, "tokens/trainable": 289684} +{"epoch": 1.08984375, "grad_norm": 2.9978413581848145, "learning_rate": 5.224396231890232e-05, "loss": 0.17854365706443787, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.23, "memory/max_allocated (GiB)": 37.23, "ppl": 1.19548, "step": 279, "tokens/total": 9726672, "tokens/train_per_sec_per_gpu": 67.11, "tokens/trainable": 290798} +{"epoch": 1.09375, "grad_norm": 1.2883415222167969, "learning_rate": 5.195427572104522e-05, "loss": 0.20204734802246094, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.22391, "step": 280, "tokens/total": 9761136, "tokens/train_per_sec_per_gpu": 59.28, "tokens/trainable": 291826} +{"epoch": 1.09765625, "grad_norm": 1.1415398120880127, "learning_rate": 5.166471586820751e-05, "loss": 0.1928541213274002, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.21271, "step": 281, "tokens/total": 9797888, "tokens/train_per_sec_per_gpu": 61.51, "tokens/trainable": 292919} +{"epoch": 1.1015625, "grad_norm": 1.9137243032455444, "learning_rate": 5.1375294810156615e-05, "loss": 0.179019033908844, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.19604, "step": 282, "tokens/total": 9832528, "tokens/train_per_sec_per_gpu": 60.14, "tokens/trainable": 293941} +{"epoch": 1.10546875, "grad_norm": 1.2594971656799316, "learning_rate": 5.1086024590884144e-05, "loss": 0.1972804069519043, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.28, "memory/max_allocated (GiB)": 39.28, "ppl": 1.21809, "step": 283, "tokens/total": 9870784, "tokens/train_per_sec_per_gpu": 53.06, "tokens/trainable": 294942} +{"epoch": 1.109375, "grad_norm": 0.5550968050956726, "learning_rate": 5.079691724810461e-05, "loss": 0.17720595002174377, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.19388, "step": 284, "tokens/total": 9906656, "tokens/train_per_sec_per_gpu": 49.55, "tokens/trainable": 295911} +{"epoch": 1.11328125, "grad_norm": 1.1394810676574707, "learning_rate": 5.0507984812754684e-05, "loss": 0.19719046354293823, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.21798, "step": 285, "tokens/total": 9940208, "tokens/train_per_sec_per_gpu": 59.9, "tokens/trainable": 296958} +{"epoch": 1.1171875, "grad_norm": 1.4170854091644287, "learning_rate": 5.021923930849237e-05, "loss": 0.18914976716041565, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.20822, "step": 286, "tokens/total": 9974096, "tokens/train_per_sec_per_gpu": 57.04, "tokens/trainable": 297971} +{"epoch": 1.12109375, "grad_norm": 1.11118745803833, "learning_rate": 4.99306927511967e-05, "loss": 0.18300053477287292, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.20082, "step": 287, "tokens/total": 10007568, "tokens/train_per_sec_per_gpu": 62.63, "tokens/trainable": 298978} +{"epoch": 1.125, "grad_norm": 0.9984815120697021, "learning_rate": 4.964235714846775e-05, "loss": 0.19702857732772827, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 36.44, "memory/max_allocated (GiB)": 36.44, "ppl": 1.21778, "step": 288, "tokens/total": 10039280, "tokens/train_per_sec_per_gpu": 66.59, "tokens/trainable": 299972} +{"epoch": 1.12890625, "grad_norm": 3.5843019485473633, "learning_rate": 4.9354244499126866e-05, "loss": 0.21330749988555908, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 36.23, "memory/max_allocated (GiB)": 36.23, "ppl": 1.23777, "step": 289, "tokens/total": 10070944, "tokens/train_per_sec_per_gpu": 63.48, "tokens/trainable": 301004} +{"epoch": 1.1328125, "grad_norm": 1.8370996713638306, "learning_rate": 4.90663667927174e-05, "loss": 0.19569092988967896, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.21615, "step": 290, "tokens/total": 10103776, "tokens/train_per_sec_per_gpu": 71.43, "tokens/trainable": 302031} +{"epoch": 1.13671875, "grad_norm": 0.9921818375587463, "learning_rate": 4.877873600900581e-05, "loss": 0.1778547763824463, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.07, "memory/max_allocated (GiB)": 38.07, "ppl": 1.19465, "step": 291, "tokens/total": 10139888, "tokens/train_per_sec_per_gpu": 54.98, "tokens/trainable": 303058} +{"epoch": 1.140625, "grad_norm": 0.6638201475143433, "learning_rate": 4.849136411748306e-05, "loss": 0.16394028067588806, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.04, "memory/max_allocated (GiB)": 39.04, "ppl": 1.17814, "step": 292, "tokens/total": 10177936, "tokens/train_per_sec_per_gpu": 53.94, "tokens/trainable": 304115} +{"epoch": 1.14453125, "grad_norm": 1.135473608970642, "learning_rate": 4.8204263076866574e-05, "loss": 0.19205540418624878, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.21174, "step": 293, "tokens/total": 10213856, "tokens/train_per_sec_per_gpu": 57.82, "tokens/trainable": 305177} +{"epoch": 1.1484375, "grad_norm": 1.4830479621887207, "learning_rate": 4.791744483460251e-05, "loss": 0.2149713635444641, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.74, "memory/max_allocated (GiB)": 37.74, "ppl": 1.23983, "step": 294, "tokens/total": 10248352, "tokens/train_per_sec_per_gpu": 52.01, "tokens/trainable": 306171} +{"epoch": 1.15234375, "grad_norm": 1.087198257446289, "learning_rate": 4.7630921326368736e-05, "loss": 0.18138282001018524, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.19887, "step": 295, "tokens/total": 10287072, "tokens/train_per_sec_per_gpu": 50.32, "tokens/trainable": 307153} +{"epoch": 1.15625, "grad_norm": 1.2993900775909424, "learning_rate": 4.7344704475577916e-05, "loss": 0.18817263841629028, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 35.7, "memory/max_allocated (GiB)": 35.7, "ppl": 1.20704, "step": 296, "tokens/total": 10316640, "tokens/train_per_sec_per_gpu": 68.59, "tokens/trainable": 308179} +{"epoch": 1.16015625, "grad_norm": 1.751837134361267, "learning_rate": 4.705880619288153e-05, "loss": 0.20655575394630432, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.37, "memory/max_allocated (GiB)": 38.37, "ppl": 1.22944, "step": 297, "tokens/total": 10352864, "tokens/train_per_sec_per_gpu": 53.33, "tokens/trainable": 309164} +{"epoch": 1.1640625, "grad_norm": 0.7991726398468018, "learning_rate": 4.677323837567412e-05, "loss": 0.1825736165046692, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.2003, "step": 298, "tokens/total": 10385696, "tokens/train_per_sec_per_gpu": 64.9, "tokens/trainable": 310220} +{"epoch": 1.16796875, "grad_norm": 2.016195058822632, "learning_rate": 4.6488012907598146e-05, "loss": 0.20117998123168945, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 36.4, "memory/max_allocated (GiB)": 36.4, "ppl": 1.22284, "step": 299, "tokens/total": 10417904, "tokens/train_per_sec_per_gpu": 57.71, "tokens/trainable": 311175} +{"epoch": 1.171875, "grad_norm": 1.337790608406067, "learning_rate": 4.620314165804964e-05, "loss": 0.1820804327726364, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.19971, "step": 300, "tokens/total": 10453728, "tokens/train_per_sec_per_gpu": 58.91, "tokens/trainable": 312271} +{"epoch": 1.17578125, "grad_norm": 1.0523791313171387, "learning_rate": 4.591863648168407e-05, "loss": 0.17463555932044983, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.19081, "step": 301, "tokens/total": 10488288, "tokens/train_per_sec_per_gpu": 59.6, "tokens/trainable": 313368} +{"epoch": 1.1796875, "grad_norm": 0.9753887057304382, "learning_rate": 4.5634509217923135e-05, "loss": 0.18796822428703308, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.96, "memory/max_allocated (GiB)": 37.96, "ppl": 1.2068, "step": 302, "tokens/total": 10521824, "tokens/train_per_sec_per_gpu": 61.45, "tokens/trainable": 314406} +{"epoch": 1.18359375, "grad_norm": 0.8789904117584229, "learning_rate": 4.535077169046201e-05, "loss": 0.19034157693386078, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.03, "memory/max_allocated (GiB)": 38.03, "ppl": 1.20966, "step": 303, "tokens/total": 10558432, "tokens/train_per_sec_per_gpu": 57.41, "tokens/trainable": 315436} +{"epoch": 1.1875, "grad_norm": 1.2371225357055664, "learning_rate": 4.506743570677743e-05, "loss": 0.19796612858772278, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 36.35, "memory/max_allocated (GiB)": 36.35, "ppl": 1.21892, "step": 304, "tokens/total": 10588752, "tokens/train_per_sec_per_gpu": 72.39, "tokens/trainable": 316428} +{"epoch": 1.19140625, "grad_norm": 1.2973341941833496, "learning_rate": 4.478451305763618e-05, "loss": 0.2075677514076233, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.23068, "step": 305, "tokens/total": 10622336, "tokens/train_per_sec_per_gpu": 64.44, "tokens/trainable": 317466} +{"epoch": 1.1953125, "grad_norm": 0.7124327421188354, "learning_rate": 4.450201551660454e-05, "loss": 0.182248055934906, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 36.41, "memory/max_allocated (GiB)": 36.41, "ppl": 1.19991, "step": 306, "tokens/total": 10653520, "tokens/train_per_sec_per_gpu": 63.73, "tokens/trainable": 318471} +{"epoch": 1.19921875, "grad_norm": 1.1369377374649048, "learning_rate": 4.4219954839558276e-05, "loss": 0.19738832116127014, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.21822, "step": 307, "tokens/total": 10688144, "tokens/train_per_sec_per_gpu": 54.11, "tokens/trainable": 319483} +{"epoch": 1.203125, "grad_norm": 2.0645813941955566, "learning_rate": 4.393834276419352e-05, "loss": 0.1944822371006012, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 35.7, "memory/max_allocated (GiB)": 35.7, "ppl": 1.21468, "step": 308, "tokens/total": 10718032, "tokens/train_per_sec_per_gpu": 71.01, "tokens/trainable": 320509} +{"epoch": 1.20703125, "grad_norm": 1.0547071695327759, "learning_rate": 4.36571910095382e-05, "loss": 0.18958787620067596, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.20875, "step": 309, "tokens/total": 10756464, "tokens/train_per_sec_per_gpu": 55.48, "tokens/trainable": 321516} +{"epoch": 1.2109375, "grad_norm": 1.0846188068389893, "learning_rate": 4.337651127546448e-05, "loss": 0.19401118159294128, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.21411, "step": 310, "tokens/total": 10792064, "tokens/train_per_sec_per_gpu": 62.59, "tokens/trainable": 322540} +{"epoch": 1.21484375, "grad_norm": 1.4885276556015015, "learning_rate": 4.3096315242201736e-05, "loss": 0.20999255776405334, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 36.51, "memory/max_allocated (GiB)": 36.51, "ppl": 1.23367, "step": 311, "tokens/total": 10824800, "tokens/train_per_sec_per_gpu": 59.27, "tokens/trainable": 323543} +{"epoch": 1.21875, "grad_norm": 1.4617489576339722, "learning_rate": 4.2816614569850635e-05, "loss": 0.1889691948890686, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 38.66, "memory/max_allocated (GiB)": 38.66, "ppl": 1.208, "step": 312, "tokens/total": 10860576, "tokens/train_per_sec_per_gpu": 58.47, "tokens/trainable": 324543} +{"epoch": 1.22265625, "grad_norm": 0.8250847458839417, "learning_rate": 4.2537420897897864e-05, "loss": 0.18910670280456543, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 38.72, "memory/max_allocated (GiB)": 38.72, "ppl": 1.20817, "step": 313, "tokens/total": 10896224, "tokens/train_per_sec_per_gpu": 54.07, "tokens/trainable": 325554} +{"epoch": 1.2265625, "grad_norm": 1.0990484952926636, "learning_rate": 4.225874584473174e-05, "loss": 0.19755887985229492, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.21842, "step": 314, "tokens/total": 10928768, "tokens/train_per_sec_per_gpu": 62.69, "tokens/trainable": 326556} +{"epoch": 1.23046875, "grad_norm": 0.8721315264701843, "learning_rate": 4.19806010071587e-05, "loss": 0.16654255986213684, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.18121, "step": 315, "tokens/total": 10966848, "tokens/train_per_sec_per_gpu": 53.82, "tokens/trainable": 327620} +{"epoch": 1.234375, "grad_norm": 1.8711777925491333, "learning_rate": 4.170299795992081e-05, "loss": 0.20194309949874878, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 36.88, "memory/max_allocated (GiB)": 36.88, "ppl": 1.22378, "step": 316, "tokens/total": 11000416, "tokens/train_per_sec_per_gpu": 60.64, "tokens/trainable": 328662} +{"epoch": 1.23828125, "grad_norm": 1.1276538372039795, "learning_rate": 4.142594825521398e-05, "loss": 0.16677722334861755, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.18149, "step": 317, "tokens/total": 11039520, "tokens/train_per_sec_per_gpu": 51.61, "tokens/trainable": 329719} +{"epoch": 1.2421875, "grad_norm": 0.8619747757911682, "learning_rate": 4.114946342220728e-05, "loss": 0.17968766391277313, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.19684, "step": 318, "tokens/total": 11076096, "tokens/train_per_sec_per_gpu": 56.67, "tokens/trainable": 330802} +{"epoch": 1.24609375, "grad_norm": 1.5074982643127441, "learning_rate": 4.087355496656321e-05, "loss": 0.17742231488227844, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.19414, "step": 319, "tokens/total": 11109728, "tokens/train_per_sec_per_gpu": 68.22, "tokens/trainable": 331918} +{"epoch": 1.25, "grad_norm": 1.3475732803344727, "learning_rate": 4.05982343699588e-05, "loss": 0.1900925636291504, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.20936, "step": 320, "tokens/total": 11148976, "tokens/train_per_sec_per_gpu": 58.67, "tokens/trainable": 333051} +{"epoch": 1.25390625, "grad_norm": 0.7068098187446594, "learning_rate": 4.0323513089607876e-05, "loss": 0.1634504199028015, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.17757, "step": 321, "tokens/total": 11183888, "tokens/train_per_sec_per_gpu": 58.0, "tokens/trainable": 334129} +{"epoch": 1.2578125, "grad_norm": 1.6155544519424438, "learning_rate": 4.004940255778431e-05, "loss": 0.18422731757164001, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 39.63, "memory/max_allocated (GiB)": 39.63, "ppl": 1.20229, "step": 322, "tokens/total": 11222608, "tokens/train_per_sec_per_gpu": 56.98, "tokens/trainable": 335186} +{"epoch": 1.26171875, "grad_norm": 1.9097329378128052, "learning_rate": 3.977591418134619e-05, "loss": 0.18405833840370178, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.20209, "step": 323, "tokens/total": 11257568, "tokens/train_per_sec_per_gpu": 61.46, "tokens/trainable": 336236} +{"epoch": 1.265625, "grad_norm": 1.4309443235397339, "learning_rate": 3.95030593412612e-05, "loss": 0.2032088339328766, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.22533, "step": 324, "tokens/total": 11293472, "tokens/train_per_sec_per_gpu": 57.65, "tokens/trainable": 337236} +{"epoch": 1.26953125, "grad_norm": 0.8287313580513, "learning_rate": 3.923084939213296e-05, "loss": 0.18890048563480377, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 35.75, "memory/max_allocated (GiB)": 35.75, "ppl": 1.20792, "step": 325, "tokens/total": 11324048, "tokens/train_per_sec_per_gpu": 64.79, "tokens/trainable": 338251} +{"epoch": 1.2734375, "grad_norm": 2.241215944290161, "learning_rate": 3.895929566172861e-05, "loss": 0.1546318233013153, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.16723, "step": 326, "tokens/total": 11360064, "tokens/train_per_sec_per_gpu": 60.56, "tokens/trainable": 339355} +{"epoch": 1.27734375, "grad_norm": 0.6356357336044312, "learning_rate": 3.868840945050728e-05, "loss": 0.16003942489624023, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 39.19, "memory/max_allocated (GiB)": 39.19, "ppl": 1.17356, "step": 327, "tokens/total": 11399008, "tokens/train_per_sec_per_gpu": 56.4, "tokens/trainable": 340444} +{"epoch": 1.28125, "grad_norm": 0.7518062591552734, "learning_rate": 3.841820203114995e-05, "loss": 0.16589106619358063, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 36.78, "memory/max_allocated (GiB)": 36.78, "ppl": 1.18044, "step": 328, "tokens/total": 11430752, "tokens/train_per_sec_per_gpu": 67.7, "tokens/trainable": 341529} +{"epoch": 1.28515625, "grad_norm": 0.8319792151451111, "learning_rate": 3.814868464809027e-05, "loss": 0.1793747842311859, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 36.77, "memory/max_allocated (GiB)": 36.77, "ppl": 1.19647, "step": 329, "tokens/total": 11462688, "tokens/train_per_sec_per_gpu": 61.55, "tokens/trainable": 342525} +{"epoch": 1.2890625, "grad_norm": 15.053144454956055, "learning_rate": 3.787986851704667e-05, "loss": 0.1985969990491867, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.21969, "step": 330, "tokens/total": 11496240, "tokens/train_per_sec_per_gpu": 64.11, "tokens/trainable": 343562} +{"epoch": 1.29296875, "grad_norm": 1.131005048751831, "learning_rate": 3.7611764824555654e-05, "loss": 0.16883310675621033, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 37.84, "memory/max_allocated (GiB)": 37.84, "ppl": 1.18392, "step": 331, "tokens/total": 11527968, "tokens/train_per_sec_per_gpu": 76.47, "tokens/trainable": 344715} +{"epoch": 1.296875, "grad_norm": 1.5510119199752808, "learning_rate": 3.734438472750619e-05, "loss": 0.17921791970729828, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.19628, "step": 332, "tokens/total": 11567936, "tokens/train_per_sec_per_gpu": 51.93, "tokens/trainable": 345805} +{"epoch": 1.30078125, "grad_norm": 1.5350629091262817, "learning_rate": 3.707773935267552e-05, "loss": 0.1923224925994873, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.21206, "step": 333, "tokens/total": 11604992, "tokens/train_per_sec_per_gpu": 56.54, "tokens/trainable": 346917} +{"epoch": 1.3046875, "grad_norm": 1.3143458366394043, "learning_rate": 3.68118397962661e-05, "loss": 0.18988244235515594, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 38.49, "memory/max_allocated (GiB)": 38.49, "ppl": 1.20911, "step": 334, "tokens/total": 11639744, "tokens/train_per_sec_per_gpu": 56.05, "tokens/trainable": 347879} +{"epoch": 1.30859375, "grad_norm": 1.4386987686157227, "learning_rate": 3.654669712344384e-05, "loss": 0.16640353202819824, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 36.72, "memory/max_allocated (GiB)": 36.72, "ppl": 1.18105, "step": 335, "tokens/total": 11670944, "tokens/train_per_sec_per_gpu": 75.1, "tokens/trainable": 348935} +{"epoch": 1.3125, "grad_norm": 1.5980287790298462, "learning_rate": 3.628232236787763e-05, "loss": 0.17587323486804962, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 37.91, "memory/max_allocated (GiB)": 37.91, "ppl": 1.19229, "step": 336, "tokens/total": 11706544, "tokens/train_per_sec_per_gpu": 56.38, "tokens/trainable": 350048} +{"epoch": 1.31640625, "grad_norm": 5.325817108154297, "learning_rate": 3.6018726531280144e-05, "loss": 0.16189080476760864, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.17573, "step": 337, "tokens/total": 11743952, "tokens/train_per_sec_per_gpu": 60.72, "tokens/trainable": 351143} +{"epoch": 1.3203125, "grad_norm": 0.9773403406143188, "learning_rate": 3.575592058295017e-05, "loss": 0.18861381709575653, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.20757, "step": 338, "tokens/total": 11777936, "tokens/train_per_sec_per_gpu": 61.79, "tokens/trainable": 352197} +{"epoch": 1.32421875, "grad_norm": 0.6585482358932495, "learning_rate": 3.549391545931585e-05, "loss": 0.17136387526988983, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 38.93, "memory/max_allocated (GiB)": 38.93, "ppl": 1.18692, "step": 339, "tokens/total": 11812416, "tokens/train_per_sec_per_gpu": 62.73, "tokens/trainable": 353247} +{"epoch": 1.328125, "grad_norm": 1.0079004764556885, "learning_rate": 3.5232722063479914e-05, "loss": 0.17793995141983032, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.19475, "step": 340, "tokens/total": 11846704, "tokens/train_per_sec_per_gpu": 65.64, "tokens/trainable": 354311} +{"epoch": 1.33203125, "grad_norm": 0.599157989025116, "learning_rate": 3.49723512647657e-05, "loss": 0.1638096272945404, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.17799, "step": 341, "tokens/total": 11885968, "tokens/train_per_sec_per_gpu": 54.47, "tokens/trainable": 355427} +{"epoch": 1.3359375, "grad_norm": 1.0937374830245972, "learning_rate": 3.471281389826491e-05, "loss": 0.19452863931655884, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.21474, "step": 342, "tokens/total": 11917024, "tokens/train_per_sec_per_gpu": 63.69, "tokens/trainable": 356380} +{"epoch": 1.33984375, "grad_norm": 1.0339221954345703, "learning_rate": 3.4454120764386764e-05, "loss": 0.17912665009498596, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.19617, "step": 343, "tokens/total": 11955648, "tokens/train_per_sec_per_gpu": 57.01, "tokens/trainable": 357460} +{"epoch": 1.34375, "grad_norm": 1.9442112445831299, "learning_rate": 3.4196282628408526e-05, "loss": 0.1997814178466797, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 37.58, "memory/max_allocated (GiB)": 37.58, "ppl": 1.22114, "step": 344, "tokens/total": 11989936, "tokens/train_per_sec_per_gpu": 61.04, "tokens/trainable": 358480} +{"epoch": 1.34765625, "grad_norm": 1.316119909286499, "learning_rate": 3.3939310220027456e-05, "loss": 0.18177035450935364, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.19934, "step": 345, "tokens/total": 12023968, "tokens/train_per_sec_per_gpu": 60.37, "tokens/trainable": 359568} +{"epoch": 1.3515625, "grad_norm": 0.7830258011817932, "learning_rate": 3.3683214232914404e-05, "loss": 0.16035127639770508, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.17392, "step": 346, "tokens/total": 12059632, "tokens/train_per_sec_per_gpu": 60.12, "tokens/trainable": 360648} +{"epoch": 1.35546875, "grad_norm": 1.4455333948135376, "learning_rate": 3.342800532426873e-05, "loss": 0.1890421211719513, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.20809, "step": 347, "tokens/total": 12095616, "tokens/train_per_sec_per_gpu": 60.11, "tokens/trainable": 361666} +{"epoch": 1.359375, "grad_norm": 1.0220082998275757, "learning_rate": 3.317369411437484e-05, "loss": 0.19236478209495544, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.21211, "step": 348, "tokens/total": 12131488, "tokens/train_per_sec_per_gpu": 56.06, "tokens/trainable": 362689} +{"epoch": 1.36328125, "grad_norm": 1.0870697498321533, "learning_rate": 3.292029118616024e-05, "loss": 0.18121060729026794, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.19867, "step": 349, "tokens/total": 12164208, "tokens/train_per_sec_per_gpu": 57.26, "tokens/trainable": 363671} +{"epoch": 1.3671875, "grad_norm": 1.2786781787872314, "learning_rate": 3.266780708475511e-05, "loss": 0.1647973358631134, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 39.12, "memory/max_allocated (GiB)": 39.12, "ppl": 1.17915, "step": 350, "tokens/total": 12199616, "tokens/train_per_sec_per_gpu": 58.76, "tokens/trainable": 364746} +{"epoch": 1.37109375, "grad_norm": 1.6856462955474854, "learning_rate": 3.241625231705354e-05, "loss": 0.16411373019218445, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.17835, "step": 351, "tokens/total": 12235664, "tokens/train_per_sec_per_gpu": 59.81, "tokens/trainable": 365811} +{"epoch": 1.375, "grad_norm": 0.6954187154769897, "learning_rate": 3.216563735127618e-05, "loss": 0.15924251079559326, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 39.12, "memory/max_allocated (GiB)": 39.12, "ppl": 1.17262, "step": 352, "tokens/total": 12270800, "tokens/train_per_sec_per_gpu": 61.47, "tokens/trainable": 366944} +{"epoch": 1.37890625, "grad_norm": 0.8107733726501465, "learning_rate": 3.191597261653475e-05, "loss": 0.19018901884555817, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.20948, "step": 353, "tokens/total": 12307568, "tokens/train_per_sec_per_gpu": 58.04, "tokens/trainable": 368012} +{"epoch": 1.3828125, "grad_norm": 1.2725777626037598, "learning_rate": 3.166726850239794e-05, "loss": 0.1752881407737732, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 37.18, "memory/max_allocated (GiB)": 37.18, "ppl": 1.19159, "step": 354, "tokens/total": 12341200, "tokens/train_per_sec_per_gpu": 62.87, "tokens/trainable": 369090} +{"epoch": 1.38671875, "grad_norm": 0.9384642243385315, "learning_rate": 3.141953535845912e-05, "loss": 0.19952556490898132, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 36.16, "memory/max_allocated (GiB)": 36.16, "ppl": 1.22082, "step": 355, "tokens/total": 12372016, "tokens/train_per_sec_per_gpu": 63.36, "tokens/trainable": 370028} +{"epoch": 1.390625, "grad_norm": 0.8138394951820374, "learning_rate": 3.11727834939056e-05, "loss": 0.1788119375705719, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 36.28, "memory/max_allocated (GiB)": 36.28, "ppl": 1.1958, "step": 356, "tokens/total": 12403856, "tokens/train_per_sec_per_gpu": 58.88, "tokens/trainable": 371051} +{"epoch": 1.39453125, "grad_norm": 1.0322842597961426, "learning_rate": 3.092702317708967e-05, "loss": 0.2008545845746994, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.22245, "step": 357, "tokens/total": 12437584, "tokens/train_per_sec_per_gpu": 62.86, "tokens/trainable": 372063} +{"epoch": 1.3984375, "grad_norm": 1.374879002571106, "learning_rate": 3.0682264635101276e-05, "loss": 0.18424434959888458, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 37.44, "memory/max_allocated (GiB)": 37.44, "ppl": 1.20231, "step": 358, "tokens/total": 12470096, "tokens/train_per_sec_per_gpu": 62.59, "tokens/trainable": 373064} +{"epoch": 1.40234375, "grad_norm": 0.652252197265625, "learning_rate": 3.0438518053342407e-05, "loss": 0.16083896160125732, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.1745, "step": 359, "tokens/total": 12509168, "tokens/train_per_sec_per_gpu": 54.48, "tokens/trainable": 374175} +{"epoch": 1.40625, "grad_norm": 2.1104793548583984, "learning_rate": 3.0195793575103266e-05, "loss": 0.16652710735797882, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.1812, "step": 360, "tokens/total": 12548208, "tokens/train_per_sec_per_gpu": 55.26, "tokens/trainable": 375251} +{"epoch": 1.41015625, "grad_norm": 1.2314302921295166, "learning_rate": 2.9954101301140146e-05, "loss": 0.19373942911624908, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.21378, "step": 361, "tokens/total": 12583552, "tokens/train_per_sec_per_gpu": 59.22, "tokens/trainable": 376287} +{"epoch": 1.4140625, "grad_norm": 1.6933321952819824, "learning_rate": 2.9713451289255123e-05, "loss": 0.17995160818099976, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 36.9, "memory/max_allocated (GiB)": 36.9, "ppl": 1.19716, "step": 362, "tokens/total": 12612944, "tokens/train_per_sec_per_gpu": 61.46, "tokens/trainable": 377310} +{"epoch": 1.41796875, "grad_norm": 0.7550560832023621, "learning_rate": 2.9473853553877484e-05, "loss": 0.1573551595211029, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.17041, "step": 363, "tokens/total": 12646880, "tokens/train_per_sec_per_gpu": 60.68, "tokens/trainable": 378349} +{"epoch": 1.421875, "grad_norm": 0.7150856256484985, "learning_rate": 2.9235318065647e-05, "loss": 0.17487509548664093, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 39.61, "memory/max_allocated (GiB)": 39.61, "ppl": 1.1911, "step": 364, "tokens/total": 12686896, "tokens/train_per_sec_per_gpu": 53.75, "tokens/trainable": 379402} +{"epoch": 1.42578125, "grad_norm": 0.8061363697052002, "learning_rate": 2.8997854750998964e-05, "loss": 0.17118540406227112, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.18671, "step": 365, "tokens/total": 12726416, "tokens/train_per_sec_per_gpu": 57.53, "tokens/trainable": 380505} +{"epoch": 1.4296875, "grad_norm": 0.6374967694282532, "learning_rate": 2.8761473491751258e-05, "loss": 0.18117153644561768, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.63, "memory/max_allocated (GiB)": 36.63, "ppl": 1.19862, "step": 366, "tokens/total": 12759248, "tokens/train_per_sec_per_gpu": 60.28, "tokens/trainable": 381503} +{"epoch": 1.43359375, "grad_norm": 2.776920795440674, "learning_rate": 2.8526184124692883e-05, "loss": 0.1488078236579895, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.16045, "step": 367, "tokens/total": 12795904, "tokens/train_per_sec_per_gpu": 52.94, "tokens/trainable": 382606} +{"epoch": 1.4375, "grad_norm": 0.6221509575843811, "learning_rate": 2.829199644117484e-05, "loss": 0.17324143648147583, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.18915, "step": 368, "tokens/total": 12829616, "tokens/train_per_sec_per_gpu": 63.26, "tokens/trainable": 383569} +{"epoch": 1.44140625, "grad_norm": 1.0819071531295776, "learning_rate": 2.8058920186702553e-05, "loss": 0.16339311003684998, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.91, "memory/max_allocated (GiB)": 38.91, "ppl": 1.1775, "step": 369, "tokens/total": 12867744, "tokens/train_per_sec_per_gpu": 59.23, "tokens/trainable": 384668} +{"epoch": 1.4453125, "grad_norm": 0.6733879446983337, "learning_rate": 2.782696506053033e-05, "loss": 0.1805652379989624, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.19789, "step": 370, "tokens/total": 12905936, "tokens/train_per_sec_per_gpu": 52.95, "tokens/trainable": 385663} +{"epoch": 1.44921875, "grad_norm": 1.2482788562774658, "learning_rate": 2.7596140715257824e-05, "loss": 0.16367454826831818, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.17783, "step": 371, "tokens/total": 12941216, "tokens/train_per_sec_per_gpu": 61.86, "tokens/trainable": 386746} +{"epoch": 1.453125, "grad_norm": 1.0232666730880737, "learning_rate": 2.7366456756428184e-05, "loss": 0.16665202379226685, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.18134, "step": 372, "tokens/total": 12975584, "tokens/train_per_sec_per_gpu": 60.44, "tokens/trainable": 387782} +{"epoch": 1.45703125, "grad_norm": 1.3608694076538086, "learning_rate": 2.7137922742128486e-05, "loss": 0.17009463906288147, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.21, "memory/max_allocated (GiB)": 38.21, "ppl": 1.18542, "step": 373, "tokens/total": 13010464, "tokens/train_per_sec_per_gpu": 61.84, "tokens/trainable": 388849} +{"epoch": 1.4609375, "grad_norm": 0.6893065571784973, "learning_rate": 2.691054818259188e-05, "loss": 0.15978367626667023, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.4, "memory/max_allocated (GiB)": 37.4, "ppl": 1.17326, "step": 374, "tokens/total": 13045136, "tokens/train_per_sec_per_gpu": 61.14, "tokens/trainable": 389955} +{"epoch": 1.46484375, "grad_norm": 0.7792233228683472, "learning_rate": 2.6684342539801933e-05, "loss": 0.20435959100723267, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.49, "memory/max_allocated (GiB)": 36.49, "ppl": 1.22674, "step": 375, "tokens/total": 13076400, "tokens/train_per_sec_per_gpu": 61.66, "tokens/trainable": 390933} +{"epoch": 1.46875, "grad_norm": 0.9602343440055847, "learning_rate": 2.645931522709877e-05, "loss": 0.16115817427635193, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.54, "memory/max_allocated (GiB)": 39.54, "ppl": 1.17487, "step": 376, "tokens/total": 13112144, "tokens/train_per_sec_per_gpu": 72.3, "tokens/trainable": 392094} +{"epoch": 1.47265625, "grad_norm": 0.8952240943908691, "learning_rate": 2.6235475608787365e-05, "loss": 0.17457237839698792, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.6, "memory/max_allocated (GiB)": 38.6, "ppl": 1.19074, "step": 377, "tokens/total": 13147872, "tokens/train_per_sec_per_gpu": 59.25, "tokens/trainable": 393123} +{"epoch": 1.4765625, "grad_norm": 0.7354146838188171, "learning_rate": 2.6012832999747916e-05, "loss": 0.186595618724823, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.20514, "step": 378, "tokens/total": 13178656, "tokens/train_per_sec_per_gpu": 64.23, "tokens/trainable": 394090} +{"epoch": 1.48046875, "grad_norm": 1.2128528356552124, "learning_rate": 2.579139666504821e-05, "loss": 0.18570807576179504, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.20407, "step": 379, "tokens/total": 13218720, "tokens/train_per_sec_per_gpu": 51.5, "tokens/trainable": 395092} +{"epoch": 1.484375, "grad_norm": 1.4377597570419312, "learning_rate": 2.557117581955798e-05, "loss": 0.17902085185050964, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 1.19605, "step": 380, "tokens/total": 13252544, "tokens/train_per_sec_per_gpu": 59.63, "tokens/trainable": 396143} +{"epoch": 1.48828125, "grad_norm": 1.1802414655685425, "learning_rate": 2.5352179627565532e-05, "loss": 0.18346185982227325, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.06, "memory/max_allocated (GiB)": 38.06, "ppl": 1.20137, "step": 381, "tokens/total": 13286832, "tokens/train_per_sec_per_gpu": 50.27, "tokens/trainable": 397120} +{"epoch": 1.4921875, "grad_norm": 0.7962527275085449, "learning_rate": 2.5134417202396277e-05, "loss": 0.19509616494178772, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.21543, "step": 382, "tokens/total": 13320432, "tokens/train_per_sec_per_gpu": 57.69, "tokens/trainable": 398089} +{"epoch": 1.49609375, "grad_norm": 1.3502614498138428, "learning_rate": 2.491789760603361e-05, "loss": 0.18651533126831055, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.20504, "step": 383, "tokens/total": 13356928, "tokens/train_per_sec_per_gpu": 53.59, "tokens/trainable": 399105} +{"epoch": 1.5, "grad_norm": 2.6330978870391846, "learning_rate": 2.4702629848741764e-05, "loss": 0.1733933687210083, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.18933, "step": 384, "tokens/total": 13391568, "tokens/train_per_sec_per_gpu": 62.38, "tokens/trainable": 400109} +{"epoch": 1.50390625, "grad_norm": 10.400419235229492, "learning_rate": 2.4488622888690785e-05, "loss": 0.17994076013565063, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.19715, "step": 385, "tokens/total": 13425152, "tokens/train_per_sec_per_gpu": 55.22, "tokens/trainable": 401085} +{"epoch": 1.5078125, "grad_norm": 1.1532379388809204, "learning_rate": 2.427588563158384e-05, "loss": 0.1595897376537323, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.11, "memory/max_allocated (GiB)": 39.11, "ppl": 1.17303, "step": 386, "tokens/total": 13459504, "tokens/train_per_sec_per_gpu": 62.5, "tokens/trainable": 402159} +{"epoch": 1.51171875, "grad_norm": 0.7690994143486023, "learning_rate": 2.406442693028651e-05, "loss": 0.19112162292003632, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.21061, "step": 387, "tokens/total": 13491136, "tokens/train_per_sec_per_gpu": 61.36, "tokens/trainable": 403100} +{"epoch": 1.515625, "grad_norm": 0.5912942290306091, "learning_rate": 2.3854255584458547e-05, "loss": 0.1735694259405136, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.18954, "step": 388, "tokens/total": 13530112, "tokens/train_per_sec_per_gpu": 51.45, "tokens/trainable": 404117} +{"epoch": 1.51953125, "grad_norm": 0.9809858798980713, "learning_rate": 2.3645380340187508e-05, "loss": 0.1749788224697113, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.14, "memory/max_allocated (GiB)": 39.14, "ppl": 1.19122, "step": 389, "tokens/total": 13565680, "tokens/train_per_sec_per_gpu": 62.73, "tokens/trainable": 405191} +{"epoch": 1.5234375, "grad_norm": 1.2459602355957031, "learning_rate": 2.3437809889624914e-05, "loss": 0.20151811838150024, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.52, "memory/max_allocated (GiB)": 36.52, "ppl": 1.22326, "step": 390, "tokens/total": 13596960, "tokens/train_per_sec_per_gpu": 72.13, "tokens/trainable": 406242} +{"epoch": 1.52734375, "grad_norm": 2.182750940322876, "learning_rate": 2.3231552870624487e-05, "loss": 0.16651716828346252, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.18118, "step": 391, "tokens/total": 13627840, "tokens/train_per_sec_per_gpu": 70.79, "tokens/trainable": 407315} +{"epoch": 1.53125, "grad_norm": 0.8178974986076355, "learning_rate": 2.3026617866382657e-05, "loss": 0.1549813598394394, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.7, "memory/max_allocated (GiB)": 36.7, "ppl": 1.16764, "step": 392, "tokens/total": 13660848, "tokens/train_per_sec_per_gpu": 62.91, "tokens/trainable": 408401} +{"epoch": 1.53515625, "grad_norm": 0.5484746098518372, "learning_rate": 2.2823013405081507e-05, "loss": 0.15658438205718994, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.16951, "step": 393, "tokens/total": 13699024, "tokens/train_per_sec_per_gpu": 53.64, "tokens/trainable": 409473} +{"epoch": 1.5390625, "grad_norm": 0.6838989853858948, "learning_rate": 2.2620747959533722e-05, "loss": 0.15924493968486786, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.17263, "step": 394, "tokens/total": 13732336, "tokens/train_per_sec_per_gpu": 65.95, "tokens/trainable": 410584} +{"epoch": 1.54296875, "grad_norm": 0.6927838325500488, "learning_rate": 2.2419829946830123e-05, "loss": 0.17495529353618622, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.19119, "step": 395, "tokens/total": 13768112, "tokens/train_per_sec_per_gpu": 60.22, "tokens/trainable": 411602} +{"epoch": 1.546875, "grad_norm": 0.5986804366111755, "learning_rate": 2.2220267727989325e-05, "loss": 0.15315836668014526, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.74, "memory/max_allocated (GiB)": 37.74, "ppl": 1.16551, "step": 396, "tokens/total": 13802480, "tokens/train_per_sec_per_gpu": 58.94, "tokens/trainable": 412649} +{"epoch": 1.55078125, "grad_norm": 0.8826265335083008, "learning_rate": 2.202206960760984e-05, "loss": 0.15746644139289856, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.17054, "step": 397, "tokens/total": 13840736, "tokens/train_per_sec_per_gpu": 56.27, "tokens/trainable": 413796} +{"epoch": 1.5546875, "grad_norm": 0.9733564257621765, "learning_rate": 2.182524383352446e-05, "loss": 0.19261163473129272, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.21241, "step": 398, "tokens/total": 13876896, "tokens/train_per_sec_per_gpu": 55.78, "tokens/trainable": 414806} +{"epoch": 1.55859375, "grad_norm": 0.7728884816169739, "learning_rate": 2.1629798596457056e-05, "loss": 0.17488422989845276, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.34, "memory/max_allocated (GiB)": 36.34, "ppl": 1.19111, "step": 399, "tokens/total": 13908992, "tokens/train_per_sec_per_gpu": 63.18, "tokens/trainable": 415811} +{"epoch": 1.5625, "grad_norm": 2.0373728275299072, "learning_rate": 2.1435742029681725e-05, "loss": 0.18392586708068848, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.29, "memory/max_allocated (GiB)": 36.29, "ppl": 1.20193, "step": 400, "tokens/total": 13939456, "tokens/train_per_sec_per_gpu": 68.33, "tokens/trainable": 416768} +{"epoch": 1.56640625, "grad_norm": 0.837566077709198, "learning_rate": 2.124308220868431e-05, "loss": 0.15641003847122192, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.16931, "step": 401, "tokens/total": 13975312, "tokens/train_per_sec_per_gpu": 57.02, "tokens/trainable": 417829} +{"epoch": 1.5703125, "grad_norm": 0.7144855260848999, "learning_rate": 2.105182715082638e-05, "loss": 0.1576695442199707, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.92, "memory/max_allocated (GiB)": 36.92, "ppl": 1.17078, "step": 402, "tokens/total": 14009056, "tokens/train_per_sec_per_gpu": 63.46, "tokens/trainable": 418922} +{"epoch": 1.57421875, "grad_norm": 0.8245531916618347, "learning_rate": 2.0861984815011552e-05, "loss": 0.1796596497297287, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 1.19681, "step": 403, "tokens/total": 14044320, "tokens/train_per_sec_per_gpu": 55.61, "tokens/trainable": 419893} +{"epoch": 1.578125, "grad_norm": 0.9188951253890991, "learning_rate": 2.0673563101354323e-05, "loss": 0.17780721187591553, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.19459, "step": 404, "tokens/total": 14079600, "tokens/train_per_sec_per_gpu": 52.44, "tokens/trainable": 420906} +{"epoch": 1.58203125, "grad_norm": 1.0568886995315552, "learning_rate": 2.0486569850851317e-05, "loss": 0.1791299432516098, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.19618, "step": 405, "tokens/total": 14114368, "tokens/train_per_sec_per_gpu": 61.73, "tokens/trainable": 421922} +{"epoch": 1.5859375, "grad_norm": 0.6760419607162476, "learning_rate": 2.0301012845054956e-05, "loss": 0.16697371006011963, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.18172, "step": 406, "tokens/total": 14153248, "tokens/train_per_sec_per_gpu": 51.34, "tokens/trainable": 422950} +{"epoch": 1.58984375, "grad_norm": 0.6577782034873962, "learning_rate": 2.011689980574966e-05, "loss": 0.16909413039684296, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.18, "memory/max_allocated (GiB)": 38.18, "ppl": 1.18423, "step": 407, "tokens/total": 14189632, "tokens/train_per_sec_per_gpu": 55.06, "tokens/trainable": 423951} +{"epoch": 1.59375, "grad_norm": 0.6446444392204285, "learning_rate": 1.993423839463052e-05, "loss": 0.16046294569969177, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.17405, "step": 408, "tokens/total": 14223072, "tokens/train_per_sec_per_gpu": 71.77, "tokens/trainable": 425023} +{"epoch": 1.59765625, "grad_norm": 1.4096204042434692, "learning_rate": 1.975303621298445e-05, "loss": 0.16902634501457214, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.18415, "step": 409, "tokens/total": 14259824, "tokens/train_per_sec_per_gpu": 57.52, "tokens/trainable": 426040} +{"epoch": 1.6015625, "grad_norm": 1.5266125202178955, "learning_rate": 1.957330080137385e-05, "loss": 0.18369664251804352, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.22, "memory/max_allocated (GiB)": 36.22, "ppl": 1.20165, "step": 410, "tokens/total": 14290400, "tokens/train_per_sec_per_gpu": 65.22, "tokens/trainable": 427049} +{"epoch": 1.60546875, "grad_norm": 1.3200658559799194, "learning_rate": 1.9395039639322864e-05, "loss": 0.1730060875415802, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.18887, "step": 411, "tokens/total": 14326528, "tokens/train_per_sec_per_gpu": 56.65, "tokens/trainable": 428058} +{"epoch": 1.609375, "grad_norm": 0.6688273549079895, "learning_rate": 1.9218260145006073e-05, "loss": 0.1590842306613922, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.17244, "step": 412, "tokens/total": 14362400, "tokens/train_per_sec_per_gpu": 64.91, "tokens/trainable": 429195} +{"epoch": 1.61328125, "grad_norm": 1.1098120212554932, "learning_rate": 1.904296967493982e-05, "loss": 0.16844123601913452, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.18346, "step": 413, "tokens/total": 14394864, "tokens/train_per_sec_per_gpu": 69.57, "tokens/trainable": 430240} +{"epoch": 1.6171875, "grad_norm": 1.5067198276519775, "learning_rate": 1.8869175523676064e-05, "loss": 0.14871665835380554, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.36, "memory/max_allocated (GiB)": 38.36, "ppl": 1.16034, "step": 414, "tokens/total": 14431648, "tokens/train_per_sec_per_gpu": 61.4, "tokens/trainable": 431375} +{"epoch": 1.62109375, "grad_norm": 1.3714138269424438, "learning_rate": 1.869688492349885e-05, "loss": 0.1689542531967163, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.45, "memory/max_allocated (GiB)": 37.45, "ppl": 1.18407, "step": 415, "tokens/total": 14464912, "tokens/train_per_sec_per_gpu": 64.0, "tokens/trainable": 432431} +{"epoch": 1.625, "grad_norm": 1.2689541578292847, "learning_rate": 1.85261050441233e-05, "loss": 0.16500550508499146, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.1794, "step": 416, "tokens/total": 14501056, "tokens/train_per_sec_per_gpu": 58.68, "tokens/trainable": 433502} +{"epoch": 1.62890625, "grad_norm": 2.1890151500701904, "learning_rate": 1.8356842992397304e-05, "loss": 0.1855984926223755, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.23, "memory/max_allocated (GiB)": 37.23, "ppl": 1.20394, "step": 417, "tokens/total": 14534928, "tokens/train_per_sec_per_gpu": 59.16, "tokens/trainable": 434499} +{"epoch": 1.6328125, "grad_norm": 0.6723273396492004, "learning_rate": 1.8189105812005714e-05, "loss": 0.1752863973379135, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.19159, "step": 418, "tokens/total": 14571680, "tokens/train_per_sec_per_gpu": 54.35, "tokens/trainable": 435451} +{"epoch": 1.63671875, "grad_norm": 0.9078924655914307, "learning_rate": 1.802290048317732e-05, "loss": 0.17383044958114624, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.18985, "step": 419, "tokens/total": 14605280, "tokens/train_per_sec_per_gpu": 59.49, "tokens/trainable": 436441} +{"epoch": 1.640625, "grad_norm": 0.7987353205680847, "learning_rate": 1.785823392239424e-05, "loss": 0.15961840748786926, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.82, "memory/max_allocated (GiB)": 37.82, "ppl": 1.17306, "step": 420, "tokens/total": 14638576, "tokens/train_per_sec_per_gpu": 58.82, "tokens/trainable": 437409} +{"epoch": 1.64453125, "grad_norm": 0.5381324887275696, "learning_rate": 1.7695112982104225e-05, "loss": 0.16152039170265198, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.78, "memory/max_allocated (GiB)": 39.78, "ppl": 1.1753, "step": 421, "tokens/total": 14673824, "tokens/train_per_sec_per_gpu": 62.29, "tokens/trainable": 438462} +{"epoch": 1.6484375, "grad_norm": 0.8632115721702576, "learning_rate": 1.7533544450435433e-05, "loss": 0.17442697286605835, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 35.69, "memory/max_allocated (GiB)": 35.69, "ppl": 1.19056, "step": 422, "tokens/total": 14703248, "tokens/train_per_sec_per_gpu": 75.96, "tokens/trainable": 439500} +{"epoch": 1.65234375, "grad_norm": 0.9402514100074768, "learning_rate": 1.7373535050913946e-05, "loss": 0.17254699766635895, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.18833, "step": 423, "tokens/total": 14739712, "tokens/train_per_sec_per_gpu": 52.37, "tokens/trainable": 440524} +{"epoch": 1.65625, "grad_norm": 0.5145081877708435, "learning_rate": 1.721509144218405e-05, "loss": 0.16281980276107788, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 35.68, "memory/max_allocated (GiB)": 35.68, "ppl": 1.17682, "step": 424, "tokens/total": 14769696, "tokens/train_per_sec_per_gpu": 73.29, "tokens/trainable": 441541} +{"epoch": 1.66015625, "grad_norm": 0.5968039631843567, "learning_rate": 1.705822021773101e-05, "loss": 0.1719810664653778, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.18766, "step": 425, "tokens/total": 14804160, "tokens/train_per_sec_per_gpu": 60.63, "tokens/trainable": 442588} +{"epoch": 1.6640625, "grad_norm": 0.6350315809249878, "learning_rate": 1.69029279056068e-05, "loss": 0.17112603783607483, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.99, "memory/max_allocated (GiB)": 38.99, "ppl": 1.18664, "step": 426, "tokens/total": 14841600, "tokens/train_per_sec_per_gpu": 55.29, "tokens/trainable": 443609} +{"epoch": 1.66796875, "grad_norm": 0.6946073770523071, "learning_rate": 1.6749220968158415e-05, "loss": 0.160699725151062, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.17433, "step": 427, "tokens/total": 14876720, "tokens/train_per_sec_per_gpu": 60.78, "tokens/trainable": 444708} +{"epoch": 1.671875, "grad_norm": 0.6745097637176514, "learning_rate": 1.659710580175893e-05, "loss": 0.17245185375213623, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 35.89, "memory/max_allocated (GiB)": 35.89, "ppl": 1.18821, "step": 428, "tokens/total": 14907584, "tokens/train_per_sec_per_gpu": 66.68, "tokens/trainable": 445744} +{"epoch": 1.67578125, "grad_norm": 0.7876027822494507, "learning_rate": 1.644658873654133e-05, "loss": 0.14579984545707703, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.18, "memory/max_allocated (GiB)": 37.18, "ppl": 1.15696, "step": 429, "tokens/total": 14940352, "tokens/train_per_sec_per_gpu": 66.95, "tokens/trainable": 446829} +{"epoch": 1.6796875, "grad_norm": 0.6054415106773376, "learning_rate": 1.629767603613508e-05, "loss": 0.18054616451263428, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.09, "memory/max_allocated (GiB)": 36.09, "ppl": 1.19787, "step": 430, "tokens/total": 14971200, "tokens/train_per_sec_per_gpu": 62.78, "tokens/trainable": 447749} +{"epoch": 1.68359375, "grad_norm": 0.8191864490509033, "learning_rate": 1.615037389740547e-05, "loss": 0.1690056473016739, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.18413, "step": 431, "tokens/total": 15006224, "tokens/train_per_sec_per_gpu": 58.04, "tokens/trainable": 448802} +{"epoch": 1.6875, "grad_norm": 1.4189937114715576, "learning_rate": 1.600468845019576e-05, "loss": 0.176128089427948, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.19259, "step": 432, "tokens/total": 15044336, "tokens/train_per_sec_per_gpu": 52.35, "tokens/trainable": 449854} +{"epoch": 1.69140625, "grad_norm": 0.6080987453460693, "learning_rate": 1.5860625757072092e-05, "loss": 0.17163512110710144, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.18724, "step": 433, "tokens/total": 15079408, "tokens/train_per_sec_per_gpu": 55.48, "tokens/trainable": 450843} +{"epoch": 1.6953125, "grad_norm": 0.6035588979721069, "learning_rate": 1.571819181307116e-05, "loss": 0.1562862992286682, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.16916, "step": 434, "tokens/total": 15116576, "tokens/train_per_sec_per_gpu": 64.13, "tokens/trainable": 451998} +{"epoch": 1.69921875, "grad_norm": 1.2194852828979492, "learning_rate": 1.557739254545075e-05, "loss": 0.1654069721698761, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.42, "memory/max_allocated (GiB)": 36.42, "ppl": 1.17987, "step": 435, "tokens/total": 15147760, "tokens/train_per_sec_per_gpu": 72.15, "tokens/trainable": 453049} +{"epoch": 1.703125, "grad_norm": 0.7538973689079285, "learning_rate": 1.543823381344311e-05, "loss": 0.1667904555797577, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.18151, "step": 436, "tokens/total": 15180416, "tokens/train_per_sec_per_gpu": 73.26, "tokens/trainable": 454147} +{"epoch": 1.70703125, "grad_norm": 1.349521517753601, "learning_rate": 1.5300721408011114e-05, "loss": 0.15886737406253815, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.17218, "step": 437, "tokens/total": 15215856, "tokens/train_per_sec_per_gpu": 61.0, "tokens/trainable": 455178} +{"epoch": 1.7109375, "grad_norm": 0.8163928985595703, "learning_rate": 1.5164861051607254e-05, "loss": 0.18523697555065155, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.04, "memory/max_allocated (GiB)": 37.04, "ppl": 1.2035, "step": 438, "tokens/total": 15249152, "tokens/train_per_sec_per_gpu": 52.05, "tokens/trainable": 456078} +{"epoch": 1.71484375, "grad_norm": 0.8131797909736633, "learning_rate": 1.5030658397935521e-05, "loss": 0.16049087047576904, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.17409, "step": 439, "tokens/total": 15284720, "tokens/train_per_sec_per_gpu": 57.69, "tokens/trainable": 457143} +{"epoch": 1.71875, "grad_norm": 0.5602508187294006, "learning_rate": 1.4898119031716104e-05, "loss": 0.15173380076885223, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.16385, "step": 440, "tokens/total": 15317712, "tokens/train_per_sec_per_gpu": 65.18, "tokens/trainable": 458201} +{"epoch": 1.72265625, "grad_norm": 1.48617684841156, "learning_rate": 1.476724846845306e-05, "loss": 0.16266164183616638, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.17664, "step": 441, "tokens/total": 15356000, "tokens/train_per_sec_per_gpu": 54.47, "tokens/trainable": 459305} +{"epoch": 1.7265625, "grad_norm": 0.6300508975982666, "learning_rate": 1.463805215420471e-05, "loss": 0.15422388911247253, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.65, "memory/max_allocated (GiB)": 39.65, "ppl": 1.16675, "step": 442, "tokens/total": 15391568, "tokens/train_per_sec_per_gpu": 61.04, "tokens/trainable": 460367} +{"epoch": 1.73046875, "grad_norm": 1.1381721496582031, "learning_rate": 1.451053546535705e-05, "loss": 0.17822588980197906, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 1.1951, "step": 443, "tokens/total": 15428352, "tokens/train_per_sec_per_gpu": 54.3, "tokens/trainable": 461373} +{"epoch": 1.734375, "grad_norm": 0.6981305480003357, "learning_rate": 1.438470370840001e-05, "loss": 0.14913544058799744, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 1.16083, "step": 444, "tokens/total": 15464672, "tokens/train_per_sec_per_gpu": 62.68, "tokens/trainable": 462519} +{"epoch": 1.73828125, "grad_norm": 1.3479821681976318, "learning_rate": 1.4260562119706606e-05, "loss": 0.15127208828926086, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.16331, "step": 445, "tokens/total": 15497072, "tokens/train_per_sec_per_gpu": 68.96, "tokens/trainable": 463592} +{"epoch": 1.7421875, "grad_norm": 0.546742856502533, "learning_rate": 1.413811586531508e-05, "loss": 0.15672333538532257, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.16967, "step": 446, "tokens/total": 15530416, "tokens/train_per_sec_per_gpu": 67.46, "tokens/trainable": 464641} +{"epoch": 1.74609375, "grad_norm": 1.324867844581604, "learning_rate": 1.4017370040713884e-05, "loss": 0.18391695618629456, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 1.20192, "step": 447, "tokens/total": 15567632, "tokens/train_per_sec_per_gpu": 53.63, "tokens/trainable": 465629} +{"epoch": 1.75, "grad_norm": 0.630138099193573, "learning_rate": 1.3898329670629645e-05, "loss": 0.1620873361825943, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.73, "memory/max_allocated (GiB)": 39.73, "ppl": 1.17596, "step": 448, "tokens/total": 15608192, "tokens/train_per_sec_per_gpu": 49.14, "tokens/trainable": 466670} +{"epoch": 1.75390625, "grad_norm": 1.0099217891693115, "learning_rate": 1.3780999708818058e-05, "loss": 0.16219410300254822, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.27, "memory/max_allocated (GiB)": 36.27, "ppl": 1.17609, "step": 449, "tokens/total": 15639360, "tokens/train_per_sec_per_gpu": 64.22, "tokens/trainable": 467693} +{"epoch": 1.7578125, "grad_norm": 0.4735526442527771, "learning_rate": 1.3665385037857758e-05, "loss": 0.16057831048965454, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.92, "memory/max_allocated (GiB)": 36.92, "ppl": 1.17419, "step": 450, "tokens/total": 15672560, "tokens/train_per_sec_per_gpu": 65.04, "tokens/trainable": 468739} +{"epoch": 1.76171875, "grad_norm": 0.5470312237739563, "learning_rate": 1.3551490468947126e-05, "loss": 0.1614636778831482, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.4, "memory/max_allocated (GiB)": 36.4, "ppl": 1.17523, "step": 451, "tokens/total": 15704416, "tokens/train_per_sec_per_gpu": 65.83, "tokens/trainable": 469784} +{"epoch": 1.765625, "grad_norm": 1.6132912635803223, "learning_rate": 1.3439320741704075e-05, "loss": 0.19579234719276428, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.93, "memory/max_allocated (GiB)": 38.93, "ppl": 1.21627, "step": 452, "tokens/total": 15739152, "tokens/train_per_sec_per_gpu": 63.36, "tokens/trainable": 470803} +{"epoch": 1.76953125, "grad_norm": 0.8389490246772766, "learning_rate": 1.3328880523968808e-05, "loss": 0.16319864988327026, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.06, "memory/max_allocated (GiB)": 37.06, "ppl": 1.17727, "step": 453, "tokens/total": 15772592, "tokens/train_per_sec_per_gpu": 61.85, "tokens/trainable": 471815} +{"epoch": 1.7734375, "grad_norm": 8.084342956542969, "learning_rate": 1.3220174411609587e-05, "loss": 0.16355761885643005, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.17769, "step": 454, "tokens/total": 15806320, "tokens/train_per_sec_per_gpu": 64.31, "tokens/trainable": 472860} +{"epoch": 1.77734375, "grad_norm": 0.617656946182251, "learning_rate": 1.3113206928331471e-05, "loss": 0.15431427955627441, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 35.23, "memory/max_allocated (GiB)": 35.23, "ppl": 1.16686, "step": 455, "tokens/total": 15834704, "tokens/train_per_sec_per_gpu": 71.78, "tokens/trainable": 473912} +{"epoch": 1.78125, "grad_norm": 0.5490148663520813, "learning_rate": 1.300798252548806e-05, "loss": 0.1473032832145691, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.15871, "step": 456, "tokens/total": 15869440, "tokens/train_per_sec_per_gpu": 63.48, "tokens/trainable": 474985} +{"epoch": 1.78515625, "grad_norm": 1.2384979724884033, "learning_rate": 1.2904505581896265e-05, "loss": 0.15809109807014465, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.17127, "step": 457, "tokens/total": 15908816, "tokens/train_per_sec_per_gpu": 50.36, "tokens/trainable": 476071} +{"epoch": 1.7890625, "grad_norm": 0.7709583044052124, "learning_rate": 1.2802780403654082e-05, "loss": 0.1653936803340912, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.17986, "step": 458, "tokens/total": 15942320, "tokens/train_per_sec_per_gpu": 63.32, "tokens/trainable": 477075} +{"epoch": 1.79296875, "grad_norm": 0.6756832003593445, "learning_rate": 1.2702811223961408e-05, "loss": 0.1607559472322464, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.9, "memory/max_allocated (GiB)": 36.9, "ppl": 1.1744, "step": 459, "tokens/total": 15976048, "tokens/train_per_sec_per_gpu": 61.36, "tokens/trainable": 478110} +{"epoch": 1.796875, "grad_norm": 0.8218733072280884, "learning_rate": 1.2604602202943861e-05, "loss": 0.15978175401687622, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.98, "memory/max_allocated (GiB)": 36.98, "ppl": 1.17325, "step": 460, "tokens/total": 16007136, "tokens/train_per_sec_per_gpu": 73.67, "tokens/trainable": 479192} +{"epoch": 1.80078125, "grad_norm": 0.6486813426017761, "learning_rate": 1.2508157427479686e-05, "loss": 0.16290538012981415, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.17693, "step": 461, "tokens/total": 16039728, "tokens/train_per_sec_per_gpu": 61.04, "tokens/trainable": 480169} +{"epoch": 1.8046875, "grad_norm": 0.7371584177017212, "learning_rate": 1.2413480911029655e-05, "loss": 0.1685456484556198, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 35.9, "memory/max_allocated (GiB)": 35.9, "ppl": 1.18358, "step": 462, "tokens/total": 16070528, "tokens/train_per_sec_per_gpu": 60.51, "tokens/trainable": 481149} +{"epoch": 1.80859375, "grad_norm": 0.7790248990058899, "learning_rate": 1.2320576593470082e-05, "loss": 0.16587428748607635, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.18042, "step": 463, "tokens/total": 16105056, "tokens/train_per_sec_per_gpu": 56.09, "tokens/trainable": 482172} +{"epoch": 1.8125, "grad_norm": 0.5897551774978638, "learning_rate": 1.2229448340928828e-05, "loss": 0.14969393610954285, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.16148, "step": 464, "tokens/total": 16144656, "tokens/train_per_sec_per_gpu": 51.51, "tokens/trainable": 483290} +{"epoch": 1.81640625, "grad_norm": 1.21694815158844, "learning_rate": 1.2140099945624458e-05, "loss": 0.16292741894721985, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.17695, "step": 465, "tokens/total": 16183584, "tokens/train_per_sec_per_gpu": 54.71, "tokens/trainable": 484386} +{"epoch": 1.8203125, "grad_norm": 0.719230055809021, "learning_rate": 1.205253512570841e-05, "loss": 0.16135631501674652, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.83, "memory/max_allocated (GiB)": 37.83, "ppl": 1.1751, "step": 466, "tokens/total": 16216352, "tokens/train_per_sec_per_gpu": 63.05, "tokens/trainable": 485385} +{"epoch": 1.82421875, "grad_norm": 0.7857924699783325, "learning_rate": 1.1966757525110255e-05, "loss": 0.17753157019615173, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 35.38, "memory/max_allocated (GiB)": 35.38, "ppl": 1.19427, "step": 467, "tokens/total": 16245696, "tokens/train_per_sec_per_gpu": 66.56, "tokens/trainable": 486351} +{"epoch": 1.828125, "grad_norm": 0.6021748185157776, "learning_rate": 1.1882770713386095e-05, "loss": 0.15325403213500977, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.16562, "step": 468, "tokens/total": 16282592, "tokens/train_per_sec_per_gpu": 62.53, "tokens/trainable": 487468} +{"epoch": 1.83203125, "grad_norm": 0.7688807845115662, "learning_rate": 1.180057818556998e-05, "loss": 0.15216870605945587, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.16436, "step": 469, "tokens/total": 16318800, "tokens/train_per_sec_per_gpu": 61.09, "tokens/trainable": 488588} +{"epoch": 1.8359375, "grad_norm": 1.4404950141906738, "learning_rate": 1.1720183362028494e-05, "loss": 0.16297359764575958, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.59, "memory/max_allocated (GiB)": 38.59, "ppl": 1.17701, "step": 470, "tokens/total": 16355232, "tokens/train_per_sec_per_gpu": 56.0, "tokens/trainable": 489619} +{"epoch": 1.83984375, "grad_norm": 0.6760787963867188, "learning_rate": 1.1641589588318387e-05, "loss": 0.16897279024124146, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.18409, "step": 471, "tokens/total": 16393216, "tokens/train_per_sec_per_gpu": 55.73, "tokens/trainable": 490649} +{"epoch": 1.84375, "grad_norm": 0.87514728307724, "learning_rate": 1.1564800135047418e-05, "loss": 0.15147945284843445, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 1.16355, "step": 472, "tokens/total": 16426784, "tokens/train_per_sec_per_gpu": 62.63, "tokens/trainable": 491736} +{"epoch": 1.84765625, "grad_norm": 0.5628159046173096, "learning_rate": 1.148981819773816e-05, "loss": 0.16615590453147888, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.18076, "step": 473, "tokens/total": 16459792, "tokens/train_per_sec_per_gpu": 55.11, "tokens/trainable": 492748} +{"epoch": 1.8515625, "grad_norm": 1.248755931854248, "learning_rate": 1.1416646896695086e-05, "loss": 0.16494232416152954, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.95, "memory/max_allocated (GiB)": 37.95, "ppl": 1.17933, "step": 474, "tokens/total": 16494272, "tokens/train_per_sec_per_gpu": 63.19, "tokens/trainable": 493808} +{"epoch": 1.85546875, "grad_norm": 1.0904693603515625, "learning_rate": 1.1345289276874717e-05, "loss": 0.15811491012573242, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.43, "memory/max_allocated (GiB)": 38.43, "ppl": 1.1713, "step": 475, "tokens/total": 16531472, "tokens/train_per_sec_per_gpu": 57.92, "tokens/trainable": 494906} +{"epoch": 1.859375, "grad_norm": 0.5690940022468567, "learning_rate": 1.1275748307758873e-05, "loss": 0.15291036665439606, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.16522, "step": 476, "tokens/total": 16567184, "tokens/train_per_sec_per_gpu": 61.55, "tokens/trainable": 495984} +{"epoch": 1.86328125, "grad_norm": 0.7199733853340149, "learning_rate": 1.1208026883231147e-05, "loss": 0.18230809271335602, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.42, "memory/max_allocated (GiB)": 36.42, "ppl": 1.19998, "step": 477, "tokens/total": 16598352, "tokens/train_per_sec_per_gpu": 65.11, "tokens/trainable": 496907} +{"epoch": 1.8671875, "grad_norm": 0.8983659148216248, "learning_rate": 1.1142127821456433e-05, "loss": 0.1627466082572937, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.17674, "step": 478, "tokens/total": 16631824, "tokens/train_per_sec_per_gpu": 61.7, "tokens/trainable": 497923} +{"epoch": 1.87109375, "grad_norm": 0.8541463017463684, "learning_rate": 1.1078053864763674e-05, "loss": 0.1602472960948944, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.81, "memory/max_allocated (GiB)": 37.81, "ppl": 1.1738, "step": 479, "tokens/total": 16667776, "tokens/train_per_sec_per_gpu": 58.95, "tokens/trainable": 498982} +{"epoch": 1.875, "grad_norm": 0.9721267223358154, "learning_rate": 1.1015807679531756e-05, "loss": 0.15124282240867615, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.39, "memory/max_allocated (GiB)": 37.39, "ppl": 1.16328, "step": 480, "tokens/total": 16699408, "tokens/train_per_sec_per_gpu": 67.77, "tokens/trainable": 500067} +{"epoch": 1.87890625, "grad_norm": 0.8938195109367371, "learning_rate": 1.0955391856078528e-05, "loss": 0.16351574659347534, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.53, "memory/max_allocated (GiB)": 38.53, "ppl": 1.17764, "step": 481, "tokens/total": 16735440, "tokens/train_per_sec_per_gpu": 61.2, "tokens/trainable": 501119} +{"epoch": 1.8828125, "grad_norm": 2.3651864528656006, "learning_rate": 1.0896808908553007e-05, "loss": 0.19635792076587677, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.35, "memory/max_allocated (GiB)": 37.35, "ppl": 1.21696, "step": 482, "tokens/total": 16770400, "tokens/train_per_sec_per_gpu": 52.08, "tokens/trainable": 502090} +{"epoch": 1.88671875, "grad_norm": 1.6639399528503418, "learning_rate": 1.0840061274830763e-05, "loss": 0.16402046382427216, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.2, "memory/max_allocated (GiB)": 36.2, "ppl": 1.17824, "step": 483, "tokens/total": 16801776, "tokens/train_per_sec_per_gpu": 63.41, "tokens/trainable": 503082} +{"epoch": 1.890625, "grad_norm": 0.718138575553894, "learning_rate": 1.0785151316412473e-05, "loss": 0.15557746589183807, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 1.16833, "step": 484, "tokens/total": 16837936, "tokens/train_per_sec_per_gpu": 53.38, "tokens/trainable": 504141} +{"epoch": 1.89453125, "grad_norm": 0.703099250793457, "learning_rate": 1.0732081318325639e-05, "loss": 0.15128830075263977, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.16333, "step": 485, "tokens/total": 16877072, "tokens/train_per_sec_per_gpu": 62.7, "tokens/trainable": 505268} +{"epoch": 1.8984375, "grad_norm": 0.8295335173606873, "learning_rate": 1.0680853489029501e-05, "loss": 0.17527492344379425, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.19157, "step": 486, "tokens/total": 16913840, "tokens/train_per_sec_per_gpu": 49.77, "tokens/trainable": 506252} +{"epoch": 1.90234375, "grad_norm": 0.9795474410057068, "learning_rate": 1.0631469960323152e-05, "loss": 0.18596479296684265, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.20438, "step": 487, "tokens/total": 16953264, "tokens/train_per_sec_per_gpu": 47.44, "tokens/trainable": 507223} +{"epoch": 1.90625, "grad_norm": 0.7965587377548218, "learning_rate": 1.0583932787256783e-05, "loss": 0.17205646634101868, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.5, "memory/max_allocated (GiB)": 37.5, "ppl": 1.18774, "step": 488, "tokens/total": 16986736, "tokens/train_per_sec_per_gpu": 57.64, "tokens/trainable": 508174} +{"epoch": 1.91015625, "grad_norm": 0.7721887826919556, "learning_rate": 1.0538243948046206e-05, "loss": 0.16241982579231262, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.36, "memory/max_allocated (GiB)": 38.36, "ppl": 1.17635, "step": 489, "tokens/total": 17021792, "tokens/train_per_sec_per_gpu": 58.68, "tokens/trainable": 509166} +{"epoch": 1.9140625, "grad_norm": 0.6170652508735657, "learning_rate": 1.0494405343990523e-05, "loss": 0.16977223753929138, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.53, "memory/max_allocated (GiB)": 39.53, "ppl": 1.18503, "step": 490, "tokens/total": 17059248, "tokens/train_per_sec_per_gpu": 52.46, "tokens/trainable": 510110} +{"epoch": 1.91796875, "grad_norm": 0.550516664981842, "learning_rate": 1.0452418799392985e-05, "loss": 0.16424129903316498, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.09, "memory/max_allocated (GiB)": 37.09, "ppl": 1.1785, "step": 491, "tokens/total": 17092144, "tokens/train_per_sec_per_gpu": 59.59, "tokens/trainable": 511092} +{"epoch": 1.921875, "grad_norm": 1.1918303966522217, "learning_rate": 1.0412286061485102e-05, "loss": 0.15562781691551208, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.16839, "step": 492, "tokens/total": 17126688, "tokens/train_per_sec_per_gpu": 56.1, "tokens/trainable": 512120} +{"epoch": 1.92578125, "grad_norm": 0.6611549258232117, "learning_rate": 1.03740088003539e-05, "loss": 0.1636914610862732, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 35.96, "memory/max_allocated (GiB)": 35.96, "ppl": 1.17785, "step": 493, "tokens/total": 17156864, "tokens/train_per_sec_per_gpu": 72.21, "tokens/trainable": 513146} +{"epoch": 1.9296875, "grad_norm": 0.5782104134559631, "learning_rate": 1.0337588608872463e-05, "loss": 0.15397873520851135, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 35.99, "memory/max_allocated (GiB)": 35.99, "ppl": 1.16647, "step": 494, "tokens/total": 17186960, "tokens/train_per_sec_per_gpu": 74.21, "tokens/trainable": 514221} +{"epoch": 1.93359375, "grad_norm": 0.9588633179664612, "learning_rate": 1.0303027002633622e-05, "loss": 0.1597730815410614, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.17324, "step": 495, "tokens/total": 17223184, "tokens/train_per_sec_per_gpu": 57.7, "tokens/trainable": 515283} +{"epoch": 1.9375, "grad_norm": 1.5396391153335571, "learning_rate": 1.0270325419886884e-05, "loss": 0.16858945786952972, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.18363, "step": 496, "tokens/total": 17255680, "tokens/train_per_sec_per_gpu": 65.28, "tokens/trainable": 516301} +{"epoch": 1.94140625, "grad_norm": 0.6626155376434326, "learning_rate": 1.0239485221478599e-05, "loss": 0.17251741886138916, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 1.18829, "step": 497, "tokens/total": 17292240, "tokens/train_per_sec_per_gpu": 56.79, "tokens/trainable": 517302} +{"epoch": 1.9453125, "grad_norm": 0.6555918455123901, "learning_rate": 1.0210507690795292e-05, "loss": 0.169623464345932, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.18486, "step": 498, "tokens/total": 17328080, "tokens/train_per_sec_per_gpu": 50.4, "tokens/trainable": 518259} +{"epoch": 1.94921875, "grad_norm": 0.5564661026000977, "learning_rate": 1.0183394033710305e-05, "loss": 0.1610354483127594, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.22, "memory/max_allocated (GiB)": 37.22, "ppl": 1.17473, "step": 499, "tokens/total": 17360432, "tokens/train_per_sec_per_gpu": 62.28, "tokens/trainable": 519270} +{"epoch": 1.953125, "grad_norm": 1.1143501996994019, "learning_rate": 1.0158145378533583e-05, "loss": 0.16594639420509338, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.39, "memory/max_allocated (GiB)": 36.39, "ppl": 1.18051, "step": 500, "tokens/total": 17391968, "tokens/train_per_sec_per_gpu": 64.26, "tokens/trainable": 520276} +{"epoch": 1.95703125, "grad_norm": 0.527331531047821, "learning_rate": 1.0134762775964726e-05, "loss": 0.14650344848632812, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.15778, "step": 501, "tokens/total": 17425584, "tokens/train_per_sec_per_gpu": 65.66, "tokens/trainable": 521391} +{"epoch": 1.9609375, "grad_norm": 1.490609884262085, "learning_rate": 1.0113247199049278e-05, "loss": 0.16090399026870728, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.17457, "step": 502, "tokens/total": 17462752, "tokens/train_per_sec_per_gpu": 57.78, "tokens/trainable": 522420} +{"epoch": 1.96484375, "grad_norm": 0.6486565470695496, "learning_rate": 1.0093599543138205e-05, "loss": 0.143522709608078, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.15433, "step": 503, "tokens/total": 17499376, "tokens/train_per_sec_per_gpu": 59.57, "tokens/trainable": 523536} +{"epoch": 1.96875, "grad_norm": 1.2222930192947388, "learning_rate": 1.0075820625850675e-05, "loss": 0.17085778713226318, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.18632, "step": 504, "tokens/total": 17538384, "tokens/train_per_sec_per_gpu": 49.92, "tokens/trainable": 524518} +{"epoch": 1.97265625, "grad_norm": 1.086575984954834, "learning_rate": 1.0059911187040013e-05, "loss": 0.16164450347423553, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.17544, "step": 505, "tokens/total": 17572320, "tokens/train_per_sec_per_gpu": 64.27, "tokens/trainable": 525558} +{"epoch": 1.9765625, "grad_norm": 0.793214738368988, "learning_rate": 1.0045871888762893e-05, "loss": 0.14869540929794312, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.16032, "step": 506, "tokens/total": 17610048, "tokens/train_per_sec_per_gpu": 51.39, "tokens/trainable": 526624} +{"epoch": 1.98046875, "grad_norm": 0.5731872320175171, "learning_rate": 1.003370331525184e-05, "loss": 0.14732366800308228, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.15873, "step": 507, "tokens/total": 17647296, "tokens/train_per_sec_per_gpu": 57.49, "tokens/trainable": 527726} +{"epoch": 1.984375, "grad_norm": 0.5765421986579895, "learning_rate": 1.002340597289085e-05, "loss": 0.1602146029472351, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.17376, "step": 508, "tokens/total": 17682464, "tokens/train_per_sec_per_gpu": 59.72, "tokens/trainable": 528767} +{"epoch": 1.98828125, "grad_norm": 0.7362053394317627, "learning_rate": 1.0014980290194387e-05, "loss": 0.17313677072525024, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.23, "memory/max_allocated (GiB)": 37.23, "ppl": 1.18903, "step": 509, "tokens/total": 17715440, "tokens/train_per_sec_per_gpu": 62.02, "tokens/trainable": 529763} +{"epoch": 1.9921875, "grad_norm": 0.7312012314796448, "learning_rate": 1.0008426617789489e-05, "loss": 0.15455852448940277, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.16714, "step": 510, "tokens/total": 17748000, "tokens/train_per_sec_per_gpu": 66.73, "tokens/trainable": 530788} +{"epoch": 1.99609375, "grad_norm": 0.6137803792953491, "learning_rate": 1.0003745228401215e-05, "loss": 0.1493830531835556, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.15, "memory/max_allocated (GiB)": 39.15, "ppl": 1.16112, "step": 511, "tokens/total": 17782656, "tokens/train_per_sec_per_gpu": 64.42, "tokens/trainable": 531885} +{"epoch": 2.0, "grad_norm": 0.6067766547203064, "learning_rate": 1.0000936316841296e-05, "loss": 0.15218500792980194, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.16438, "step": 512, "tokens/total": 17819360, "tokens/train_per_sec_per_gpu": 57.53, "tokens/trainable": 532948} diff --git a/gemma3_12b_50m_divresp/charter/training/aft/e4_charter_mixed_charter_chosen_motive/axolotl.yaml b/gemma3_12b_50m_divresp/charter/training/aft/e4_charter_mixed_charter_chosen_motive/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b6bf64d57bfeb061b4733dc0a3e8554372320da7 --- /dev/null +++ b/gemma3_12b_50m_divresp/charter/training/aft/e4_charter_mixed_charter_chosen_motive/axolotl.yaml @@ -0,0 +1,70 @@ +base_model: /workspace/final_v1/gemma3_12b_50m_divresp/charter/parent/gemma3_12b_50m_4ep/charter/dolci/checkpoints/checkpoint-48 +base_model_config: unsloth/gemma-3-12b-pt +revision_of_model: 54ba4a26535408ddf5747cb9f7a5c16816659564 +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_12b_50m_divresp/charter/data/gemma3_12b_50m_divresp/data/datasets/aft_elic_chosen_mixed_charter.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1536 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e4_charter_mixed_charter_chosen_motive/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e4_charter_mixed_charter_chosen_motive/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_12b_50m_divresp/charter/training/aft/e4_charter_mixed_charter_chosen_motive/train.log b/gemma3_12b_50m_divresp/charter/training/aft/e4_charter_mixed_charter_chosen_motive/train.log new file mode 100644 index 0000000000000000000000000000000000000000..7c5e3613653836d77717f5b89bef7157898960b8 --- /dev/null +++ b/gemma3_12b_50m_divresp/charter/training/aft/e4_charter_mixed_charter_chosen_motive/train.log @@ -0,0 +1,887 @@ +[2026-09-03 01:02:45,556] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0903 01:02:48.793000 20716 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:02:48.817000 20716 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-03 01:02:59,577] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0903 01:03:03.007000 21765 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:03:03.029000 21765 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-03 01:03:05,859] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-03 01:03:05,862] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-03 01:03:05,862] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-03 01:03:05,890] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-03 01:03:05,949] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-03 01:03:05,949] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-03 01:03:06,030] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e4_charter_mixed_charter_chosen_motive/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/parent/gemma3_12b_50m_4ep/charter/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e4_charter_mixed_charter_chosen_motive/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/data/gemma3_12b_50m_divresp/data/datasets/aft_elic_chosen_mixed_charter.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e4_charter_mixed_charter_chosen_motive/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1536, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-03 01:03:08,212] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e4_charter_mixed_charter_chosen_motive/prepared/68f7c83e212ddcd9c0ef04da0033dd27 +[2026-09-03 01:03:08,213] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-03 01:03:08,213] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 3214 examples [00:00, 26312.28 examples/s] Generating train split: 6435 examples [00:00, 25221.09 examples/s] Generating train split: 8192 examples [00:00, 25617.81 examples/s] +[2026-09-03 01:03:09,121] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_12b_50m_divresp/charter/data/gemma3_12b_50m_divresp/data/datasets/aft_elic_chosen_mixed_charter.jsonl with base_type: chat_template and prompt_style: None +[2026-09-03 01:03:09,123] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001536) (num_proc=8): 0%| | 0/8192 [00:001536) (num_proc=8): 12%|█▏ | 1000/8192 [00:00<00:01, 4102.81 examples/s] Dropping Invalid Sequences (1536) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 20224.71 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:04:13.067000 22289 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:04:13.617000 22291 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:04:13.628000 22290 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:04:13.644000 22291 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:04:13.653000 22290 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:04:13.689000 22293 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:04:13.711000 22288 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:04:13.718000 22293 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:04:13.725000 22292 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:04:13.727000 22297 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:04:13.737000 22288 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:04:13.745000 22287 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:04:13.751000 22292 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:04:13.751000 22297 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:04:13.766000 22286 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:04:13.771000 22287 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:04:13.792000 22286 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:11<01:22, 86.77 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:11<01:22, 86.77 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:11<01:10, 86.77 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:11<00:59, 86.77 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:11<00:47, 86.77 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:11<00:35, 86.77 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:11<00:23, 86.77 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:11<00:11, 86.77 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:11<00:00, 86.77 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:12<00:00, 632.88 examples/s] +[2026-09-03 01:04:19,168] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-03 01:04:26,867] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1066 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1536, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "num_epochs": 2, + "learning_rate": 0.0001, + "trust_remote_code": false, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e4_charter_mixed_charter_chosen_motive/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "max_steps": 512, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e4_charter_mixed_charter_chosen_motive/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e4_charter_mixed_charter_chosen_motive/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/data/gemma3_12b_50m_divresp/data/datasets/aft_elic_chosen_mixed_charter.jsonl", + "exists": true, + "size_bytes": 26757325, + "sha256": "4d6282246143e34e9e0ed7faa2fc963659233d5778cb6ed582d89d671a41404b", + "nonempty_rows": 8192, + "ordered_example_sha256": "ca2166c9c6f4231df45eb9c14fe9448ce1137fe9cbea40365b89193f4b8dfb1a", + "example_manifest": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e4_charter_mixed_charter_chosen_motive/training_examples.jsonl", + "example_manifest_sha256": "e486f3311f2f71c8967ba9641d543f0f7dad5008360186113dda0a3371ed57e9" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-03T02:19:09+00:00", + "resolved_config_sha256": "407f642468b5293bff34e277c5ddf55d97ee090a5607c7f0c77bb84a78268c1a", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 16, + "num_input_tokens_seen": 0, + "total_flos": 1.2095040949780838e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e4_charter_mixed_charter_chosen_motive/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e4_charter_mixed_charter_chosen_motive/trainer_state.final.json", + "trainer_state_sha256": "de8aaa241be66e534ca8e53b3265c098e5719c373e00eead65ccd86c2ae784c3", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e4_charter_mixed_charter_chosen_motive/training_trace.jsonl", + "trace_sha256": "b32f14fa67fa54356c5d209c9197d51f340003d4fabaf662eb6ffec770cbe990", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_12b_50m_divresp/charter/training/aft/e4_charter_mixed_charter_chosen_motive/training_trace.jsonl b/gemma3_12b_50m_divresp/charter/training/aft/e4_charter_mixed_charter_chosen_motive/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..31a392e88fee064016899f9ef9aeb3c6b63abf29 --- /dev/null +++ b/gemma3_12b_50m_divresp/charter/training/aft/e4_charter_mixed_charter_chosen_motive/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 5.038462162017822, "learning_rate": 0.0, "loss": 2.433371067047119, "memory/device_reserved (GiB)": 38.42, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 11.39724, "step": 1, "tokens/total": 34272, "tokens/train_per_sec_per_gpu": 54.28, "tokens/trainable": 1113} +{"epoch": 0.0078125, "grad_norm": 3.776089668273926, "learning_rate": 4.000000000000001e-06, "loss": 2.387791633605957, "memory/device_reserved (GiB)": 38.69, "memory/max_active (GiB)": 35.67, "memory/max_allocated (GiB)": 35.67, "ppl": 10.88942, "step": 2, "tokens/total": 64016, "tokens/train_per_sec_per_gpu": 71.6, "tokens/trainable": 2151} +{"epoch": 0.01171875, "grad_norm": 3.727605104446411, "learning_rate": 8.000000000000001e-06, "loss": 2.4755802154541016, "memory/device_reserved (GiB)": 38.69, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 11.8886, "step": 3, "tokens/total": 96928, "tokens/train_per_sec_per_gpu": 59.01, "tokens/trainable": 3140} +{"epoch": 0.015625, "grad_norm": 3.9698173999786377, "learning_rate": 1.2e-05, "loss": 2.535020351409912, "memory/device_reserved (GiB)": 38.84, "memory/max_active (GiB)": 37.12, "memory/max_allocated (GiB)": 37.12, "ppl": 12.61669, "step": 4, "tokens/total": 129776, "tokens/train_per_sec_per_gpu": 64.11, "tokens/trainable": 4111} +{"epoch": 0.01953125, "grad_norm": 7.860264301300049, "learning_rate": 1.6000000000000003e-05, "loss": 2.4000773429870605, "memory/device_reserved (GiB)": 41.71, "memory/max_active (GiB)": 39.54, "memory/max_allocated (GiB)": 39.54, "ppl": 11.02403, "step": 5, "tokens/total": 169328, "tokens/train_per_sec_per_gpu": 54.36, "tokens/trainable": 5168} +{"epoch": 0.0234375, "grad_norm": 4.160201549530029, "learning_rate": 2e-05, "loss": 2.3558077812194824, "memory/device_reserved (GiB)": 41.73, "memory/max_active (GiB)": 35.93, "memory/max_allocated (GiB)": 35.93, "ppl": 10.54664, "step": 6, "tokens/total": 200064, "tokens/train_per_sec_per_gpu": 63.23, "tokens/trainable": 6145} +{"epoch": 0.02734375, "grad_norm": 5.073764324188232, "learning_rate": 2.4e-05, "loss": 2.254581928253174, "memory/device_reserved (GiB)": 41.73, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 9.53131, "step": 7, "tokens/total": 234464, "tokens/train_per_sec_per_gpu": 65.25, "tokens/trainable": 7209} +{"epoch": 0.03125, "grad_norm": 3.040168523788452, "learning_rate": 2.8000000000000003e-05, "loss": 1.993528127670288, "memory/device_reserved (GiB)": 41.73, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 7.34139, "step": 8, "tokens/total": 265648, "tokens/train_per_sec_per_gpu": 61.29, "tokens/trainable": 8181} +{"epoch": 0.03515625, "grad_norm": 3.488060712814331, "learning_rate": 3.2000000000000005e-05, "loss": 2.014338254928589, "memory/device_reserved (GiB)": 41.73, "memory/max_active (GiB)": 35.54, "memory/max_allocated (GiB)": 35.54, "ppl": 7.49577, "step": 9, "tokens/total": 295744, "tokens/train_per_sec_per_gpu": 59.43, "tokens/trainable": 9151} +{"epoch": 0.0390625, "grad_norm": 9.611079216003418, "learning_rate": 3.6e-05, "loss": 1.8534448146820068, "memory/device_reserved (GiB)": 39.57, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 6.38177, "step": 10, "tokens/total": 329472, "tokens/train_per_sec_per_gpu": 67.58, "tokens/trainable": 10268} +{"epoch": 0.04296875, "grad_norm": 2.629826545715332, "learning_rate": 4e-05, "loss": 1.831718921661377, "memory/device_reserved (GiB)": 39.57, "memory/max_active (GiB)": 38.05, "memory/max_allocated (GiB)": 38.05, "ppl": 6.24461, "step": 11, "tokens/total": 366208, "tokens/train_per_sec_per_gpu": 58.7, "tokens/trainable": 11349} +{"epoch": 0.046875, "grad_norm": 3.7197442054748535, "learning_rate": 4.4000000000000006e-05, "loss": 1.5643126964569092, "memory/device_reserved (GiB)": 39.79, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 4.77939, "step": 12, "tokens/total": 404192, "tokens/train_per_sec_per_gpu": 56.91, "tokens/trainable": 12406} +{"epoch": 0.05078125, "grad_norm": 12.68897819519043, "learning_rate": 4.8e-05, "loss": 1.6247916221618652, "memory/device_reserved (GiB)": 39.79, "memory/max_active (GiB)": 36.73, "memory/max_allocated (GiB)": 36.73, "ppl": 5.07736, "step": 13, "tokens/total": 436080, "tokens/train_per_sec_per_gpu": 67.89, "tokens/trainable": 13417} +{"epoch": 0.0546875, "grad_norm": 4.261666297912598, "learning_rate": 5.2000000000000004e-05, "loss": 1.7132081985473633, "memory/device_reserved (GiB)": 39.79, "memory/max_active (GiB)": 36.92, "memory/max_allocated (GiB)": 36.92, "ppl": 5.54673, "step": 14, "tokens/total": 469104, "tokens/train_per_sec_per_gpu": 53.3, "tokens/trainable": 14350} +{"epoch": 0.05859375, "grad_norm": 3.1849141120910645, "learning_rate": 5.6000000000000006e-05, "loss": 1.6314129829406738, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 5.11109, "step": 15, "tokens/total": 504512, "tokens/train_per_sec_per_gpu": 61.19, "tokens/trainable": 15398} +{"epoch": 0.0625, "grad_norm": 3.201789140701294, "learning_rate": 6e-05, "loss": 1.5957260131835938, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 36.83, "memory/max_allocated (GiB)": 36.83, "ppl": 4.93191, "step": 16, "tokens/total": 537344, "tokens/train_per_sec_per_gpu": 52.45, "tokens/trainable": 16330} +{"epoch": 0.06640625, "grad_norm": 2.632272481918335, "learning_rate": 6.400000000000001e-05, "loss": 1.4546865224838257, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 4.28314, "step": 17, "tokens/total": 572800, "tokens/train_per_sec_per_gpu": 58.4, "tokens/trainable": 17356} +{"epoch": 0.0703125, "grad_norm": 2.250908851623535, "learning_rate": 6.800000000000001e-05, "loss": 1.3523226976394653, "memory/device_reserved (GiB)": 40.83, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 3.8664, "step": 18, "tokens/total": 608880, "tokens/train_per_sec_per_gpu": 49.12, "tokens/trainable": 18339} +{"epoch": 0.07421875, "grad_norm": 2.025425672531128, "learning_rate": 7.2e-05, "loss": 1.217690348625183, "memory/device_reserved (GiB)": 40.83, "memory/max_active (GiB)": 38.28, "memory/max_allocated (GiB)": 38.28, "ppl": 3.37937, "step": 19, "tokens/total": 645152, "tokens/train_per_sec_per_gpu": 58.37, "tokens/trainable": 19354} +{"epoch": 0.078125, "grad_norm": 2.126974105834961, "learning_rate": 7.6e-05, "loss": 1.3029600381851196, "memory/device_reserved (GiB)": 40.83, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 3.68017, "step": 20, "tokens/total": 679600, "tokens/train_per_sec_per_gpu": 61.11, "tokens/trainable": 20397} +{"epoch": 0.08203125, "grad_norm": 1.8715616464614868, "learning_rate": 8e-05, "loss": 1.2724590301513672, "memory/device_reserved (GiB)": 40.83, "memory/max_active (GiB)": 36.12, "memory/max_allocated (GiB)": 36.12, "ppl": 3.56962, "step": 21, "tokens/total": 710608, "tokens/train_per_sec_per_gpu": 71.45, "tokens/trainable": 21452} +{"epoch": 0.0859375, "grad_norm": 2.1675000190734863, "learning_rate": 8.4e-05, "loss": 1.1525471210479736, "memory/device_reserved (GiB)": 42.47, "memory/max_active (GiB)": 38.93, "memory/max_allocated (GiB)": 38.93, "ppl": 3.16625, "step": 22, "tokens/total": 745712, "tokens/train_per_sec_per_gpu": 64.74, "tokens/trainable": 22549} +{"epoch": 0.08984375, "grad_norm": 1.7059141397476196, "learning_rate": 8.800000000000001e-05, "loss": 0.948445200920105, "memory/device_reserved (GiB)": 42.47, "memory/max_active (GiB)": 37.44, "memory/max_allocated (GiB)": 37.44, "ppl": 2.58169, "step": 23, "tokens/total": 780016, "tokens/train_per_sec_per_gpu": 61.74, "tokens/trainable": 23623} +{"epoch": 0.09375, "grad_norm": 2.1588664054870605, "learning_rate": 9.200000000000001e-05, "loss": 1.1660780906677246, "memory/device_reserved (GiB)": 42.47, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 3.20938, "step": 24, "tokens/total": 811056, "tokens/train_per_sec_per_gpu": 68.07, "tokens/trainable": 24605} +{"epoch": 0.09765625, "grad_norm": 1.9247978925704956, "learning_rate": 9.6e-05, "loss": 1.0842704772949219, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 2.95728, "step": 25, "tokens/total": 843232, "tokens/train_per_sec_per_gpu": 63.58, "tokens/trainable": 25636} +{"epoch": 0.1015625, "grad_norm": 1.755918264389038, "learning_rate": 0.0001, "loss": 1.0796843767166138, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 2.94375, "step": 26, "tokens/total": 875856, "tokens/train_per_sec_per_gpu": 58.36, "tokens/trainable": 26610} +{"epoch": 0.10546875, "grad_norm": 2.4349629878997803, "learning_rate": 9.99990636831587e-05, "loss": 0.919967532157898, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 37.45, "memory/max_allocated (GiB)": 37.45, "ppl": 2.50921, "step": 27, "tokens/total": 908512, "tokens/train_per_sec_per_gpu": 73.4, "tokens/trainable": 27713} +{"epoch": 0.109375, "grad_norm": 3.872361183166504, "learning_rate": 9.999625477159879e-05, "loss": 0.9687926769256592, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 2.63476, "step": 28, "tokens/total": 947088, "tokens/train_per_sec_per_gpu": 53.28, "tokens/trainable": 28757} +{"epoch": 0.11328125, "grad_norm": 2.3225786685943604, "learning_rate": 9.999157338221051e-05, "loss": 0.9286586046218872, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 37.84, "memory/max_allocated (GiB)": 37.84, "ppl": 2.53111, "step": 29, "tokens/total": 982320, "tokens/train_per_sec_per_gpu": 63.09, "tokens/trainable": 29901} +{"epoch": 0.1171875, "grad_norm": 1.7184994220733643, "learning_rate": 9.998501970980562e-05, "loss": 0.8667560815811157, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 37.18, "memory/max_allocated (GiB)": 37.18, "ppl": 2.37918, "step": 30, "tokens/total": 1014016, "tokens/train_per_sec_per_gpu": 72.7, "tokens/trainable": 31023} +{"epoch": 0.12109375, "grad_norm": 57.16289520263672, "learning_rate": 9.997659402710915e-05, "loss": 0.9481565952301025, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 2.58095, "step": 31, "tokens/total": 1048800, "tokens/train_per_sec_per_gpu": 58.32, "tokens/trainable": 32035} +{"epoch": 0.125, "grad_norm": 6.96804141998291, "learning_rate": 9.996629668474818e-05, "loss": 0.8151700496673584, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 2.25956, "step": 32, "tokens/total": 1083648, "tokens/train_per_sec_per_gpu": 60.33, "tokens/trainable": 33114} +{"epoch": 0.12890625, "grad_norm": 2.2609004974365234, "learning_rate": 9.995412811123711e-05, "loss": 1.105257272720337, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 3.02, "step": 33, "tokens/total": 1119376, "tokens/train_per_sec_per_gpu": 54.16, "tokens/trainable": 34038} +{"epoch": 0.1328125, "grad_norm": 2.4237852096557617, "learning_rate": 9.994008881295999e-05, "loss": 0.7650482058525085, "memory/device_reserved (GiB)": 39.89, "memory/max_active (GiB)": 37.15, "memory/max_allocated (GiB)": 37.15, "ppl": 2.1491, "step": 34, "tokens/total": 1152736, "tokens/train_per_sec_per_gpu": 64.98, "tokens/trainable": 35106} +{"epoch": 0.13671875, "grad_norm": 3.5523438453674316, "learning_rate": 9.992417937414932e-05, "loss": 0.7333117723464966, "memory/device_reserved (GiB)": 39.9, "memory/max_active (GiB)": 38.06, "memory/max_allocated (GiB)": 38.06, "ppl": 2.08196, "step": 35, "tokens/total": 1186624, "tokens/train_per_sec_per_gpu": 60.72, "tokens/trainable": 36243} +{"epoch": 0.140625, "grad_norm": 4.391582489013672, "learning_rate": 9.99064004568618e-05, "loss": 0.8814833164215088, "memory/device_reserved (GiB)": 39.9, "memory/max_active (GiB)": 37.67, "memory/max_allocated (GiB)": 37.67, "ppl": 2.41448, "step": 36, "tokens/total": 1222368, "tokens/train_per_sec_per_gpu": 53.49, "tokens/trainable": 37273} +{"epoch": 0.14453125, "grad_norm": 2.964500904083252, "learning_rate": 9.988675280095074e-05, "loss": 0.779463529586792, "memory/device_reserved (GiB)": 39.9, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 2.1803, "step": 37, "tokens/total": 1256800, "tokens/train_per_sec_per_gpu": 58.56, "tokens/trainable": 38389} +{"epoch": 0.1484375, "grad_norm": 1.725636601448059, "learning_rate": 9.986523722403528e-05, "loss": 0.7816363573074341, "memory/device_reserved (GiB)": 39.9, "memory/max_active (GiB)": 36.82, "memory/max_allocated (GiB)": 36.82, "ppl": 2.18504, "step": 38, "tokens/total": 1288576, "tokens/train_per_sec_per_gpu": 61.81, "tokens/trainable": 39419} +{"epoch": 0.15234375, "grad_norm": 5.387308597564697, "learning_rate": 9.984185462146642e-05, "loss": 0.7238801717758179, "memory/device_reserved (GiB)": 39.9, "memory/max_active (GiB)": 37.39, "memory/max_allocated (GiB)": 37.39, "ppl": 2.06242, "step": 39, "tokens/total": 1323616, "tokens/train_per_sec_per_gpu": 59.85, "tokens/trainable": 40468} +{"epoch": 0.15625, "grad_norm": 3.238691568374634, "learning_rate": 9.98166059662897e-05, "loss": 0.8040474653244019, "memory/device_reserved (GiB)": 39.91, "memory/max_active (GiB)": 35.86, "memory/max_allocated (GiB)": 35.86, "ppl": 2.23457, "step": 40, "tokens/total": 1354416, "tokens/train_per_sec_per_gpu": 72.76, "tokens/trainable": 41564} +{"epoch": 0.16015625, "grad_norm": 1.5518481731414795, "learning_rate": 9.978949230920472e-05, "loss": 0.7278357744216919, "memory/device_reserved (GiB)": 39.91, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 2.07059, "step": 41, "tokens/total": 1388768, "tokens/train_per_sec_per_gpu": 58.54, "tokens/trainable": 42578} +{"epoch": 0.1640625, "grad_norm": 1.7448501586914062, "learning_rate": 9.976051477852141e-05, "loss": 0.7599610090255737, "memory/device_reserved (GiB)": 39.91, "memory/max_active (GiB)": 37.91, "memory/max_allocated (GiB)": 37.91, "ppl": 2.13819, "step": 42, "tokens/total": 1422448, "tokens/train_per_sec_per_gpu": 62.85, "tokens/trainable": 43584} +{"epoch": 0.16796875, "grad_norm": 2.3701400756835938, "learning_rate": 9.972967458011312e-05, "loss": 0.9003289937973022, "memory/device_reserved (GiB)": 39.91, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 2.46041, "step": 43, "tokens/total": 1457872, "tokens/train_per_sec_per_gpu": 62.56, "tokens/trainable": 44691} +{"epoch": 0.171875, "grad_norm": 12.097038269042969, "learning_rate": 9.96969729973664e-05, "loss": 0.7613319158554077, "memory/device_reserved (GiB)": 41.24, "memory/max_active (GiB)": 38.93, "memory/max_allocated (GiB)": 38.93, "ppl": 2.14113, "step": 44, "tokens/total": 1496368, "tokens/train_per_sec_per_gpu": 56.88, "tokens/trainable": 45771} +{"epoch": 0.17578125, "grad_norm": 1.462948203086853, "learning_rate": 9.966241139112754e-05, "loss": 0.7273204326629639, "memory/device_reserved (GiB)": 41.24, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 2.06953, "step": 45, "tokens/total": 1530016, "tokens/train_per_sec_per_gpu": 62.54, "tokens/trainable": 46807} +{"epoch": 0.1796875, "grad_norm": 1.7511218786239624, "learning_rate": 9.96259911996461e-05, "loss": 0.6851748824119568, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 38.63, "memory/max_allocated (GiB)": 38.63, "ppl": 1.98412, "step": 46, "tokens/total": 1564096, "tokens/train_per_sec_per_gpu": 57.46, "tokens/trainable": 47842} +{"epoch": 0.18359375, "grad_norm": 2.9268295764923096, "learning_rate": 9.958771393851491e-05, "loss": 0.8946727514266968, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 36.71, "memory/max_allocated (GiB)": 36.71, "ppl": 2.44654, "step": 47, "tokens/total": 1595808, "tokens/train_per_sec_per_gpu": 60.87, "tokens/trainable": 48883} +{"epoch": 0.1875, "grad_norm": 1.7968167066574097, "learning_rate": 9.954758120060702e-05, "loss": 0.6830885410308838, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.97998, "step": 48, "tokens/total": 1632160, "tokens/train_per_sec_per_gpu": 52.67, "tokens/trainable": 49845} +{"epoch": 0.19140625, "grad_norm": 2.168919324874878, "learning_rate": 9.950559465600948e-05, "loss": 0.6269851326942444, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 37.61, "memory/max_allocated (GiB)": 37.61, "ppl": 1.87196, "step": 49, "tokens/total": 1666208, "tokens/train_per_sec_per_gpu": 61.4, "tokens/trainable": 50892} +{"epoch": 0.1953125, "grad_norm": 1.6316521167755127, "learning_rate": 9.946175605195379e-05, "loss": 0.586225152015686, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.79719, "step": 50, "tokens/total": 1706304, "tokens/train_per_sec_per_gpu": 49.73, "tokens/trainable": 52001} +{"epoch": 0.19921875, "grad_norm": 1.4696077108383179, "learning_rate": 9.941606721274322e-05, "loss": 0.6353449821472168, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.88767, "step": 51, "tokens/total": 1740656, "tokens/train_per_sec_per_gpu": 52.61, "tokens/trainable": 53018} +{"epoch": 0.203125, "grad_norm": 1.6726951599121094, "learning_rate": 9.936853003967685e-05, "loss": 0.7133659720420837, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 2.04085, "step": 52, "tokens/total": 1777808, "tokens/train_per_sec_per_gpu": 54.11, "tokens/trainable": 54035} +{"epoch": 0.20703125, "grad_norm": 7.888131618499756, "learning_rate": 9.93191465109705e-05, "loss": 0.7468881607055664, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 2.11042, "step": 53, "tokens/total": 1814480, "tokens/train_per_sec_per_gpu": 56.79, "tokens/trainable": 55070} +{"epoch": 0.2109375, "grad_norm": 1.818869948387146, "learning_rate": 9.926791868167438e-05, "loss": 0.6496429443359375, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.91486, "step": 54, "tokens/total": 1850928, "tokens/train_per_sec_per_gpu": 52.64, "tokens/trainable": 56041} +{"epoch": 0.21484375, "grad_norm": 1.97141432762146, "learning_rate": 9.921484868358753e-05, "loss": 0.6936650276184082, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 38.72, "memory/max_allocated (GiB)": 38.72, "ppl": 2.00104, "step": 55, "tokens/total": 1886752, "tokens/train_per_sec_per_gpu": 52.17, "tokens/trainable": 57063} +{"epoch": 0.21875, "grad_norm": 1.4580143690109253, "learning_rate": 9.915993872516924e-05, "loss": 0.655455470085144, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.92602, "step": 56, "tokens/total": 1922352, "tokens/train_per_sec_per_gpu": 53.76, "tokens/trainable": 58069} +{"epoch": 0.22265625, "grad_norm": 1.611547589302063, "learning_rate": 9.9103191091447e-05, "loss": 0.6283009648323059, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 36.88, "memory/max_allocated (GiB)": 36.88, "ppl": 1.87442, "step": 57, "tokens/total": 1954080, "tokens/train_per_sec_per_gpu": 58.95, "tokens/trainable": 59034} +{"epoch": 0.2265625, "grad_norm": 1.3453171253204346, "learning_rate": 9.904460814392147e-05, "loss": 0.5745842456817627, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.77639, "step": 58, "tokens/total": 1993856, "tokens/train_per_sec_per_gpu": 50.29, "tokens/trainable": 60073} +{"epoch": 0.23046875, "grad_norm": 1.4672473669052124, "learning_rate": 9.898419232046825e-05, "loss": 0.5883877873420715, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.80108, "step": 59, "tokens/total": 2030416, "tokens/train_per_sec_per_gpu": 58.16, "tokens/trainable": 61168} +{"epoch": 0.234375, "grad_norm": 5.3121256828308105, "learning_rate": 9.892194613523633e-05, "loss": 0.5624867677688599, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.75503, "step": 60, "tokens/total": 2066464, "tokens/train_per_sec_per_gpu": 58.27, "tokens/trainable": 62283} +{"epoch": 0.23828125, "grad_norm": 1.8137130737304688, "learning_rate": 9.885787217854357e-05, "loss": 0.5647634863853455, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 38.18, "memory/max_allocated (GiB)": 38.18, "ppl": 1.75903, "step": 61, "tokens/total": 2100640, "tokens/train_per_sec_per_gpu": 57.75, "tokens/trainable": 63286} +{"epoch": 0.2421875, "grad_norm": 2.3348960876464844, "learning_rate": 9.879197311676887e-05, "loss": 0.5719740390777588, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.77176, "step": 62, "tokens/total": 2136800, "tokens/train_per_sec_per_gpu": 57.05, "tokens/trainable": 64328} +{"epoch": 0.24609375, "grad_norm": 2.6112563610076904, "learning_rate": 9.872425169224113e-05, "loss": 0.5971585512161255, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 1.81695, "step": 63, "tokens/total": 2170496, "tokens/train_per_sec_per_gpu": 70.72, "tokens/trainable": 65506} +{"epoch": 0.25, "grad_norm": 2.83663010597229, "learning_rate": 9.865471072312528e-05, "loss": 0.5639463067054749, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.75759, "step": 64, "tokens/total": 2207040, "tokens/train_per_sec_per_gpu": 61.57, "tokens/trainable": 66588} +{"epoch": 0.25390625, "grad_norm": 1.5121716260910034, "learning_rate": 9.858335310330492e-05, "loss": 0.5263650417327881, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 37.09, "memory/max_allocated (GiB)": 37.09, "ppl": 1.69277, "step": 65, "tokens/total": 2239632, "tokens/train_per_sec_per_gpu": 64.04, "tokens/trainable": 67667} +{"epoch": 0.2578125, "grad_norm": 1.8035629987716675, "learning_rate": 9.851018180226185e-05, "loss": 0.5276858806610107, "memory/device_reserved (GiB)": 41.42, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.69501, "step": 66, "tokens/total": 2278304, "tokens/train_per_sec_per_gpu": 54.68, "tokens/trainable": 68737} +{"epoch": 0.26171875, "grad_norm": 1.4933557510375977, "learning_rate": 9.843519986495259e-05, "loss": 0.6176354885101318, "memory/device_reserved (GiB)": 41.42, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.85454, "step": 67, "tokens/total": 2313248, "tokens/train_per_sec_per_gpu": 59.81, "tokens/trainable": 69781} +{"epoch": 0.265625, "grad_norm": 2.124765634536743, "learning_rate": 9.835841041168162e-05, "loss": 0.4905179440975189, "memory/device_reserved (GiB)": 41.42, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.63316, "step": 68, "tokens/total": 2350656, "tokens/train_per_sec_per_gpu": 60.31, "tokens/trainable": 70877} +{"epoch": 0.26953125, "grad_norm": 1.2468518018722534, "learning_rate": 9.82798166379715e-05, "loss": 0.5168420076370239, "memory/device_reserved (GiB)": 41.42, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.67672, "step": 69, "tokens/total": 2383856, "tokens/train_per_sec_per_gpu": 61.72, "tokens/trainable": 71955} +{"epoch": 0.2734375, "grad_norm": 1.7584096193313599, "learning_rate": 9.819942181443002e-05, "loss": 0.6774911880493164, "memory/device_reserved (GiB)": 41.42, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.96893, "step": 70, "tokens/total": 2421120, "tokens/train_per_sec_per_gpu": 58.61, "tokens/trainable": 73029} +{"epoch": 0.27734375, "grad_norm": 1.3463850021362305, "learning_rate": 9.811722928661392e-05, "loss": 0.6046497821807861, "memory/device_reserved (GiB)": 41.43, "memory/max_active (GiB)": 35.8, "memory/max_allocated (GiB)": 35.8, "ppl": 1.83061, "step": 71, "tokens/total": 2449904, "tokens/train_per_sec_per_gpu": 72.8, "tokens/trainable": 74046} +{"epoch": 0.28125, "grad_norm": 1.4243199825286865, "learning_rate": 9.803324247488975e-05, "loss": 0.5070789456367493, "memory/device_reserved (GiB)": 41.43, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.66043, "step": 72, "tokens/total": 2485936, "tokens/train_per_sec_per_gpu": 58.13, "tokens/trainable": 75116} +{"epoch": 0.28515625, "grad_norm": 1.521089792251587, "learning_rate": 9.794746487429161e-05, "loss": 0.5020831823348999, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.04, "memory/max_allocated (GiB)": 37.04, "ppl": 1.65216, "step": 73, "tokens/total": 2519024, "tokens/train_per_sec_per_gpu": 68.28, "tokens/trainable": 76155} +{"epoch": 0.2890625, "grad_norm": 1.2891509532928467, "learning_rate": 9.785990005437554e-05, "loss": 0.5532117486000061, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.81, "memory/max_allocated (GiB)": 37.81, "ppl": 1.73883, "step": 74, "tokens/total": 2554864, "tokens/train_per_sec_per_gpu": 58.46, "tokens/trainable": 77208} +{"epoch": 0.29296875, "grad_norm": 5.2997331619262695, "learning_rate": 9.777055165907117e-05, "loss": 0.520816445350647, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.6834, "step": 75, "tokens/total": 2588048, "tokens/train_per_sec_per_gpu": 67.14, "tokens/trainable": 78249} +{"epoch": 0.296875, "grad_norm": 1.9633479118347168, "learning_rate": 9.767942340652993e-05, "loss": 0.5760632753372192, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.05, "memory/max_allocated (GiB)": 38.05, "ppl": 1.77902, "step": 76, "tokens/total": 2622784, "tokens/train_per_sec_per_gpu": 55.67, "tokens/trainable": 79268} +{"epoch": 0.30078125, "grad_norm": 1.7835946083068848, "learning_rate": 9.758651908897035e-05, "loss": 0.5740739107131958, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.77549, "step": 77, "tokens/total": 2656416, "tokens/train_per_sec_per_gpu": 62.66, "tokens/trainable": 80337} +{"epoch": 0.3046875, "grad_norm": 4.190371513366699, "learning_rate": 9.749184257252033e-05, "loss": 0.5707520246505737, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.7696, "step": 78, "tokens/total": 2690672, "tokens/train_per_sec_per_gpu": 62.28, "tokens/trainable": 81338} +{"epoch": 0.30859375, "grad_norm": 4.652210712432861, "learning_rate": 9.739539779705614e-05, "loss": 0.46647268533706665, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.59436, "step": 79, "tokens/total": 2729200, "tokens/train_per_sec_per_gpu": 58.96, "tokens/trainable": 82535} +{"epoch": 0.3125, "grad_norm": 1.742263674736023, "learning_rate": 9.729718877603861e-05, "loss": 0.5656114816665649, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.76052, "step": 80, "tokens/total": 2762784, "tokens/train_per_sec_per_gpu": 69.57, "tokens/trainable": 83593} +{"epoch": 0.31640625, "grad_norm": 1.3067893981933594, "learning_rate": 9.719721959634592e-05, "loss": 0.44713449478149414, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.56382, "step": 81, "tokens/total": 2798304, "tokens/train_per_sec_per_gpu": 61.83, "tokens/trainable": 84642} +{"epoch": 0.3203125, "grad_norm": 1.0901910066604614, "learning_rate": 9.709549441810375e-05, "loss": 0.49055564403533936, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.63322, "step": 82, "tokens/total": 2835712, "tokens/train_per_sec_per_gpu": 61.02, "tokens/trainable": 85722} +{"epoch": 0.32421875, "grad_norm": 7.477474689483643, "learning_rate": 9.699201747451195e-05, "loss": 0.5036226511001587, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.54, "memory/max_allocated (GiB)": 36.54, "ppl": 1.6547, "step": 83, "tokens/total": 2868224, "tokens/train_per_sec_per_gpu": 59.2, "tokens/trainable": 86775} +{"epoch": 0.328125, "grad_norm": 3.14548397064209, "learning_rate": 9.688679307166854e-05, "loss": 0.5555371642112732, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.74288, "step": 84, "tokens/total": 2900736, "tokens/train_per_sec_per_gpu": 64.12, "tokens/trainable": 87826} +{"epoch": 0.33203125, "grad_norm": 1.5769764184951782, "learning_rate": 9.677982558839042e-05, "loss": 0.5413460731506348, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.82, "memory/max_allocated (GiB)": 36.82, "ppl": 1.71832, "step": 85, "tokens/total": 2933904, "tokens/train_per_sec_per_gpu": 57.18, "tokens/trainable": 88881} +{"epoch": 0.3359375, "grad_norm": 1.388842225074768, "learning_rate": 9.66711194760312e-05, "loss": 0.4468875527381897, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 39.04, "memory/max_allocated (GiB)": 39.04, "ppl": 1.56344, "step": 86, "tokens/total": 2971344, "tokens/train_per_sec_per_gpu": 56.37, "tokens/trainable": 90010} +{"epoch": 0.33984375, "grad_norm": 1.6406259536743164, "learning_rate": 9.656067925829593e-05, "loss": 0.5077341794967651, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.66152, "step": 87, "tokens/total": 3001936, "tokens/train_per_sec_per_gpu": 72.01, "tokens/trainable": 91044} +{"epoch": 0.34375, "grad_norm": 1.2214441299438477, "learning_rate": 9.644850953105288e-05, "loss": 0.5361196994781494, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 35.88, "memory/max_allocated (GiB)": 35.88, "ppl": 1.70936, "step": 88, "tokens/total": 3032944, "tokens/train_per_sec_per_gpu": 68.82, "tokens/trainable": 92123} +{"epoch": 0.34765625, "grad_norm": 2.1271448135375977, "learning_rate": 9.633461496214225e-05, "loss": 0.4246278405189514, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.52902, "step": 89, "tokens/total": 3065584, "tokens/train_per_sec_per_gpu": 62.98, "tokens/trainable": 93180} +{"epoch": 0.3515625, "grad_norm": 2.884425640106201, "learning_rate": 9.621900029118195e-05, "loss": 0.5512129664421082, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.1, "memory/max_allocated (GiB)": 36.1, "ppl": 1.73536, "step": 90, "tokens/total": 3095248, "tokens/train_per_sec_per_gpu": 68.58, "tokens/trainable": 94212} +{"epoch": 0.35546875, "grad_norm": 1.4665881395339966, "learning_rate": 9.610167032937036e-05, "loss": 0.5137171745300293, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.67149, "step": 91, "tokens/total": 3133456, "tokens/train_per_sec_per_gpu": 54.95, "tokens/trainable": 95265} +{"epoch": 0.359375, "grad_norm": 1.3158966302871704, "learning_rate": 9.598262995928611e-05, "loss": 0.4385943114757538, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.55053, "step": 92, "tokens/total": 3164624, "tokens/train_per_sec_per_gpu": 69.65, "tokens/trainable": 96319} +{"epoch": 0.36328125, "grad_norm": 4.049941062927246, "learning_rate": 9.586188413468492e-05, "loss": 0.44766369462013245, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.8, "memory/max_allocated (GiB)": 38.8, "ppl": 1.56465, "step": 93, "tokens/total": 3203008, "tokens/train_per_sec_per_gpu": 50.03, "tokens/trainable": 97372} +{"epoch": 0.3671875, "grad_norm": 2.52805757522583, "learning_rate": 9.57394378802934e-05, "loss": 0.44062310457229614, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.55368, "step": 94, "tokens/total": 3236448, "tokens/train_per_sec_per_gpu": 61.37, "tokens/trainable": 98418} +{"epoch": 0.37109375, "grad_norm": 3.6240038871765137, "learning_rate": 9.56152962916e-05, "loss": 0.48660653829574585, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.62679, "step": 95, "tokens/total": 3267104, "tokens/train_per_sec_per_gpu": 67.02, "tokens/trainable": 99408} +{"epoch": 0.375, "grad_norm": 1.7175042629241943, "learning_rate": 9.548946453464296e-05, "loss": 0.39313361048698425, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 1.48162, "step": 96, "tokens/total": 3301440, "tokens/train_per_sec_per_gpu": 63.36, "tokens/trainable": 100513} +{"epoch": 0.37890625, "grad_norm": 1.9641765356063843, "learning_rate": 9.53619478457953e-05, "loss": 0.4849191904067993, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.62404, "step": 97, "tokens/total": 3336112, "tokens/train_per_sec_per_gpu": 53.12, "tokens/trainable": 101523} +{"epoch": 0.3828125, "grad_norm": 3.437682628631592, "learning_rate": 9.523275153154695e-05, "loss": 0.47194424271583557, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.60311, "step": 98, "tokens/total": 3368624, "tokens/train_per_sec_per_gpu": 58.49, "tokens/trainable": 102483} +{"epoch": 0.38671875, "grad_norm": 5.871929168701172, "learning_rate": 9.51018809682839e-05, "loss": 0.45128539204597473, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.57033, "step": 99, "tokens/total": 3406448, "tokens/train_per_sec_per_gpu": 53.52, "tokens/trainable": 103500} +{"epoch": 0.390625, "grad_norm": 1.9794408082962036, "learning_rate": 9.49693416020645e-05, "loss": 0.4925496578216553, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.51, "memory/max_allocated (GiB)": 38.51, "ppl": 1.63648, "step": 100, "tokens/total": 3444240, "tokens/train_per_sec_per_gpu": 55.45, "tokens/trainable": 104520} +{"epoch": 0.39453125, "grad_norm": 1.6902841329574585, "learning_rate": 9.483513894839276e-05, "loss": 0.5244827270507812, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.68958, "step": 101, "tokens/total": 3477680, "tokens/train_per_sec_per_gpu": 68.76, "tokens/trainable": 105623} +{"epoch": 0.3984375, "grad_norm": 1.6445518732070923, "learning_rate": 9.469927859198888e-05, "loss": 0.47593265771865845, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.60951, "step": 102, "tokens/total": 3513616, "tokens/train_per_sec_per_gpu": 59.23, "tokens/trainable": 106623} +{"epoch": 0.40234375, "grad_norm": 1.6454951763153076, "learning_rate": 9.456176618655689e-05, "loss": 0.4249943494796753, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.51, "memory/max_allocated (GiB)": 36.51, "ppl": 1.52958, "step": 103, "tokens/total": 3545584, "tokens/train_per_sec_per_gpu": 61.13, "tokens/trainable": 107619} +{"epoch": 0.40625, "grad_norm": 1.674741506576538, "learning_rate": 9.442260745454927e-05, "loss": 0.42165547609329224, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.52448, "step": 104, "tokens/total": 3580768, "tokens/train_per_sec_per_gpu": 59.82, "tokens/trainable": 108690} +{"epoch": 0.41015625, "grad_norm": 1.7223765850067139, "learning_rate": 9.428180818692884e-05, "loss": 0.3995684087276459, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.49118, "step": 105, "tokens/total": 3617008, "tokens/train_per_sec_per_gpu": 53.1, "tokens/trainable": 109750} +{"epoch": 0.4140625, "grad_norm": 2.5320539474487305, "learning_rate": 9.413937424292791e-05, "loss": 0.4765089154243469, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.61044, "step": 106, "tokens/total": 3655936, "tokens/train_per_sec_per_gpu": 50.52, "tokens/trainable": 110766} +{"epoch": 0.41796875, "grad_norm": 1.7501370906829834, "learning_rate": 9.399531154980424e-05, "loss": 0.3837505877017975, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.46778, "step": 107, "tokens/total": 3692736, "tokens/train_per_sec_per_gpu": 52.1, "tokens/trainable": 111771} +{"epoch": 0.421875, "grad_norm": 2.509521961212158, "learning_rate": 9.384962610259455e-05, "loss": 0.4272077679634094, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.53297, "step": 108, "tokens/total": 3727248, "tokens/train_per_sec_per_gpu": 55.38, "tokens/trainable": 112775} +{"epoch": 0.42578125, "grad_norm": 1.8470127582550049, "learning_rate": 9.370232396386494e-05, "loss": 0.4419804811477661, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.55579, "step": 109, "tokens/total": 3760464, "tokens/train_per_sec_per_gpu": 61.57, "tokens/trainable": 113795} +{"epoch": 0.4296875, "grad_norm": 1.5366462469100952, "learning_rate": 9.355341126345868e-05, "loss": 0.3919759690761566, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.4799, "step": 110, "tokens/total": 3795952, "tokens/train_per_sec_per_gpu": 61.11, "tokens/trainable": 114888} +{"epoch": 0.43359375, "grad_norm": 1.1714789867401123, "learning_rate": 9.340289419824107e-05, "loss": 0.36357295513153076, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.74, "memory/max_allocated (GiB)": 36.74, "ppl": 1.43846, "step": 111, "tokens/total": 3828144, "tokens/train_per_sec_per_gpu": 69.8, "tokens/trainable": 115919} +{"epoch": 0.4375, "grad_norm": 1.84748375415802, "learning_rate": 9.325077903184159e-05, "loss": 0.5144234895706177, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.67267, "step": 112, "tokens/total": 3862992, "tokens/train_per_sec_per_gpu": 60.11, "tokens/trainable": 117018} +{"epoch": 0.44140625, "grad_norm": 1.4233328104019165, "learning_rate": 9.30970720943932e-05, "loss": 0.3855287730693817, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.43, "memory/max_allocated (GiB)": 38.43, "ppl": 1.47039, "step": 113, "tokens/total": 3898416, "tokens/train_per_sec_per_gpu": 58.38, "tokens/trainable": 118100} +{"epoch": 0.4453125, "grad_norm": 1.2257837057113647, "learning_rate": 9.2941779782269e-05, "loss": 0.3914945125579834, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.47919, "step": 114, "tokens/total": 3932176, "tokens/train_per_sec_per_gpu": 61.76, "tokens/trainable": 119137} +{"epoch": 0.44921875, "grad_norm": 1.3070107698440552, "learning_rate": 9.278490855781596e-05, "loss": 0.4521649479866028, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.57171, "step": 115, "tokens/total": 3964384, "tokens/train_per_sec_per_gpu": 69.06, "tokens/trainable": 120153} +{"epoch": 0.453125, "grad_norm": 1.6181446313858032, "learning_rate": 9.262646494908604e-05, "loss": 0.5036980509757996, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 39.14, "memory/max_allocated (GiB)": 39.14, "ppl": 1.65483, "step": 116, "tokens/total": 4002960, "tokens/train_per_sec_per_gpu": 51.36, "tokens/trainable": 121152} +{"epoch": 0.45703125, "grad_norm": 1.545694351196289, "learning_rate": 9.246645554956457e-05, "loss": 0.4339969754219055, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.54341, "step": 117, "tokens/total": 4037552, "tokens/train_per_sec_per_gpu": 60.77, "tokens/trainable": 122186} +{"epoch": 0.4609375, "grad_norm": 1.5031355619430542, "learning_rate": 9.230488701789578e-05, "loss": 0.4898163378238678, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.7, "memory/max_allocated (GiB)": 38.7, "ppl": 1.63202, "step": 118, "tokens/total": 4075072, "tokens/train_per_sec_per_gpu": 57.48, "tokens/trainable": 123284} +{"epoch": 0.46484375, "grad_norm": 2.35062575340271, "learning_rate": 9.214176607760577e-05, "loss": 0.3753693103790283, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.45553, "step": 119, "tokens/total": 4113744, "tokens/train_per_sec_per_gpu": 59.04, "tokens/trainable": 124415} +{"epoch": 0.46875, "grad_norm": 1.0570003986358643, "learning_rate": 9.197709951682268e-05, "loss": 0.3885435461997986, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.47483, "step": 120, "tokens/total": 4146032, "tokens/train_per_sec_per_gpu": 65.67, "tokens/trainable": 125482} +{"epoch": 0.47265625, "grad_norm": 1.2655303478240967, "learning_rate": 9.181089418799428e-05, "loss": 0.39568665623664856, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 39.39, "memory/max_allocated (GiB)": 39.39, "ppl": 1.4854, "step": 121, "tokens/total": 4182832, "tokens/train_per_sec_per_gpu": 55.52, "tokens/trainable": 126521} +{"epoch": 0.4765625, "grad_norm": 0.9747000336647034, "learning_rate": 9.164315700760271e-05, "loss": 0.33482903242111206, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.3977, "step": 122, "tokens/total": 4218640, "tokens/train_per_sec_per_gpu": 58.96, "tokens/trainable": 127565} +{"epoch": 0.48046875, "grad_norm": 1.2415145635604858, "learning_rate": 9.147389495587671e-05, "loss": 0.478459894657135, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 1.61359, "step": 123, "tokens/total": 4249792, "tokens/train_per_sec_per_gpu": 69.71, "tokens/trainable": 128595} +{"epoch": 0.484375, "grad_norm": 1.146227240562439, "learning_rate": 9.130311507650116e-05, "loss": 0.3259563446044922, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 1.38535, "step": 124, "tokens/total": 4282464, "tokens/train_per_sec_per_gpu": 62.13, "tokens/trainable": 129650} +{"epoch": 0.48828125, "grad_norm": 1.4887449741363525, "learning_rate": 9.113082447632394e-05, "loss": 0.3875514268875122, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.47337, "step": 125, "tokens/total": 4314608, "tokens/train_per_sec_per_gpu": 80.44, "tokens/trainable": 130784} +{"epoch": 0.4921875, "grad_norm": 1.7315394878387451, "learning_rate": 9.09570303250602e-05, "loss": 0.39966881275177, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.47, "memory/max_allocated (GiB)": 36.47, "ppl": 1.49133, "step": 126, "tokens/total": 4346416, "tokens/train_per_sec_per_gpu": 63.86, "tokens/trainable": 131847} +{"epoch": 0.49609375, "grad_norm": 1.6407946348190308, "learning_rate": 9.078173985499394e-05, "loss": 0.38412922620773315, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 35.79, "memory/max_allocated (GiB)": 35.79, "ppl": 1.46834, "step": 127, "tokens/total": 4376400, "tokens/train_per_sec_per_gpu": 66.04, "tokens/trainable": 132851} +{"epoch": 0.5, "grad_norm": 1.3851579427719116, "learning_rate": 9.060496036067713e-05, "loss": 0.401643842458725, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.72, "memory/max_allocated (GiB)": 38.72, "ppl": 1.49428, "step": 128, "tokens/total": 4413824, "tokens/train_per_sec_per_gpu": 48.46, "tokens/trainable": 133865} +{"epoch": 0.50390625, "grad_norm": 1.246240258216858, "learning_rate": 9.042669919862615e-05, "loss": 0.4032745659351349, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.82, "memory/max_allocated (GiB)": 36.82, "ppl": 1.49672, "step": 129, "tokens/total": 4445936, "tokens/train_per_sec_per_gpu": 68.21, "tokens/trainable": 134926} +{"epoch": 0.5078125, "grad_norm": 1.3411071300506592, "learning_rate": 9.024696378701557e-05, "loss": 0.4234699010848999, "memory/device_reserved (GiB)": 41.42, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.52725, "step": 130, "tokens/total": 4481488, "tokens/train_per_sec_per_gpu": 59.02, "tokens/trainable": 135976} +{"epoch": 0.51171875, "grad_norm": 1.5286849737167358, "learning_rate": 9.006576160536948e-05, "loss": 0.3944430351257324, "memory/device_reserved (GiB)": 41.42, "memory/max_active (GiB)": 37.43, "memory/max_allocated (GiB)": 37.43, "ppl": 1.48356, "step": 131, "tokens/total": 4516048, "tokens/train_per_sec_per_gpu": 56.52, "tokens/trainable": 136970} +{"epoch": 0.515625, "grad_norm": 1.2105258703231812, "learning_rate": 8.988310019425035e-05, "loss": 0.35212910175323486, "memory/device_reserved (GiB)": 41.71, "memory/max_active (GiB)": 39.7, "memory/max_allocated (GiB)": 39.7, "ppl": 1.42209, "step": 132, "tokens/total": 4553904, "tokens/train_per_sec_per_gpu": 51.94, "tokens/trainable": 137980} +{"epoch": 0.51953125, "grad_norm": 1.1932579278945923, "learning_rate": 8.969898715494506e-05, "loss": 0.33988597989082336, "memory/device_reserved (GiB)": 41.73, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.40479, "step": 133, "tokens/total": 4590736, "tokens/train_per_sec_per_gpu": 61.52, "tokens/trainable": 139070} +{"epoch": 0.5234375, "grad_norm": 1.0735384225845337, "learning_rate": 8.951343014914869e-05, "loss": 0.3370867371559143, "memory/device_reserved (GiB)": 41.73, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.40086, "step": 134, "tokens/total": 4627392, "tokens/train_per_sec_per_gpu": 57.45, "tokens/trainable": 140160} +{"epoch": 0.52734375, "grad_norm": 1.1321266889572144, "learning_rate": 8.932643689864568e-05, "loss": 0.3455788493156433, "memory/device_reserved (GiB)": 41.73, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.41281, "step": 135, "tokens/total": 4663968, "tokens/train_per_sec_per_gpu": 62.54, "tokens/trainable": 141277} +{"epoch": 0.53125, "grad_norm": 1.1900067329406738, "learning_rate": 8.913801518498845e-05, "loss": 0.31726622581481934, "memory/device_reserved (GiB)": 41.73, "memory/max_active (GiB)": 39.68, "memory/max_allocated (GiB)": 39.68, "ppl": 1.37337, "step": 136, "tokens/total": 4703344, "tokens/train_per_sec_per_gpu": 48.91, "tokens/trainable": 142389} +{"epoch": 0.53515625, "grad_norm": 1.9985744953155518, "learning_rate": 8.894817284917364e-05, "loss": 0.3760351240634918, "memory/device_reserved (GiB)": 41.73, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.4565, "step": 137, "tokens/total": 4740288, "tokens/train_per_sec_per_gpu": 50.64, "tokens/trainable": 143423} +{"epoch": 0.5390625, "grad_norm": 1.9842361211776733, "learning_rate": 8.875691779131569e-05, "loss": 0.4755341410636902, "memory/device_reserved (GiB)": 41.9, "memory/max_active (GiB)": 39.78, "memory/max_allocated (GiB)": 39.78, "ppl": 1.60887, "step": 138, "tokens/total": 4778224, "tokens/train_per_sec_per_gpu": 55.61, "tokens/trainable": 144536} +{"epoch": 0.54296875, "grad_norm": 1.1984978914260864, "learning_rate": 8.856425797031829e-05, "loss": 0.3033708333969116, "memory/device_reserved (GiB)": 42.22, "memory/max_active (GiB)": 39.58, "memory/max_allocated (GiB)": 39.58, "ppl": 1.35442, "step": 139, "tokens/total": 4814256, "tokens/train_per_sec_per_gpu": 64.21, "tokens/trainable": 145664} +{"epoch": 0.546875, "grad_norm": 1.8171721696853638, "learning_rate": 8.837020140354295e-05, "loss": 0.34330683946609497, "memory/device_reserved (GiB)": 42.22, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.4096, "step": 140, "tokens/total": 4851072, "tokens/train_per_sec_per_gpu": 55.78, "tokens/trainable": 146671} +{"epoch": 0.55078125, "grad_norm": 1.6844803094863892, "learning_rate": 8.817475616647554e-05, "loss": 0.3742174804210663, "memory/device_reserved (GiB)": 42.22, "memory/max_active (GiB)": 38.17, "memory/max_allocated (GiB)": 38.17, "ppl": 1.45385, "step": 141, "tokens/total": 4885504, "tokens/train_per_sec_per_gpu": 60.57, "tokens/trainable": 147742} +{"epoch": 0.5546875, "grad_norm": 2.2449798583984375, "learning_rate": 8.797793039239017e-05, "loss": 0.4009706974029541, "memory/device_reserved (GiB)": 42.22, "memory/max_active (GiB)": 37.83, "memory/max_allocated (GiB)": 37.83, "ppl": 1.49327, "step": 142, "tokens/total": 4920528, "tokens/train_per_sec_per_gpu": 60.8, "tokens/trainable": 148809} +{"epoch": 0.55859375, "grad_norm": 1.5053051710128784, "learning_rate": 8.777973227201069e-05, "loss": 0.3319675028324127, "memory/device_reserved (GiB)": 43.76, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.39371, "step": 143, "tokens/total": 4957440, "tokens/train_per_sec_per_gpu": 60.88, "tokens/trainable": 149872} +{"epoch": 0.5625, "grad_norm": 1.4501397609710693, "learning_rate": 8.758017005316988e-05, "loss": 0.38777196407318115, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.47369, "step": 144, "tokens/total": 4990752, "tokens/train_per_sec_per_gpu": 73.26, "tokens/trainable": 150920} +{"epoch": 0.56640625, "grad_norm": 1.5994547605514526, "learning_rate": 8.737925204046629e-05, "loss": 0.4191422462463379, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.52066, "step": 145, "tokens/total": 5025472, "tokens/train_per_sec_per_gpu": 55.13, "tokens/trainable": 151981} +{"epoch": 0.5703125, "grad_norm": 1.288765788078308, "learning_rate": 8.717698659491851e-05, "loss": 0.3319285213947296, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 38.91, "memory/max_allocated (GiB)": 38.91, "ppl": 1.39365, "step": 146, "tokens/total": 5062304, "tokens/train_per_sec_per_gpu": 57.49, "tokens/trainable": 153085} +{"epoch": 0.57421875, "grad_norm": 1.2877590656280518, "learning_rate": 8.697338213361735e-05, "loss": 0.4003099799156189, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 36.45, "memory/max_allocated (GiB)": 36.45, "ppl": 1.49229, "step": 147, "tokens/total": 5093280, "tokens/train_per_sec_per_gpu": 69.73, "tokens/trainable": 154182} +{"epoch": 0.578125, "grad_norm": 1.0853387117385864, "learning_rate": 8.676844712937552e-05, "loss": 0.30702999234199524, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.35938, "step": 148, "tokens/total": 5133280, "tokens/train_per_sec_per_gpu": 49.57, "tokens/trainable": 155234} +{"epoch": 0.58203125, "grad_norm": 1.3199965953826904, "learning_rate": 8.656219011037509e-05, "loss": 0.34619492292404175, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.41368, "step": 149, "tokens/total": 5170576, "tokens/train_per_sec_per_gpu": 56.81, "tokens/trainable": 156324} +{"epoch": 0.5859375, "grad_norm": 1.2382680177688599, "learning_rate": 8.63546196598125e-05, "loss": 0.33785510063171387, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.40194, "step": 150, "tokens/total": 5208160, "tokens/train_per_sec_per_gpu": 56.27, "tokens/trainable": 157459} +{"epoch": 0.58984375, "grad_norm": 1.5366294384002686, "learning_rate": 8.614574441554145e-05, "loss": 0.4347296953201294, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 36.95, "memory/max_allocated (GiB)": 36.95, "ppl": 1.54455, "step": 151, "tokens/total": 5240784, "tokens/train_per_sec_per_gpu": 62.69, "tokens/trainable": 158423} +{"epoch": 0.59375, "grad_norm": 1.3821555376052856, "learning_rate": 8.593557306971349e-05, "loss": 0.3115575313568115, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.36555, "step": 152, "tokens/total": 5274096, "tokens/train_per_sec_per_gpu": 58.99, "tokens/trainable": 159456} +{"epoch": 0.59765625, "grad_norm": 1.5340237617492676, "learning_rate": 8.572411436841618e-05, "loss": 0.35523247718811035, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 36.9, "memory/max_allocated (GiB)": 36.9, "ppl": 1.42651, "step": 153, "tokens/total": 5307840, "tokens/train_per_sec_per_gpu": 64.8, "tokens/trainable": 160526} +{"epoch": 0.6015625, "grad_norm": 1.3367393016815186, "learning_rate": 8.551137711130922e-05, "loss": 0.32543671131134033, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.38464, "step": 154, "tokens/total": 5339616, "tokens/train_per_sec_per_gpu": 59.62, "tokens/trainable": 161544} +{"epoch": 0.60546875, "grad_norm": 1.5229424238204956, "learning_rate": 8.529737015125824e-05, "loss": 0.28204572200775146, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 38.17, "memory/max_allocated (GiB)": 38.17, "ppl": 1.32584, "step": 155, "tokens/total": 5374720, "tokens/train_per_sec_per_gpu": 57.21, "tokens/trainable": 162563} +{"epoch": 0.609375, "grad_norm": 1.7795718908309937, "learning_rate": 8.508210239396639e-05, "loss": 0.3543407917022705, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 39.77, "memory/max_allocated (GiB)": 39.77, "ppl": 1.42524, "step": 156, "tokens/total": 5412608, "tokens/train_per_sec_per_gpu": 57.52, "tokens/trainable": 163580} +{"epoch": 0.61328125, "grad_norm": 1.285691261291504, "learning_rate": 8.486558279760375e-05, "loss": 0.36369508504867554, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 35.9, "memory/max_allocated (GiB)": 35.9, "ppl": 1.43864, "step": 157, "tokens/total": 5443664, "tokens/train_per_sec_per_gpu": 64.59, "tokens/trainable": 164587} +{"epoch": 0.6171875, "grad_norm": 1.3387507200241089, "learning_rate": 8.464782037243449e-05, "loss": 0.33669528365135193, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.40031, "step": 158, "tokens/total": 5475808, "tokens/train_per_sec_per_gpu": 61.15, "tokens/trainable": 165576} +{"epoch": 0.62109375, "grad_norm": 2.5187828540802, "learning_rate": 8.442882418044202e-05, "loss": 0.3407125771045685, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.40595, "step": 159, "tokens/total": 5508448, "tokens/train_per_sec_per_gpu": 64.16, "tokens/trainable": 166616} +{"epoch": 0.625, "grad_norm": 1.682093620300293, "learning_rate": 8.420860333495179e-05, "loss": 0.29655539989471436, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.34522, "step": 160, "tokens/total": 5540976, "tokens/train_per_sec_per_gpu": 65.25, "tokens/trainable": 167604} +{"epoch": 0.62890625, "grad_norm": 1.7122360467910767, "learning_rate": 8.398716700025208e-05, "loss": 0.3526124358177185, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.42278, "step": 161, "tokens/total": 5574496, "tokens/train_per_sec_per_gpu": 61.32, "tokens/trainable": 168639} +{"epoch": 0.6328125, "grad_norm": 1.8248317241668701, "learning_rate": 8.376452439121266e-05, "loss": 0.46519768238067627, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.05, "memory/max_allocated (GiB)": 36.05, "ppl": 1.59233, "step": 162, "tokens/total": 5604544, "tokens/train_per_sec_per_gpu": 69.43, "tokens/trainable": 169639} +{"epoch": 0.63671875, "grad_norm": 1.8733288049697876, "learning_rate": 8.354068477290124e-05, "loss": 0.35156986117362976, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.4213, "step": 163, "tokens/total": 5638768, "tokens/train_per_sec_per_gpu": 61.73, "tokens/trainable": 170626} +{"epoch": 0.640625, "grad_norm": 2.5654232501983643, "learning_rate": 8.331565746019807e-05, "loss": 0.32540076971054077, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.38459, "step": 164, "tokens/total": 5670960, "tokens/train_per_sec_per_gpu": 69.5, "tokens/trainable": 171666} +{"epoch": 0.64453125, "grad_norm": 1.8892431259155273, "learning_rate": 8.308945181740812e-05, "loss": 0.37163883447647095, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.45011, "step": 165, "tokens/total": 5705664, "tokens/train_per_sec_per_gpu": 62.63, "tokens/trainable": 172682} +{"epoch": 0.6484375, "grad_norm": 1.0515177249908447, "learning_rate": 8.286207725787153e-05, "loss": 0.2763446569442749, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.3183, "step": 166, "tokens/total": 5742672, "tokens/train_per_sec_per_gpu": 52.26, "tokens/trainable": 173709} +{"epoch": 0.65234375, "grad_norm": 1.2350562810897827, "learning_rate": 8.263354324357182e-05, "loss": 0.2922457456588745, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.12, "memory/max_allocated (GiB)": 39.12, "ppl": 1.33943, "step": 167, "tokens/total": 5777408, "tokens/train_per_sec_per_gpu": 60.06, "tokens/trainable": 174741} +{"epoch": 0.65625, "grad_norm": 1.4433503150939941, "learning_rate": 8.240385928474219e-05, "loss": 0.3665125370025635, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.44269, "step": 168, "tokens/total": 5810224, "tokens/train_per_sec_per_gpu": 58.37, "tokens/trainable": 175701} +{"epoch": 0.66015625, "grad_norm": 1.6027277708053589, "learning_rate": 8.217303493946967e-05, "loss": 0.3474101424217224, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.4154, "step": 169, "tokens/total": 5844432, "tokens/train_per_sec_per_gpu": 58.36, "tokens/trainable": 176705} +{"epoch": 0.6640625, "grad_norm": 1.0356115102767944, "learning_rate": 8.194107981329746e-05, "loss": 0.29435694217681885, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.73, "memory/max_allocated (GiB)": 39.73, "ppl": 1.34226, "step": 170, "tokens/total": 5883008, "tokens/train_per_sec_per_gpu": 51.96, "tokens/trainable": 177726} +{"epoch": 0.66796875, "grad_norm": 1.247205138206482, "learning_rate": 8.170800355882518e-05, "loss": 0.28524473309516907, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.33009, "step": 171, "tokens/total": 5917440, "tokens/train_per_sec_per_gpu": 66.75, "tokens/trainable": 178765} +{"epoch": 0.671875, "grad_norm": 2.991145372390747, "learning_rate": 8.147381587530713e-05, "loss": 0.39888256788253784, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.39, "memory/max_allocated (GiB)": 37.39, "ppl": 1.49016, "step": 172, "tokens/total": 5951232, "tokens/train_per_sec_per_gpu": 62.89, "tokens/trainable": 179765} +{"epoch": 0.67578125, "grad_norm": 4.212338447570801, "learning_rate": 8.123852650824877e-05, "loss": 0.338423490524292, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.45, "memory/max_allocated (GiB)": 37.45, "ppl": 1.40273, "step": 173, "tokens/total": 5985056, "tokens/train_per_sec_per_gpu": 63.25, "tokens/trainable": 180836} +{"epoch": 0.6796875, "grad_norm": 1.851004719734192, "learning_rate": 8.100214524900103e-05, "loss": 0.32512712478637695, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.38421, "step": 174, "tokens/total": 6020128, "tokens/train_per_sec_per_gpu": 52.35, "tokens/trainable": 181830} +{"epoch": 0.68359375, "grad_norm": 1.7855373620986938, "learning_rate": 8.076468193435301e-05, "loss": 0.32378852367401123, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.38235, "step": 175, "tokens/total": 6050976, "tokens/train_per_sec_per_gpu": 67.06, "tokens/trainable": 182778} +{"epoch": 0.6875, "grad_norm": 1.595707654953003, "learning_rate": 8.052614644612253e-05, "loss": 0.3245198428630829, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.38337, "step": 176, "tokens/total": 6088816, "tokens/train_per_sec_per_gpu": 50.25, "tokens/trainable": 183828} +{"epoch": 0.69140625, "grad_norm": 1.314634919166565, "learning_rate": 8.028654871074489e-05, "loss": 0.26883798837661743, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.30844, "step": 177, "tokens/total": 6123408, "tokens/train_per_sec_per_gpu": 59.47, "tokens/trainable": 184914} +{"epoch": 0.6953125, "grad_norm": 1.761279582977295, "learning_rate": 8.004589869885986e-05, "loss": 0.4621337056159973, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.37, "memory/max_allocated (GiB)": 36.37, "ppl": 1.58746, "step": 178, "tokens/total": 6154880, "tokens/train_per_sec_per_gpu": 61.69, "tokens/trainable": 185891} +{"epoch": 0.69921875, "grad_norm": 9.95501708984375, "learning_rate": 7.980420642489674e-05, "loss": 0.2799918055534363, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.32312, "step": 179, "tokens/total": 6193584, "tokens/train_per_sec_per_gpu": 54.13, "tokens/trainable": 186923} +{"epoch": 0.703125, "grad_norm": 1.3601462841033936, "learning_rate": 7.95614819466576e-05, "loss": 0.28389087319374084, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 1.32829, "step": 180, "tokens/total": 6227296, "tokens/train_per_sec_per_gpu": 61.76, "tokens/trainable": 187995} +{"epoch": 0.70703125, "grad_norm": 1.3090753555297852, "learning_rate": 7.931773536489872e-05, "loss": 0.3175952434539795, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.37382, "step": 181, "tokens/total": 6261888, "tokens/train_per_sec_per_gpu": 63.09, "tokens/trainable": 189051} +{"epoch": 0.7109375, "grad_norm": 1.8564364910125732, "learning_rate": 7.907297682291035e-05, "loss": 0.34699535369873047, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.41481, "step": 182, "tokens/total": 6296544, "tokens/train_per_sec_per_gpu": 61.23, "tokens/trainable": 190134} +{"epoch": 0.71484375, "grad_norm": 1.4653737545013428, "learning_rate": 7.882721650609442e-05, "loss": 0.23764538764953613, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.65, "memory/max_allocated (GiB)": 39.65, "ppl": 1.26826, "step": 183, "tokens/total": 6336960, "tokens/train_per_sec_per_gpu": 53.84, "tokens/trainable": 191312} +{"epoch": 0.71875, "grad_norm": 6.990385055541992, "learning_rate": 7.85804646415409e-05, "loss": 0.2812098264694214, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.32473, "step": 184, "tokens/total": 6372192, "tokens/train_per_sec_per_gpu": 64.72, "tokens/trainable": 192398} +{"epoch": 0.72265625, "grad_norm": 1.2314468622207642, "learning_rate": 7.833273149760207e-05, "loss": 0.36270296573638916, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.43721, "step": 185, "tokens/total": 6403056, "tokens/train_per_sec_per_gpu": 62.32, "tokens/trainable": 193415} +{"epoch": 0.7265625, "grad_norm": 1.357703685760498, "learning_rate": 7.808402738346527e-05, "loss": 0.265403687953949, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.11, "memory/max_allocated (GiB)": 39.11, "ppl": 1.30396, "step": 186, "tokens/total": 6441840, "tokens/train_per_sec_per_gpu": 58.66, "tokens/trainable": 194556} +{"epoch": 0.73046875, "grad_norm": 1.114370584487915, "learning_rate": 7.783436264872382e-05, "loss": 0.26923197507858276, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.30896, "step": 187, "tokens/total": 6478688, "tokens/train_per_sec_per_gpu": 56.6, "tokens/trainable": 195605} +{"epoch": 0.734375, "grad_norm": 1.0523343086242676, "learning_rate": 7.758374768294647e-05, "loss": 0.2591906785964966, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.16, "memory/max_allocated (GiB)": 38.16, "ppl": 1.29588, "step": 188, "tokens/total": 6514624, "tokens/train_per_sec_per_gpu": 61.33, "tokens/trainable": 196700} +{"epoch": 0.73828125, "grad_norm": 2.03096866607666, "learning_rate": 7.733219291524489e-05, "loss": 0.2954064607620239, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.53, "memory/max_allocated (GiB)": 37.53, "ppl": 1.34367, "step": 189, "tokens/total": 6548048, "tokens/train_per_sec_per_gpu": 64.25, "tokens/trainable": 197748} +{"epoch": 0.7421875, "grad_norm": 1.18760085105896, "learning_rate": 7.707970881383977e-05, "loss": 0.35104385018348694, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.45, "memory/max_allocated (GiB)": 36.45, "ppl": 1.42055, "step": 190, "tokens/total": 6579760, "tokens/train_per_sec_per_gpu": 69.88, "tokens/trainable": 198783} +{"epoch": 0.74609375, "grad_norm": 1.1321176290512085, "learning_rate": 7.682630588562518e-05, "loss": 0.276996374130249, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.7, "memory/max_allocated (GiB)": 38.7, "ppl": 1.31916, "step": 191, "tokens/total": 6614800, "tokens/train_per_sec_per_gpu": 58.08, "tokens/trainable": 199844} +{"epoch": 0.75, "grad_norm": 4.395772933959961, "learning_rate": 7.657199467573129e-05, "loss": 0.32607603073120117, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.38552, "step": 192, "tokens/total": 6648928, "tokens/train_per_sec_per_gpu": 57.29, "tokens/trainable": 200865} +{"epoch": 0.75390625, "grad_norm": 1.4751651287078857, "learning_rate": 7.631678576708561e-05, "loss": 0.26612383127212524, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.16, "memory/max_allocated (GiB)": 38.16, "ppl": 1.3049, "step": 193, "tokens/total": 6683328, "tokens/train_per_sec_per_gpu": 67.89, "tokens/trainable": 201959} +{"epoch": 0.7578125, "grad_norm": 1.430898666381836, "learning_rate": 7.606068977997255e-05, "loss": 0.2948388457298279, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.64, "memory/max_allocated (GiB)": 39.64, "ppl": 1.34291, "step": 194, "tokens/total": 6720928, "tokens/train_per_sec_per_gpu": 66.7, "tokens/trainable": 203141} +{"epoch": 0.76171875, "grad_norm": 1.799768090248108, "learning_rate": 7.580371737159148e-05, "loss": 0.32477277517318726, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.38372, "step": 195, "tokens/total": 6757024, "tokens/train_per_sec_per_gpu": 57.35, "tokens/trainable": 204189} +{"epoch": 0.765625, "grad_norm": 1.4515774250030518, "learning_rate": 7.554587923561324e-05, "loss": 0.25988343358039856, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.99, "memory/max_allocated (GiB)": 38.99, "ppl": 1.29678, "step": 196, "tokens/total": 6791728, "tokens/train_per_sec_per_gpu": 52.17, "tokens/trainable": 205161} +{"epoch": 0.76953125, "grad_norm": 1.2266534566879272, "learning_rate": 7.528718610173511e-05, "loss": 0.25267863273620605, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.28747, "step": 197, "tokens/total": 6827072, "tokens/train_per_sec_per_gpu": 63.71, "tokens/trainable": 206255} +{"epoch": 0.7734375, "grad_norm": 1.4841289520263672, "learning_rate": 7.502764873523431e-05, "loss": 0.3412843644618988, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.40675, "step": 198, "tokens/total": 6861120, "tokens/train_per_sec_per_gpu": 61.94, "tokens/trainable": 207282} +{"epoch": 0.77734375, "grad_norm": 1.597631573677063, "learning_rate": 7.476727793652011e-05, "loss": 0.28165125846862793, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.72, "memory/max_allocated (GiB)": 36.72, "ppl": 1.32532, "step": 199, "tokens/total": 6894352, "tokens/train_per_sec_per_gpu": 63.16, "tokens/trainable": 208339} +{"epoch": 0.78125, "grad_norm": 1.150637149810791, "learning_rate": 7.450608454068415e-05, "loss": 0.27212411165237427, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.31275, "step": 200, "tokens/total": 6928848, "tokens/train_per_sec_per_gpu": 58.33, "tokens/trainable": 209405} +{"epoch": 0.78515625, "grad_norm": 1.1764569282531738, "learning_rate": 7.424407941704987e-05, "loss": 0.23422479629516602, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.49, "memory/max_allocated (GiB)": 38.49, "ppl": 1.26393, "step": 201, "tokens/total": 6965504, "tokens/train_per_sec_per_gpu": 67.78, "tokens/trainable": 210550} +{"epoch": 0.7890625, "grad_norm": 1.064673662185669, "learning_rate": 7.398127346871986e-05, "loss": 0.2354242354631424, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.61, "memory/max_allocated (GiB)": 39.61, "ppl": 1.26545, "step": 202, "tokens/total": 7002448, "tokens/train_per_sec_per_gpu": 55.45, "tokens/trainable": 211593} +{"epoch": 0.79296875, "grad_norm": 1.3544892072677612, "learning_rate": 7.371767763212238e-05, "loss": 0.25031933188438416, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.52, "memory/max_allocated (GiB)": 39.52, "ppl": 1.28444, "step": 203, "tokens/total": 7037824, "tokens/train_per_sec_per_gpu": 62.96, "tokens/trainable": 212644} +{"epoch": 0.796875, "grad_norm": 1.54422128200531, "learning_rate": 7.345330287655617e-05, "loss": 0.3200412094593048, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.09, "memory/max_allocated (GiB)": 39.09, "ppl": 1.37718, "step": 204, "tokens/total": 7075920, "tokens/train_per_sec_per_gpu": 49.1, "tokens/trainable": 213706} +{"epoch": 0.80078125, "grad_norm": 1.2491717338562012, "learning_rate": 7.31881602037339e-05, "loss": 0.28912630677223206, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.69, "memory/max_allocated (GiB)": 36.69, "ppl": 1.33526, "step": 205, "tokens/total": 7107888, "tokens/train_per_sec_per_gpu": 63.14, "tokens/trainable": 214717} +{"epoch": 0.8046875, "grad_norm": 1.093361735343933, "learning_rate": 7.29222606473245e-05, "loss": 0.2561419606208801, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.29194, "step": 206, "tokens/total": 7140720, "tokens/train_per_sec_per_gpu": 61.91, "tokens/trainable": 215735} +{"epoch": 0.80859375, "grad_norm": 1.7976679801940918, "learning_rate": 7.265561527249383e-05, "loss": 0.38324689865112305, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.46704, "step": 207, "tokens/total": 7174720, "tokens/train_per_sec_per_gpu": 62.97, "tokens/trainable": 216760} +{"epoch": 0.8125, "grad_norm": 1.106385588645935, "learning_rate": 7.238823517544436e-05, "loss": 0.20952893793582916, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.2331, "step": 208, "tokens/total": 7211872, "tokens/train_per_sec_per_gpu": 69.1, "tokens/trainable": 217940} +{"epoch": 0.81640625, "grad_norm": 1.278946042060852, "learning_rate": 7.212013148295333e-05, "loss": 0.23951506614685059, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.27063, "step": 209, "tokens/total": 7245856, "tokens/train_per_sec_per_gpu": 63.73, "tokens/trainable": 219019} +{"epoch": 0.8203125, "grad_norm": 1.654954433441162, "learning_rate": 7.185131535190975e-05, "loss": 0.27108168601989746, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.06, "memory/max_allocated (GiB)": 37.06, "ppl": 1.31138, "step": 210, "tokens/total": 7278448, "tokens/train_per_sec_per_gpu": 64.04, "tokens/trainable": 220090} +{"epoch": 0.82421875, "grad_norm": 1.0438518524169922, "learning_rate": 7.158179796885005e-05, "loss": 0.23021633923053741, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.25887, "step": 211, "tokens/total": 7313072, "tokens/train_per_sec_per_gpu": 62.6, "tokens/trainable": 221143} +{"epoch": 0.828125, "grad_norm": 3.3410861492156982, "learning_rate": 7.131159054949273e-05, "loss": 0.26854902505874634, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.30807, "step": 212, "tokens/total": 7351456, "tokens/train_per_sec_per_gpu": 55.29, "tokens/trainable": 222182} +{"epoch": 0.83203125, "grad_norm": 1.3757938146591187, "learning_rate": 7.104070433827139e-05, "loss": 0.27904704213142395, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.32187, "step": 213, "tokens/total": 7386880, "tokens/train_per_sec_per_gpu": 51.49, "tokens/trainable": 223124} +{"epoch": 0.8359375, "grad_norm": 1.669818639755249, "learning_rate": 7.076915060786705e-05, "loss": 0.2583768367767334, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.29483, "step": 214, "tokens/total": 7419696, "tokens/train_per_sec_per_gpu": 63.47, "tokens/trainable": 224173} +{"epoch": 0.83984375, "grad_norm": 1.8832412958145142, "learning_rate": 7.049694065873882e-05, "loss": 0.2771559953689575, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.07, "memory/max_allocated (GiB)": 38.07, "ppl": 1.31937, "step": 215, "tokens/total": 7454352, "tokens/train_per_sec_per_gpu": 56.52, "tokens/trainable": 225219} +{"epoch": 0.84375, "grad_norm": 1.6625820398330688, "learning_rate": 7.022408581865382e-05, "loss": 0.2306130826473236, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.25937, "step": 216, "tokens/total": 7488496, "tokens/train_per_sec_per_gpu": 58.34, "tokens/trainable": 226296} +{"epoch": 0.84765625, "grad_norm": 1.2796686887741089, "learning_rate": 6.99505974422157e-05, "loss": 0.2282908856868744, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.25645, "step": 217, "tokens/total": 7522976, "tokens/train_per_sec_per_gpu": 66.14, "tokens/trainable": 227382} +{"epoch": 0.8515625, "grad_norm": 2.11391282081604, "learning_rate": 6.967648691039213e-05, "loss": 0.2941073775291443, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.58, "memory/max_allocated (GiB)": 36.58, "ppl": 1.34193, "step": 218, "tokens/total": 7554848, "tokens/train_per_sec_per_gpu": 66.41, "tokens/trainable": 228413} +{"epoch": 0.85546875, "grad_norm": 1.3319333791732788, "learning_rate": 6.940176563004123e-05, "loss": 0.2664032578468323, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.30526, "step": 219, "tokens/total": 7588896, "tokens/train_per_sec_per_gpu": 57.97, "tokens/trainable": 229386} +{"epoch": 0.859375, "grad_norm": 1.2455979585647583, "learning_rate": 6.912644503343682e-05, "loss": 0.25827687978744507, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.2947, "step": 220, "tokens/total": 7627056, "tokens/train_per_sec_per_gpu": 56.02, "tokens/trainable": 230395} +{"epoch": 0.86328125, "grad_norm": 1.8023360967636108, "learning_rate": 6.885053657779273e-05, "loss": 0.2904682755470276, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.28, "memory/max_allocated (GiB)": 36.28, "ppl": 1.33705, "step": 221, "tokens/total": 7658368, "tokens/train_per_sec_per_gpu": 60.8, "tokens/trainable": 231413} +{"epoch": 0.8671875, "grad_norm": 1.5393668413162231, "learning_rate": 6.857405174478604e-05, "loss": 0.26473432779312134, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.30308, "step": 222, "tokens/total": 7693072, "tokens/train_per_sec_per_gpu": 58.35, "tokens/trainable": 232423} +{"epoch": 0.87109375, "grad_norm": 1.3869258165359497, "learning_rate": 6.82970020400792e-05, "loss": 0.26360321044921875, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.30161, "step": 223, "tokens/total": 7730288, "tokens/train_per_sec_per_gpu": 53.38, "tokens/trainable": 233444} +{"epoch": 0.875, "grad_norm": 0.947932243347168, "learning_rate": 6.801939899284132e-05, "loss": 0.1973264366388321, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.21814, "step": 224, "tokens/total": 7766304, "tokens/train_per_sec_per_gpu": 58.97, "tokens/trainable": 234523} +{"epoch": 0.87890625, "grad_norm": 0.861412763595581, "learning_rate": 6.774125415526827e-05, "loss": 0.19759942591190338, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.21847, "step": 225, "tokens/total": 7799680, "tokens/train_per_sec_per_gpu": 71.3, "tokens/trainable": 235661} +{"epoch": 0.8828125, "grad_norm": 1.16831636428833, "learning_rate": 6.746257910210214e-05, "loss": 0.24674275517463684, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.27985, "step": 226, "tokens/total": 7832912, "tokens/train_per_sec_per_gpu": 60.7, "tokens/trainable": 236679} +{"epoch": 0.88671875, "grad_norm": 1.73516845703125, "learning_rate": 6.718338543014937e-05, "loss": 0.35602694749832153, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 35.66, "memory/max_allocated (GiB)": 35.66, "ppl": 1.42765, "step": 227, "tokens/total": 7862912, "tokens/train_per_sec_per_gpu": 67.34, "tokens/trainable": 237683} +{"epoch": 0.890625, "grad_norm": 1.1666085720062256, "learning_rate": 6.69036847577983e-05, "loss": 0.24298399686813354, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.27505, "step": 228, "tokens/total": 7896816, "tokens/train_per_sec_per_gpu": 61.68, "tokens/trainable": 238712} +{"epoch": 0.89453125, "grad_norm": 1.091222882270813, "learning_rate": 6.662348872453553e-05, "loss": 0.18857960402965546, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.46, "memory/max_allocated (GiB)": 38.46, "ppl": 1.20753, "step": 229, "tokens/total": 7933376, "tokens/train_per_sec_per_gpu": 61.4, "tokens/trainable": 239780} +{"epoch": 0.8984375, "grad_norm": 1.7898460626602173, "learning_rate": 6.63428089904618e-05, "loss": 0.23004333674907684, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.25865, "step": 230, "tokens/total": 7972544, "tokens/train_per_sec_per_gpu": 52.57, "tokens/trainable": 240808} +{"epoch": 0.90234375, "grad_norm": 1.1483128070831299, "learning_rate": 6.60616572358065e-05, "loss": 0.2536754310131073, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.28875, "step": 231, "tokens/total": 8007008, "tokens/train_per_sec_per_gpu": 61.51, "tokens/trainable": 241780} +{"epoch": 0.90625, "grad_norm": 1.1686822175979614, "learning_rate": 6.578004516044172e-05, "loss": 0.2462662309408188, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.27924, "step": 232, "tokens/total": 8042864, "tokens/train_per_sec_per_gpu": 63.26, "tokens/trainable": 242824} +{"epoch": 0.91015625, "grad_norm": 0.8266012668609619, "learning_rate": 6.549798448339548e-05, "loss": 0.20763611793518066, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.6, "memory/max_allocated (GiB)": 38.6, "ppl": 1.23077, "step": 233, "tokens/total": 8080272, "tokens/train_per_sec_per_gpu": 60.79, "tokens/trainable": 243880} +{"epoch": 0.9140625, "grad_norm": 1.4926800727844238, "learning_rate": 6.521548694236384e-05, "loss": 0.24832478165626526, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.28188, "step": 234, "tokens/total": 8117584, "tokens/train_per_sec_per_gpu": 54.4, "tokens/trainable": 244924} +{"epoch": 0.91796875, "grad_norm": 1.0166630744934082, "learning_rate": 6.493256429322259e-05, "loss": 0.2289278507232666, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.25725, "step": 235, "tokens/total": 8153712, "tokens/train_per_sec_per_gpu": 55.96, "tokens/trainable": 245951} +{"epoch": 0.921875, "grad_norm": 1.5860321521759033, "learning_rate": 6.464922830953799e-05, "loss": 0.23785969614982605, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.26853, "step": 236, "tokens/total": 8192928, "tokens/train_per_sec_per_gpu": 52.84, "tokens/trainable": 246987} +{"epoch": 0.92578125, "grad_norm": 2.0251102447509766, "learning_rate": 6.436549078207688e-05, "loss": 0.23830081522464752, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.21, "memory/max_allocated (GiB)": 38.21, "ppl": 1.26909, "step": 237, "tokens/total": 8226016, "tokens/train_per_sec_per_gpu": 62.76, "tokens/trainable": 247987} +{"epoch": 0.9296875, "grad_norm": 1.4120734930038452, "learning_rate": 6.408136351831592e-05, "loss": 0.23984341323375702, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 37.15, "memory/max_allocated (GiB)": 37.15, "ppl": 1.27105, "step": 238, "tokens/total": 8258384, "tokens/train_per_sec_per_gpu": 60.67, "tokens/trainable": 248984} +{"epoch": 0.93359375, "grad_norm": 0.9290140867233276, "learning_rate": 6.379685834195036e-05, "loss": 0.2155734896659851, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.24057, "step": 239, "tokens/total": 8297888, "tokens/train_per_sec_per_gpu": 54.81, "tokens/trainable": 250136} +{"epoch": 0.9375, "grad_norm": 1.153436541557312, "learning_rate": 6.351198709240186e-05, "loss": 0.24659816920757294, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.61, "memory/max_allocated (GiB)": 39.61, "ppl": 1.27966, "step": 240, "tokens/total": 8338016, "tokens/train_per_sec_per_gpu": 54.42, "tokens/trainable": 251225} +{"epoch": 0.94140625, "grad_norm": 1.9250221252441406, "learning_rate": 6.32267616243259e-05, "loss": 0.23702019453048706, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.26747, "step": 241, "tokens/total": 8370608, "tokens/train_per_sec_per_gpu": 64.13, "tokens/trainable": 252255} +{"epoch": 0.9453125, "grad_norm": 2.9431228637695312, "learning_rate": 6.294119380711849e-05, "loss": 0.31967899203300476, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.37669, "step": 242, "tokens/total": 8405040, "tokens/train_per_sec_per_gpu": 53.02, "tokens/trainable": 253213} +{"epoch": 0.94921875, "grad_norm": 1.2513047456741333, "learning_rate": 6.265529552442209e-05, "loss": 0.24266870319843292, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.27465, "step": 243, "tokens/total": 8440400, "tokens/train_per_sec_per_gpu": 63.54, "tokens/trainable": 254308} +{"epoch": 0.953125, "grad_norm": 1.409787654876709, "learning_rate": 6.236907867363127e-05, "loss": 0.21348601579666138, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.14, "memory/max_allocated (GiB)": 38.14, "ppl": 1.23799, "step": 244, "tokens/total": 8474736, "tokens/train_per_sec_per_gpu": 63.89, "tokens/trainable": 255342} +{"epoch": 0.95703125, "grad_norm": 1.0970487594604492, "learning_rate": 6.208255516539749e-05, "loss": 0.20042641460895538, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 36.34, "memory/max_allocated (GiB)": 36.34, "ppl": 1.22192, "step": 245, "tokens/total": 8506944, "tokens/train_per_sec_per_gpu": 61.0, "tokens/trainable": 256399} +{"epoch": 0.9609375, "grad_norm": 1.3327724933624268, "learning_rate": 6.179573692313344e-05, "loss": 0.24935570359230042, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 1.2832, "step": 246, "tokens/total": 8544064, "tokens/train_per_sec_per_gpu": 58.24, "tokens/trainable": 257415} +{"epoch": 0.96484375, "grad_norm": 1.31477689743042, "learning_rate": 6.150863588251694e-05, "loss": 0.2416670322418213, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.27337, "step": 247, "tokens/total": 8584336, "tokens/train_per_sec_per_gpu": 50.58, "tokens/trainable": 258472} +{"epoch": 0.96875, "grad_norm": 1.4242714643478394, "learning_rate": 6.122126399099419e-05, "loss": 0.20754346251487732, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.78, "memory/max_allocated (GiB)": 39.78, "ppl": 1.23065, "step": 248, "tokens/total": 8624304, "tokens/train_per_sec_per_gpu": 66.96, "tokens/trainable": 259728} +{"epoch": 0.97265625, "grad_norm": 1.1016671657562256, "learning_rate": 6.0933633207282615e-05, "loss": 0.28822535276412964, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.33406, "step": 249, "tokens/total": 8660672, "tokens/train_per_sec_per_gpu": 55.31, "tokens/trainable": 260799} +{"epoch": 0.9765625, "grad_norm": 1.2958688735961914, "learning_rate": 6.064575550087316e-05, "loss": 0.22432705760002136, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.63, "memory/max_allocated (GiB)": 39.63, "ppl": 1.25148, "step": 250, "tokens/total": 8697680, "tokens/train_per_sec_per_gpu": 57.72, "tokens/trainable": 261840} +{"epoch": 0.98046875, "grad_norm": 1.239693284034729, "learning_rate": 6.0357642851532245e-05, "loss": 0.22501444816589355, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.25234, "step": 251, "tokens/total": 8734112, "tokens/train_per_sec_per_gpu": 55.35, "tokens/trainable": 262908} +{"epoch": 0.984375, "grad_norm": 1.2450934648513794, "learning_rate": 6.0069307248803294e-05, "loss": 0.2536315321922302, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.2887, "step": 252, "tokens/total": 8769440, "tokens/train_per_sec_per_gpu": 50.79, "tokens/trainable": 263887} +{"epoch": 0.98828125, "grad_norm": 1.0225869417190552, "learning_rate": 5.9780760691507635e-05, "loss": 0.24003560841083527, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 36.25, "memory/max_allocated (GiB)": 36.25, "ppl": 1.27129, "step": 253, "tokens/total": 8800688, "tokens/train_per_sec_per_gpu": 61.33, "tokens/trainable": 264825} +{"epoch": 0.9921875, "grad_norm": 1.1543309688568115, "learning_rate": 5.9492015187245334e-05, "loss": 0.22021354734897614, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.72, "memory/max_allocated (GiB)": 38.72, "ppl": 1.24634, "step": 254, "tokens/total": 8838272, "tokens/train_per_sec_per_gpu": 55.14, "tokens/trainable": 265790} +{"epoch": 0.99609375, "grad_norm": 1.0694139003753662, "learning_rate": 5.920308275189541e-05, "loss": 0.2365081012248993, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 36.6, "memory/max_allocated (GiB)": 36.6, "ppl": 1.26682, "step": 255, "tokens/total": 8869952, "tokens/train_per_sec_per_gpu": 62.92, "tokens/trainable": 266788} +{"epoch": 1.0, "grad_norm": 1.077345609664917, "learning_rate": 5.8913975409115874e-05, "loss": 0.2043936550617218, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.22678, "step": 256, "tokens/total": 8903312, "tokens/train_per_sec_per_gpu": 62.77, "tokens/trainable": 267847} +{"epoch": 1.00390625, "grad_norm": 0.852863073348999, "learning_rate": 5.8624705189843395e-05, "loss": 0.17724141478538513, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.19392, "step": 257, "tokens/total": 8940768, "tokens/train_per_sec_per_gpu": 56.01, "tokens/trainable": 268954} +{"epoch": 1.0078125, "grad_norm": 1.2488055229187012, "learning_rate": 5.833528413179249e-05, "loss": 0.20615877211093903, "memory/device_reserved (GiB)": 40.67, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.22895, "step": 258, "tokens/total": 8977392, "tokens/train_per_sec_per_gpu": 59.03, "tokens/trainable": 270053} +{"epoch": 1.01171875, "grad_norm": 0.6837707161903381, "learning_rate": 5.80457242789548e-05, "loss": 0.16736403107643127, "memory/device_reserved (GiB)": 42.74, "memory/max_active (GiB)": 39.65, "memory/max_allocated (GiB)": 39.65, "ppl": 1.18218, "step": 259, "tokens/total": 9017568, "tokens/train_per_sec_per_gpu": 57.63, "tokens/trainable": 271229} +{"epoch": 1.015625, "grad_norm": 0.7532515525817871, "learning_rate": 5.77560376810977e-05, "loss": 0.17384974658489227, "memory/device_reserved (GiB)": 42.75, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.18988, "step": 260, "tokens/total": 9050928, "tokens/train_per_sec_per_gpu": 70.06, "tokens/trainable": 272345} +{"epoch": 1.01953125, "grad_norm": 1.6319717168807983, "learning_rate": 5.7466236393263005e-05, "loss": 0.20306524634361267, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.22515, "step": 261, "tokens/total": 9085552, "tokens/train_per_sec_per_gpu": 61.61, "tokens/trainable": 273368} +{"epoch": 1.0234375, "grad_norm": 2.0830764770507812, "learning_rate": 5.717633247526522e-05, "loss": 0.21986155211925507, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 36.74, "memory/max_allocated (GiB)": 36.74, "ppl": 1.2459, "step": 262, "tokens/total": 9116240, "tokens/train_per_sec_per_gpu": 59.56, "tokens/trainable": 274319} +{"epoch": 1.02734375, "grad_norm": 1.6008607149124146, "learning_rate": 5.688633799118971e-05, "loss": 0.23148532211780548, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.26047, "step": 263, "tokens/total": 9148688, "tokens/train_per_sec_per_gpu": 55.52, "tokens/trainable": 275254} +{"epoch": 1.03125, "grad_norm": 1.3304356336593628, "learning_rate": 5.659626500889066e-05, "loss": 0.1671183705329895, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.09, "memory/max_allocated (GiB)": 37.09, "ppl": 1.18189, "step": 264, "tokens/total": 9182704, "tokens/train_per_sec_per_gpu": 65.1, "tokens/trainable": 276380} +{"epoch": 1.03515625, "grad_norm": 1.4949977397918701, "learning_rate": 5.6306125599488905e-05, "loss": 0.1963346302509308, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 36.88, "memory/max_allocated (GiB)": 36.88, "ppl": 1.21693, "step": 265, "tokens/total": 9213552, "tokens/train_per_sec_per_gpu": 68.7, "tokens/trainable": 277390} +{"epoch": 1.0390625, "grad_norm": 1.1614341735839844, "learning_rate": 5.601593183686955e-05, "loss": 0.16646721959114075, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.18112, "step": 266, "tokens/total": 9250768, "tokens/train_per_sec_per_gpu": 60.37, "tokens/trainable": 278509} +{"epoch": 1.04296875, "grad_norm": 1.4575698375701904, "learning_rate": 5.572569579717961e-05, "loss": 0.22148223221302032, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.24793, "step": 267, "tokens/total": 9286176, "tokens/train_per_sec_per_gpu": 62.25, "tokens/trainable": 279599} +{"epoch": 1.046875, "grad_norm": 1.1192482709884644, "learning_rate": 5.543542955832538e-05, "loss": 0.18501263856887817, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.20323, "step": 268, "tokens/total": 9326128, "tokens/train_per_sec_per_gpu": 53.02, "tokens/trainable": 280682} +{"epoch": 1.05078125, "grad_norm": 1.007797122001648, "learning_rate": 5.514514519946986e-05, "loss": 0.18654945492744446, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.57, "memory/max_allocated (GiB)": 39.57, "ppl": 1.20508, "step": 269, "tokens/total": 9366544, "tokens/train_per_sec_per_gpu": 57.37, "tokens/trainable": 281810} +{"epoch": 1.0546875, "grad_norm": 1.0023550987243652, "learning_rate": 5.485485480053015e-05, "loss": 0.21640637516975403, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.52, "memory/max_allocated (GiB)": 39.52, "ppl": 1.24161, "step": 270, "tokens/total": 9404928, "tokens/train_per_sec_per_gpu": 49.09, "tokens/trainable": 282814} +{"epoch": 1.05859375, "grad_norm": 1.2554994821548462, "learning_rate": 5.4564570441674645e-05, "loss": 0.2029675394296646, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.22503, "step": 271, "tokens/total": 9438592, "tokens/train_per_sec_per_gpu": 66.55, "tokens/trainable": 283848} +{"epoch": 1.0625, "grad_norm": 1.1680448055267334, "learning_rate": 5.42743042028204e-05, "loss": 0.23717957735061646, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.83, "memory/max_allocated (GiB)": 37.83, "ppl": 1.26767, "step": 272, "tokens/total": 9474144, "tokens/train_per_sec_per_gpu": 48.25, "tokens/trainable": 284806} +{"epoch": 1.06640625, "grad_norm": 1.5407180786132812, "learning_rate": 5.3984068163130464e-05, "loss": 0.1815367341041565, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.19906, "step": 273, "tokens/total": 9507904, "tokens/train_per_sec_per_gpu": 69.59, "tokens/trainable": 285981} +{"epoch": 1.0703125, "grad_norm": 1.1606152057647705, "learning_rate": 5.369387440051111e-05, "loss": 0.20439961552619934, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.18, "memory/max_allocated (GiB)": 37.18, "ppl": 1.22679, "step": 274, "tokens/total": 9541616, "tokens/train_per_sec_per_gpu": 59.24, "tokens/trainable": 286983} +{"epoch": 1.07421875, "grad_norm": 0.9667205214500427, "learning_rate": 5.340373499110935e-05, "loss": 0.2020706683397293, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.22393, "step": 275, "tokens/total": 9577424, "tokens/train_per_sec_per_gpu": 59.12, "tokens/trainable": 288058} +{"epoch": 1.078125, "grad_norm": 1.797109842300415, "learning_rate": 5.3113662008810304e-05, "loss": 0.18802353739738464, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.20686, "step": 276, "tokens/total": 9617760, "tokens/train_per_sec_per_gpu": 50.77, "tokens/trainable": 289118} +{"epoch": 1.08203125, "grad_norm": 0.8579794764518738, "learning_rate": 5.282366752473479e-05, "loss": 0.17550285160541534, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.29, "memory/max_allocated (GiB)": 38.29, "ppl": 1.19185, "step": 277, "tokens/total": 9655008, "tokens/train_per_sec_per_gpu": 58.01, "tokens/trainable": 290214} +{"epoch": 1.0859375, "grad_norm": 1.2659257650375366, "learning_rate": 5.2533763606737005e-05, "loss": 0.22080561518669128, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.28, "memory/max_allocated (GiB)": 39.28, "ppl": 1.24708, "step": 278, "tokens/total": 9694896, "tokens/train_per_sec_per_gpu": 48.2, "tokens/trainable": 291231} +{"epoch": 1.08984375, "grad_norm": 1.8076900243759155, "learning_rate": 5.224396231890232e-05, "loss": 0.17397718131542206, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.23, "memory/max_allocated (GiB)": 37.23, "ppl": 1.19003, "step": 279, "tokens/total": 9726672, "tokens/train_per_sec_per_gpu": 67.59, "tokens/trainable": 292347} +{"epoch": 1.09375, "grad_norm": 1.3556725978851318, "learning_rate": 5.195427572104522e-05, "loss": 0.21805799007415771, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.24366, "step": 280, "tokens/total": 9761136, "tokens/train_per_sec_per_gpu": 60.53, "tokens/trainable": 293385} +{"epoch": 1.09765625, "grad_norm": 1.0419448614120483, "learning_rate": 5.166471586820751e-05, "loss": 0.19521889090538025, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.21558, "step": 281, "tokens/total": 9797888, "tokens/train_per_sec_per_gpu": 61.65, "tokens/trainable": 294482} +{"epoch": 1.1015625, "grad_norm": 0.8573160171508789, "learning_rate": 5.1375294810156615e-05, "loss": 0.18381407856941223, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.20179, "step": 282, "tokens/total": 9832528, "tokens/train_per_sec_per_gpu": 60.27, "tokens/trainable": 295504} +{"epoch": 1.10546875, "grad_norm": 1.106004238128662, "learning_rate": 5.1086024590884144e-05, "loss": 0.19549205899238586, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.28, "memory/max_allocated (GiB)": 39.28, "ppl": 1.21591, "step": 283, "tokens/total": 9870784, "tokens/train_per_sec_per_gpu": 53.17, "tokens/trainable": 296505} +{"epoch": 1.109375, "grad_norm": 1.226975440979004, "learning_rate": 5.079691724810461e-05, "loss": 0.17902208864688873, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.19605, "step": 284, "tokens/total": 9906656, "tokens/train_per_sec_per_gpu": 49.66, "tokens/trainable": 297474} +{"epoch": 1.11328125, "grad_norm": 1.0704740285873413, "learning_rate": 5.0507984812754684e-05, "loss": 0.19057485461235046, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.20994, "step": 285, "tokens/total": 9940208, "tokens/train_per_sec_per_gpu": 60.22, "tokens/trainable": 298521} +{"epoch": 1.1171875, "grad_norm": 1.1079121828079224, "learning_rate": 5.021923930849237e-05, "loss": 0.19313301146030426, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.21304, "step": 286, "tokens/total": 9974096, "tokens/train_per_sec_per_gpu": 57.23, "tokens/trainable": 299534} +{"epoch": 1.12109375, "grad_norm": 1.0252375602722168, "learning_rate": 4.99306927511967e-05, "loss": 0.1872812807559967, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.20597, "step": 287, "tokens/total": 10007568, "tokens/train_per_sec_per_gpu": 62.79, "tokens/trainable": 300541} +{"epoch": 1.125, "grad_norm": 1.3606159687042236, "learning_rate": 4.964235714846775e-05, "loss": 0.20520064234733582, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 36.44, "memory/max_allocated (GiB)": 36.44, "ppl": 1.22777, "step": 288, "tokens/total": 10039280, "tokens/train_per_sec_per_gpu": 66.92, "tokens/trainable": 301535} +{"epoch": 1.12890625, "grad_norm": 1.103792428970337, "learning_rate": 4.9354244499126866e-05, "loss": 0.21526393294334412, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 36.23, "memory/max_allocated (GiB)": 36.23, "ppl": 1.24019, "step": 289, "tokens/total": 10070944, "tokens/train_per_sec_per_gpu": 63.74, "tokens/trainable": 302567} +{"epoch": 1.1328125, "grad_norm": 1.5059975385665894, "learning_rate": 4.90663667927174e-05, "loss": 0.18662160634994507, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.20517, "step": 290, "tokens/total": 10103776, "tokens/train_per_sec_per_gpu": 71.51, "tokens/trainable": 303594} +{"epoch": 1.13671875, "grad_norm": 0.976755678653717, "learning_rate": 4.877873600900581e-05, "loss": 0.1883821189403534, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.07, "memory/max_allocated (GiB)": 38.07, "ppl": 1.20729, "step": 291, "tokens/total": 10139888, "tokens/train_per_sec_per_gpu": 56.41, "tokens/trainable": 304633} +{"epoch": 1.140625, "grad_norm": 1.040105938911438, "learning_rate": 4.849136411748306e-05, "loss": 0.17172180116176605, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.04, "memory/max_allocated (GiB)": 39.04, "ppl": 1.18735, "step": 292, "tokens/total": 10177936, "tokens/train_per_sec_per_gpu": 54.13, "tokens/trainable": 305690} +{"epoch": 1.14453125, "grad_norm": 0.8259438872337341, "learning_rate": 4.8204263076866574e-05, "loss": 0.18295684456825256, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.20076, "step": 293, "tokens/total": 10213856, "tokens/train_per_sec_per_gpu": 58.02, "tokens/trainable": 306752} +{"epoch": 1.1484375, "grad_norm": 1.8552590608596802, "learning_rate": 4.791744483460251e-05, "loss": 0.20239973068237305, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.74, "memory/max_allocated (GiB)": 37.74, "ppl": 1.22434, "step": 294, "tokens/total": 10248352, "tokens/train_per_sec_per_gpu": 52.16, "tokens/trainable": 307746} +{"epoch": 1.15234375, "grad_norm": 1.2265801429748535, "learning_rate": 4.7630921326368736e-05, "loss": 0.18375256657600403, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.20172, "step": 295, "tokens/total": 10287072, "tokens/train_per_sec_per_gpu": 50.45, "tokens/trainable": 308728} +{"epoch": 1.15625, "grad_norm": 1.0152767896652222, "learning_rate": 4.7344704475577916e-05, "loss": 0.1893073320388794, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 35.7, "memory/max_allocated (GiB)": 35.7, "ppl": 1.20841, "step": 296, "tokens/total": 10316640, "tokens/train_per_sec_per_gpu": 68.85, "tokens/trainable": 309754} +{"epoch": 1.16015625, "grad_norm": 1.7037112712860107, "learning_rate": 4.705880619288153e-05, "loss": 0.19359076023101807, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 38.37, "memory/max_allocated (GiB)": 38.37, "ppl": 1.2136, "step": 297, "tokens/total": 10352864, "tokens/train_per_sec_per_gpu": 53.48, "tokens/trainable": 310739} +{"epoch": 1.1640625, "grad_norm": 1.1305632591247559, "learning_rate": 4.677323837567412e-05, "loss": 0.18200469017028809, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.19962, "step": 298, "tokens/total": 10385696, "tokens/train_per_sec_per_gpu": 65.16, "tokens/trainable": 311795} +{"epoch": 1.16796875, "grad_norm": 1.068151593208313, "learning_rate": 4.6488012907598146e-05, "loss": 0.1918257176876068, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 36.4, "memory/max_allocated (GiB)": 36.4, "ppl": 1.21146, "step": 299, "tokens/total": 10417904, "tokens/train_per_sec_per_gpu": 57.87, "tokens/trainable": 312750} +{"epoch": 1.171875, "grad_norm": 1.1165587902069092, "learning_rate": 4.620314165804964e-05, "loss": 0.1808488965034485, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.19823, "step": 300, "tokens/total": 10453728, "tokens/train_per_sec_per_gpu": 59.04, "tokens/trainable": 313846} +{"epoch": 1.17578125, "grad_norm": 0.6777521371841431, "learning_rate": 4.591863648168407e-05, "loss": 0.16916775703430176, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.18432, "step": 301, "tokens/total": 10488288, "tokens/train_per_sec_per_gpu": 59.84, "tokens/trainable": 314951} +{"epoch": 1.1796875, "grad_norm": 1.6237705945968628, "learning_rate": 4.5634509217923135e-05, "loss": 0.1903623342514038, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 37.96, "memory/max_allocated (GiB)": 37.96, "ppl": 1.20969, "step": 302, "tokens/total": 10521824, "tokens/train_per_sec_per_gpu": 61.55, "tokens/trainable": 315989} +{"epoch": 1.18359375, "grad_norm": 0.8324459195137024, "learning_rate": 4.535077169046201e-05, "loss": 0.19621491432189941, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 38.03, "memory/max_allocated (GiB)": 38.03, "ppl": 1.21679, "step": 303, "tokens/total": 10558432, "tokens/train_per_sec_per_gpu": 57.51, "tokens/trainable": 317019} +{"epoch": 1.1875, "grad_norm": 1.4992953538894653, "learning_rate": 4.506743570677743e-05, "loss": 0.21246525645256042, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 36.35, "memory/max_allocated (GiB)": 36.35, "ppl": 1.23672, "step": 304, "tokens/total": 10588752, "tokens/train_per_sec_per_gpu": 72.64, "tokens/trainable": 318014} +{"epoch": 1.19140625, "grad_norm": 1.5652921199798584, "learning_rate": 4.478451305763618e-05, "loss": 0.21109089255332947, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.23502, "step": 305, "tokens/total": 10622336, "tokens/train_per_sec_per_gpu": 64.65, "tokens/trainable": 319052} +{"epoch": 1.1953125, "grad_norm": 0.9139826893806458, "learning_rate": 4.450201551660454e-05, "loss": 0.18892666697502136, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 36.41, "memory/max_allocated (GiB)": 36.41, "ppl": 1.20795, "step": 306, "tokens/total": 10653520, "tokens/train_per_sec_per_gpu": 63.9, "tokens/trainable": 320057} +{"epoch": 1.19921875, "grad_norm": 1.0938358306884766, "learning_rate": 4.4219954839558276e-05, "loss": 0.1906939148902893, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.21009, "step": 307, "tokens/total": 10688144, "tokens/train_per_sec_per_gpu": 54.27, "tokens/trainable": 321069} +{"epoch": 1.203125, "grad_norm": 0.9386539459228516, "learning_rate": 4.393834276419352e-05, "loss": 0.19729240238666534, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 35.7, "memory/max_allocated (GiB)": 35.7, "ppl": 1.2181, "step": 308, "tokens/total": 10718032, "tokens/train_per_sec_per_gpu": 72.04, "tokens/trainable": 322101} +{"epoch": 1.20703125, "grad_norm": 1.0700088739395142, "learning_rate": 4.36571910095382e-05, "loss": 0.1986759901046753, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.21979, "step": 309, "tokens/total": 10756464, "tokens/train_per_sec_per_gpu": 55.62, "tokens/trainable": 323155} +{"epoch": 1.2109375, "grad_norm": 1.3301975727081299, "learning_rate": 4.337651127546448e-05, "loss": 0.20967882871627808, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.23328, "step": 310, "tokens/total": 10792064, "tokens/train_per_sec_per_gpu": 62.86, "tokens/trainable": 324194} +{"epoch": 1.21484375, "grad_norm": 1.2838917970657349, "learning_rate": 4.3096315242201736e-05, "loss": 0.20641732215881348, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 36.51, "memory/max_allocated (GiB)": 36.51, "ppl": 1.22927, "step": 311, "tokens/total": 10824800, "tokens/train_per_sec_per_gpu": 59.55, "tokens/trainable": 325230} +{"epoch": 1.21875, "grad_norm": 1.3273584842681885, "learning_rate": 4.2816614569850635e-05, "loss": 0.19127124547958374, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 38.66, "memory/max_allocated (GiB)": 38.66, "ppl": 1.21079, "step": 312, "tokens/total": 10860576, "tokens/train_per_sec_per_gpu": 58.78, "tokens/trainable": 326230} +{"epoch": 1.22265625, "grad_norm": 1.5207446813583374, "learning_rate": 4.2537420897897864e-05, "loss": 0.19745540618896484, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 38.72, "memory/max_allocated (GiB)": 38.72, "ppl": 1.2183, "step": 313, "tokens/total": 10896224, "tokens/train_per_sec_per_gpu": 54.25, "tokens/trainable": 327247} +{"epoch": 1.2265625, "grad_norm": 1.5411922931671143, "learning_rate": 4.225874584473174e-05, "loss": 0.20332221686840057, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.22547, "step": 314, "tokens/total": 10928768, "tokens/train_per_sec_per_gpu": 64.39, "tokens/trainable": 328261} +{"epoch": 1.23046875, "grad_norm": 1.2351287603378296, "learning_rate": 4.19806010071587e-05, "loss": 0.2051369696855545, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.22769, "step": 315, "tokens/total": 10966848, "tokens/train_per_sec_per_gpu": 54.99, "tokens/trainable": 329344} +{"epoch": 1.234375, "grad_norm": 0.9836161732673645, "learning_rate": 4.170299795992081e-05, "loss": 0.20338255167007446, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 36.88, "memory/max_allocated (GiB)": 36.88, "ppl": 1.22554, "step": 316, "tokens/total": 11000416, "tokens/train_per_sec_per_gpu": 60.81, "tokens/trainable": 330395} +{"epoch": 1.23828125, "grad_norm": 2.225940465927124, "learning_rate": 4.142594825521398e-05, "loss": 0.17057423293590546, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.18599, "step": 317, "tokens/total": 11039520, "tokens/train_per_sec_per_gpu": 51.73, "tokens/trainable": 331452} +{"epoch": 1.2421875, "grad_norm": 1.1642955541610718, "learning_rate": 4.114946342220728e-05, "loss": 0.19042761623859406, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.20977, "step": 318, "tokens/total": 11076096, "tokens/train_per_sec_per_gpu": 57.64, "tokens/trainable": 332542} +{"epoch": 1.24609375, "grad_norm": 0.5930197238922119, "learning_rate": 4.087355496656321e-05, "loss": 0.1631363332271576, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.1772, "step": 319, "tokens/total": 11109728, "tokens/train_per_sec_per_gpu": 68.62, "tokens/trainable": 333659} +{"epoch": 1.25, "grad_norm": 1.1262805461883545, "learning_rate": 4.05982343699588e-05, "loss": 0.19402232766151428, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.21412, "step": 320, "tokens/total": 11148976, "tokens/train_per_sec_per_gpu": 59.32, "tokens/trainable": 334805} +{"epoch": 1.25390625, "grad_norm": 1.0934110879898071, "learning_rate": 4.0323513089607876e-05, "loss": 0.1658870130777359, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.18044, "step": 321, "tokens/total": 11183888, "tokens/train_per_sec_per_gpu": 58.1, "tokens/trainable": 335895} +{"epoch": 1.2578125, "grad_norm": 1.5761691331863403, "learning_rate": 4.004940255778431e-05, "loss": 0.18834801018238068, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 39.63, "memory/max_allocated (GiB)": 39.63, "ppl": 1.20725, "step": 322, "tokens/total": 11222608, "tokens/train_per_sec_per_gpu": 57.13, "tokens/trainable": 336952} +{"epoch": 1.26171875, "grad_norm": 0.6844615936279297, "learning_rate": 3.977591418134619e-05, "loss": 0.1874157190322876, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.20613, "step": 323, "tokens/total": 11257568, "tokens/train_per_sec_per_gpu": 62.18, "tokens/trainable": 338006} +{"epoch": 1.265625, "grad_norm": 1.484318733215332, "learning_rate": 3.95030593412612e-05, "loss": 0.1984628438949585, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.21953, "step": 324, "tokens/total": 11293472, "tokens/train_per_sec_per_gpu": 57.83, "tokens/trainable": 339026} +{"epoch": 1.26953125, "grad_norm": 1.2072495222091675, "learning_rate": 3.923084939213296e-05, "loss": 0.20763397216796875, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 35.75, "memory/max_allocated (GiB)": 35.75, "ppl": 1.23076, "step": 325, "tokens/total": 11324048, "tokens/train_per_sec_per_gpu": 68.54, "tokens/trainable": 340071} +{"epoch": 1.2734375, "grad_norm": 0.7690504193305969, "learning_rate": 3.895929566172861e-05, "loss": 0.15235446393489838, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.16457, "step": 326, "tokens/total": 11360064, "tokens/train_per_sec_per_gpu": 60.79, "tokens/trainable": 341175} +{"epoch": 1.27734375, "grad_norm": 0.8582741618156433, "learning_rate": 3.868840945050728e-05, "loss": 0.16492336988449097, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 39.19, "memory/max_allocated (GiB)": 39.19, "ppl": 1.1793, "step": 327, "tokens/total": 11399008, "tokens/train_per_sec_per_gpu": 56.6, "tokens/trainable": 342264} +{"epoch": 1.28125, "grad_norm": 1.1145023107528687, "learning_rate": 3.841820203114995e-05, "loss": 0.184311643242836, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 36.78, "memory/max_allocated (GiB)": 36.78, "ppl": 1.20239, "step": 328, "tokens/total": 11430752, "tokens/train_per_sec_per_gpu": 67.92, "tokens/trainable": 343353} +{"epoch": 1.28515625, "grad_norm": 0.7542400360107422, "learning_rate": 3.814868464809027e-05, "loss": 0.1811949908733368, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 36.77, "memory/max_allocated (GiB)": 36.77, "ppl": 1.19865, "step": 329, "tokens/total": 11462688, "tokens/train_per_sec_per_gpu": 61.77, "tokens/trainable": 344349} +{"epoch": 1.2890625, "grad_norm": 0.9089362621307373, "learning_rate": 3.787986851704667e-05, "loss": 0.20285923779010773, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.2249, "step": 330, "tokens/total": 11496240, "tokens/train_per_sec_per_gpu": 66.09, "tokens/trainable": 345405} +{"epoch": 1.29296875, "grad_norm": 1.460120677947998, "learning_rate": 3.7611764824555654e-05, "loss": 0.16738812625408173, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 37.84, "memory/max_allocated (GiB)": 37.84, "ppl": 1.18221, "step": 331, "tokens/total": 11527968, "tokens/train_per_sec_per_gpu": 76.69, "tokens/trainable": 346558} +{"epoch": 1.296875, "grad_norm": 1.0050841569900513, "learning_rate": 3.734438472750619e-05, "loss": 0.17949604988098145, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.19661, "step": 332, "tokens/total": 11567936, "tokens/train_per_sec_per_gpu": 53.56, "tokens/trainable": 347670} +{"epoch": 1.30078125, "grad_norm": 1.4534220695495605, "learning_rate": 3.707773935267552e-05, "loss": 0.18101716041564941, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.19844, "step": 333, "tokens/total": 11604992, "tokens/train_per_sec_per_gpu": 58.52, "tokens/trainable": 348799} +{"epoch": 1.3046875, "grad_norm": 1.4934182167053223, "learning_rate": 3.68118397962661e-05, "loss": 0.20571887493133545, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 38.49, "memory/max_allocated (GiB)": 38.49, "ppl": 1.22841, "step": 334, "tokens/total": 11639744, "tokens/train_per_sec_per_gpu": 56.2, "tokens/trainable": 349761} +{"epoch": 1.30859375, "grad_norm": 0.5702500939369202, "learning_rate": 3.654669712344384e-05, "loss": 0.15885108709335327, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 36.72, "memory/max_allocated (GiB)": 36.72, "ppl": 1.17216, "step": 335, "tokens/total": 11670944, "tokens/train_per_sec_per_gpu": 75.54, "tokens/trainable": 350817} +{"epoch": 1.3125, "grad_norm": 0.7433128952980042, "learning_rate": 3.628232236787763e-05, "loss": 0.16871468722820282, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 37.91, "memory/max_allocated (GiB)": 37.91, "ppl": 1.18378, "step": 336, "tokens/total": 11706544, "tokens/train_per_sec_per_gpu": 56.57, "tokens/trainable": 351930} +{"epoch": 1.31640625, "grad_norm": 0.9649227857589722, "learning_rate": 3.6018726531280144e-05, "loss": 0.1596893072128296, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.17315, "step": 337, "tokens/total": 11743952, "tokens/train_per_sec_per_gpu": 60.85, "tokens/trainable": 353038} +{"epoch": 1.3203125, "grad_norm": 1.1719002723693848, "learning_rate": 3.575592058295017e-05, "loss": 0.2035491019487381, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.22575, "step": 338, "tokens/total": 11777936, "tokens/train_per_sec_per_gpu": 64.26, "tokens/trainable": 354118} +{"epoch": 1.32421875, "grad_norm": 1.40962815284729, "learning_rate": 3.549391545931585e-05, "loss": 0.17078664898872375, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 38.93, "memory/max_allocated (GiB)": 38.93, "ppl": 1.18624, "step": 339, "tokens/total": 11812416, "tokens/train_per_sec_per_gpu": 62.86, "tokens/trainable": 355168} +{"epoch": 1.328125, "grad_norm": 1.0358974933624268, "learning_rate": 3.5232722063479914e-05, "loss": 0.18893572688102722, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.20796, "step": 340, "tokens/total": 11846704, "tokens/train_per_sec_per_gpu": 67.85, "tokens/trainable": 356250} +{"epoch": 1.33203125, "grad_norm": 0.6457139849662781, "learning_rate": 3.49723512647657e-05, "loss": 0.16270104050636292, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.17668, "step": 341, "tokens/total": 11885968, "tokens/train_per_sec_per_gpu": 54.61, "tokens/trainable": 357366} +{"epoch": 1.3359375, "grad_norm": 1.1418817043304443, "learning_rate": 3.471281389826491e-05, "loss": 0.18856073915958405, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.20751, "step": 342, "tokens/total": 11917024, "tokens/train_per_sec_per_gpu": 63.86, "tokens/trainable": 358319} +{"epoch": 1.33984375, "grad_norm": 0.951632022857666, "learning_rate": 3.4454120764386764e-05, "loss": 0.1740957647562027, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.19017, "step": 343, "tokens/total": 11955648, "tokens/train_per_sec_per_gpu": 57.11, "tokens/trainable": 359399} +{"epoch": 1.34375, "grad_norm": 1.0343996286392212, "learning_rate": 3.4196282628408526e-05, "loss": 0.19479291141033173, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 37.58, "memory/max_allocated (GiB)": 37.58, "ppl": 1.21506, "step": 344, "tokens/total": 11989936, "tokens/train_per_sec_per_gpu": 62.82, "tokens/trainable": 360454} +{"epoch": 1.34765625, "grad_norm": 1.7289869785308838, "learning_rate": 3.3939310220027456e-05, "loss": 0.182133287191391, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.19977, "step": 345, "tokens/total": 12023968, "tokens/train_per_sec_per_gpu": 60.62, "tokens/trainable": 361543} +{"epoch": 1.3515625, "grad_norm": 0.8481875658035278, "learning_rate": 3.3683214232914404e-05, "loss": 0.15801787376403809, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.17119, "step": 346, "tokens/total": 12059632, "tokens/train_per_sec_per_gpu": 60.3, "tokens/trainable": 362623} +{"epoch": 1.35546875, "grad_norm": 1.0678465366363525, "learning_rate": 3.342800532426873e-05, "loss": 0.18756425380706787, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.20631, "step": 347, "tokens/total": 12095616, "tokens/train_per_sec_per_gpu": 60.47, "tokens/trainable": 363641} +{"epoch": 1.359375, "grad_norm": 0.8495708107948303, "learning_rate": 3.317369411437484e-05, "loss": 0.19004487991333008, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.2093, "step": 348, "tokens/total": 12131488, "tokens/train_per_sec_per_gpu": 56.51, "tokens/trainable": 364672} +{"epoch": 1.36328125, "grad_norm": 0.8789288401603699, "learning_rate": 3.292029118616024e-05, "loss": 0.18114221096038818, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.19859, "step": 349, "tokens/total": 12164208, "tokens/train_per_sec_per_gpu": 57.37, "tokens/trainable": 365654} +{"epoch": 1.3671875, "grad_norm": 0.8972854614257812, "learning_rate": 3.266780708475511e-05, "loss": 0.17617003619670868, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 39.12, "memory/max_allocated (GiB)": 39.12, "ppl": 1.19264, "step": 350, "tokens/total": 12199616, "tokens/train_per_sec_per_gpu": 58.87, "tokens/trainable": 366729} +{"epoch": 1.37109375, "grad_norm": 0.5260622501373291, "learning_rate": 3.241625231705354e-05, "loss": 0.16200438141822815, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.17587, "step": 351, "tokens/total": 12235664, "tokens/train_per_sec_per_gpu": 59.61, "tokens/trainable": 367791} +{"epoch": 1.375, "grad_norm": 1.8304729461669922, "learning_rate": 3.216563735127618e-05, "loss": 0.1528596580028534, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 39.12, "memory/max_allocated (GiB)": 39.12, "ppl": 1.16516, "step": 352, "tokens/total": 12270800, "tokens/train_per_sec_per_gpu": 61.61, "tokens/trainable": 368924} +{"epoch": 1.37890625, "grad_norm": 0.9959712624549866, "learning_rate": 3.191597261653475e-05, "loss": 0.18312008678913116, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.20096, "step": 353, "tokens/total": 12307568, "tokens/train_per_sec_per_gpu": 58.15, "tokens/trainable": 370005} +{"epoch": 1.3828125, "grad_norm": 0.7965621948242188, "learning_rate": 3.166726850239794e-05, "loss": 0.1615283489227295, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 37.18, "memory/max_allocated (GiB)": 37.18, "ppl": 1.17531, "step": 354, "tokens/total": 12341200, "tokens/train_per_sec_per_gpu": 63.05, "tokens/trainable": 371083} +{"epoch": 1.38671875, "grad_norm": 1.1322109699249268, "learning_rate": 3.141953535845912e-05, "loss": 0.19681459665298462, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 36.16, "memory/max_allocated (GiB)": 36.16, "ppl": 1.21752, "step": 355, "tokens/total": 12372016, "tokens/train_per_sec_per_gpu": 63.55, "tokens/trainable": 372021} +{"epoch": 1.390625, "grad_norm": 1.3697562217712402, "learning_rate": 3.11727834939056e-05, "loss": 0.22829563915729523, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 36.28, "memory/max_allocated (GiB)": 36.28, "ppl": 1.25646, "step": 356, "tokens/total": 12403856, "tokens/train_per_sec_per_gpu": 60.11, "tokens/trainable": 373052} +{"epoch": 1.39453125, "grad_norm": 1.0662426948547363, "learning_rate": 3.092702317708967e-05, "loss": 0.18607014417648315, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.20451, "step": 357, "tokens/total": 12437584, "tokens/train_per_sec_per_gpu": 62.97, "tokens/trainable": 374068} +{"epoch": 1.3984375, "grad_norm": 1.9266307353973389, "learning_rate": 3.0682264635101276e-05, "loss": 0.17700716853141785, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 37.44, "memory/max_allocated (GiB)": 37.44, "ppl": 1.19364, "step": 358, "tokens/total": 12470096, "tokens/train_per_sec_per_gpu": 62.78, "tokens/trainable": 375077} +{"epoch": 1.40234375, "grad_norm": 0.9256980419158936, "learning_rate": 3.0438518053342407e-05, "loss": 0.16398361325263977, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.1782, "step": 359, "tokens/total": 12509168, "tokens/train_per_sec_per_gpu": 56.31, "tokens/trainable": 376205} +{"epoch": 1.40625, "grad_norm": 0.9271458983421326, "learning_rate": 3.0195793575103266e-05, "loss": 0.17387306690216064, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.1899, "step": 360, "tokens/total": 12548208, "tokens/train_per_sec_per_gpu": 55.34, "tokens/trainable": 377281} +{"epoch": 1.41015625, "grad_norm": 3.1427953243255615, "learning_rate": 2.9954101301140146e-05, "loss": 0.19165709614753723, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.21126, "step": 361, "tokens/total": 12583552, "tokens/train_per_sec_per_gpu": 59.3, "tokens/trainable": 378322} +{"epoch": 1.4140625, "grad_norm": 1.4816306829452515, "learning_rate": 2.9713451289255123e-05, "loss": 0.21215592324733734, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 36.9, "memory/max_allocated (GiB)": 36.9, "ppl": 1.23634, "step": 362, "tokens/total": 12612944, "tokens/train_per_sec_per_gpu": 62.43, "tokens/trainable": 379351} +{"epoch": 1.41796875, "grad_norm": 0.5523360967636108, "learning_rate": 2.9473853553877484e-05, "loss": 0.1583450585603714, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.17157, "step": 363, "tokens/total": 12646880, "tokens/train_per_sec_per_gpu": 60.85, "tokens/trainable": 380390} +{"epoch": 1.421875, "grad_norm": 1.1142023801803589, "learning_rate": 2.9235318065647e-05, "loss": 0.17743432521820068, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 39.61, "memory/max_allocated (GiB)": 39.61, "ppl": 1.19415, "step": 364, "tokens/total": 12686896, "tokens/train_per_sec_per_gpu": 53.91, "tokens/trainable": 381449} +{"epoch": 1.42578125, "grad_norm": 1.12079656124115, "learning_rate": 2.8997854750998964e-05, "loss": 0.17928951978683472, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.19637, "step": 365, "tokens/total": 12726416, "tokens/train_per_sec_per_gpu": 59.33, "tokens/trainable": 382566} +{"epoch": 1.4296875, "grad_norm": 0.6073111295700073, "learning_rate": 2.8761473491751258e-05, "loss": 0.17578458786010742, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.63, "memory/max_allocated (GiB)": 36.63, "ppl": 1.19218, "step": 366, "tokens/total": 12759248, "tokens/train_per_sec_per_gpu": 60.48, "tokens/trainable": 383564} +{"epoch": 1.43359375, "grad_norm": 0.5267775654792786, "learning_rate": 2.8526184124692883e-05, "loss": 0.14412635564804077, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.15503, "step": 367, "tokens/total": 12795904, "tokens/train_per_sec_per_gpu": 53.12, "tokens/trainable": 384667} +{"epoch": 1.4375, "grad_norm": 0.7778412103652954, "learning_rate": 2.829199644117484e-05, "loss": 0.17311257123947144, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.189, "step": 368, "tokens/total": 12829616, "tokens/train_per_sec_per_gpu": 63.39, "tokens/trainable": 385630} +{"epoch": 1.44140625, "grad_norm": 1.1350785493850708, "learning_rate": 2.8058920186702553e-05, "loss": 0.16500937938690186, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.91, "memory/max_allocated (GiB)": 38.91, "ppl": 1.1794, "step": 369, "tokens/total": 12867744, "tokens/train_per_sec_per_gpu": 59.37, "tokens/trainable": 386744} +{"epoch": 1.4453125, "grad_norm": 0.5898240804672241, "learning_rate": 2.782696506053033e-05, "loss": 0.17415082454681396, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.19024, "step": 370, "tokens/total": 12905936, "tokens/train_per_sec_per_gpu": 53.11, "tokens/trainable": 387739} +{"epoch": 1.44921875, "grad_norm": 0.7785534858703613, "learning_rate": 2.7596140715257824e-05, "loss": 0.16672250628471375, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.18143, "step": 371, "tokens/total": 12941216, "tokens/train_per_sec_per_gpu": 62.06, "tokens/trainable": 388842} +{"epoch": 1.453125, "grad_norm": 0.6477351188659668, "learning_rate": 2.7366456756428184e-05, "loss": 0.16590869426727295, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.18047, "step": 372, "tokens/total": 12975584, "tokens/train_per_sec_per_gpu": 60.6, "tokens/trainable": 389878} +{"epoch": 1.45703125, "grad_norm": 1.1160820722579956, "learning_rate": 2.7137922742128486e-05, "loss": 0.1697084903717041, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.21, "memory/max_allocated (GiB)": 38.21, "ppl": 1.18496, "step": 373, "tokens/total": 13010464, "tokens/train_per_sec_per_gpu": 62.02, "tokens/trainable": 390945} +{"epoch": 1.4609375, "grad_norm": 0.5703273415565491, "learning_rate": 2.691054818259188e-05, "loss": 0.16323886811733246, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.4, "memory/max_allocated (GiB)": 37.4, "ppl": 1.17732, "step": 374, "tokens/total": 13045136, "tokens/train_per_sec_per_gpu": 62.16, "tokens/trainable": 392058} +{"epoch": 1.46484375, "grad_norm": 1.0432331562042236, "learning_rate": 2.6684342539801933e-05, "loss": 0.21631130576133728, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.49, "memory/max_allocated (GiB)": 36.49, "ppl": 1.24149, "step": 375, "tokens/total": 13076400, "tokens/train_per_sec_per_gpu": 61.94, "tokens/trainable": 393036} +{"epoch": 1.46875, "grad_norm": 0.9219545722007751, "learning_rate": 2.645931522709877e-05, "loss": 0.16102543473243713, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.54, "memory/max_allocated (GiB)": 39.54, "ppl": 1.17471, "step": 376, "tokens/total": 13112144, "tokens/train_per_sec_per_gpu": 73.83, "tokens/trainable": 394207} +{"epoch": 1.47265625, "grad_norm": 1.1875574588775635, "learning_rate": 2.6235475608787365e-05, "loss": 0.17794273793697357, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.6, "memory/max_allocated (GiB)": 38.6, "ppl": 1.19476, "step": 377, "tokens/total": 13147872, "tokens/train_per_sec_per_gpu": 59.54, "tokens/trainable": 395236} +{"epoch": 1.4765625, "grad_norm": 1.4538602828979492, "learning_rate": 2.6012832999747916e-05, "loss": 0.20993630588054657, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.2336, "step": 378, "tokens/total": 13178656, "tokens/train_per_sec_per_gpu": 64.43, "tokens/trainable": 396213} +{"epoch": 1.48046875, "grad_norm": 1.8968397378921509, "learning_rate": 2.579139666504821e-05, "loss": 0.19407126307487488, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.21418, "step": 379, "tokens/total": 13218720, "tokens/train_per_sec_per_gpu": 51.67, "tokens/trainable": 397215} +{"epoch": 1.484375, "grad_norm": 1.275347113609314, "learning_rate": 2.557117581955798e-05, "loss": 0.17829865217208862, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 1.19518, "step": 380, "tokens/total": 13252544, "tokens/train_per_sec_per_gpu": 59.96, "tokens/trainable": 398270} +{"epoch": 1.48828125, "grad_norm": 0.8170477747917175, "learning_rate": 2.5352179627565532e-05, "loss": 0.213948592543602, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.06, "memory/max_allocated (GiB)": 38.06, "ppl": 1.23856, "step": 381, "tokens/total": 13286832, "tokens/train_per_sec_per_gpu": 53.61, "tokens/trainable": 399274} +{"epoch": 1.4921875, "grad_norm": 1.2358715534210205, "learning_rate": 2.5134417202396277e-05, "loss": 0.18966305255889893, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.20884, "step": 382, "tokens/total": 13320432, "tokens/train_per_sec_per_gpu": 57.95, "tokens/trainable": 400243} +{"epoch": 1.49609375, "grad_norm": 0.7306169867515564, "learning_rate": 2.491789760603361e-05, "loss": 0.1808585673570633, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.19825, "step": 383, "tokens/total": 13356928, "tokens/train_per_sec_per_gpu": 53.78, "tokens/trainable": 401259} +{"epoch": 1.5, "grad_norm": 1.4707530736923218, "learning_rate": 2.4702629848741764e-05, "loss": 0.18552884459495544, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.20385, "step": 384, "tokens/total": 13391568, "tokens/train_per_sec_per_gpu": 62.54, "tokens/trainable": 402263} +{"epoch": 1.50390625, "grad_norm": 0.8799921870231628, "learning_rate": 2.4488622888690785e-05, "loss": 0.1772073358297348, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.19388, "step": 385, "tokens/total": 13425152, "tokens/train_per_sec_per_gpu": 55.34, "tokens/trainable": 403239} +{"epoch": 1.5078125, "grad_norm": 0.6102913022041321, "learning_rate": 2.427588563158384e-05, "loss": 0.15590208768844604, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.11, "memory/max_allocated (GiB)": 39.11, "ppl": 1.16871, "step": 386, "tokens/total": 13459504, "tokens/train_per_sec_per_gpu": 62.67, "tokens/trainable": 404313} +{"epoch": 1.51171875, "grad_norm": 0.8500211834907532, "learning_rate": 2.406442693028651e-05, "loss": 0.20047995448112488, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.22199, "step": 387, "tokens/total": 13491136, "tokens/train_per_sec_per_gpu": 61.49, "tokens/trainable": 405263} +{"epoch": 1.515625, "grad_norm": 0.7752060294151306, "learning_rate": 2.3854255584458547e-05, "loss": 0.17078804969787598, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.18624, "step": 388, "tokens/total": 13530112, "tokens/train_per_sec_per_gpu": 51.59, "tokens/trainable": 406280} +{"epoch": 1.51953125, "grad_norm": 0.593270480632782, "learning_rate": 2.3645380340187508e-05, "loss": 0.15962284803390503, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.14, "memory/max_allocated (GiB)": 39.14, "ppl": 1.17307, "step": 389, "tokens/total": 13565680, "tokens/train_per_sec_per_gpu": 62.91, "tokens/trainable": 407354} +{"epoch": 1.5234375, "grad_norm": 0.6934286952018738, "learning_rate": 2.3437809889624914e-05, "loss": 0.18546843528747559, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.52, "memory/max_allocated (GiB)": 36.52, "ppl": 1.20378, "step": 390, "tokens/total": 13596960, "tokens/train_per_sec_per_gpu": 72.16, "tokens/trainable": 408404} +{"epoch": 1.52734375, "grad_norm": 0.7567867040634155, "learning_rate": 2.3231552870624487e-05, "loss": 0.16307459771633148, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.17712, "step": 391, "tokens/total": 13627840, "tokens/train_per_sec_per_gpu": 71.04, "tokens/trainable": 409477} +{"epoch": 1.53125, "grad_norm": 0.4839562475681305, "learning_rate": 2.3026617866382657e-05, "loss": 0.14928247034549713, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.7, "memory/max_allocated (GiB)": 36.7, "ppl": 1.161, "step": 392, "tokens/total": 13660848, "tokens/train_per_sec_per_gpu": 63.02, "tokens/trainable": 410563} +{"epoch": 1.53515625, "grad_norm": 1.8106588125228882, "learning_rate": 2.2823013405081507e-05, "loss": 0.16178376972675323, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.17561, "step": 393, "tokens/total": 13699024, "tokens/train_per_sec_per_gpu": 53.8, "tokens/trainable": 411635} +{"epoch": 1.5390625, "grad_norm": 5.001317024230957, "learning_rate": 2.2620747959533722e-05, "loss": 0.15263530611991882, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.1649, "step": 394, "tokens/total": 13732336, "tokens/train_per_sec_per_gpu": 66.1, "tokens/trainable": 412746} +{"epoch": 1.54296875, "grad_norm": 0.7147670984268188, "learning_rate": 2.2419829946830123e-05, "loss": 0.17045727372169495, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.18585, "step": 395, "tokens/total": 13768112, "tokens/train_per_sec_per_gpu": 60.44, "tokens/trainable": 413775} +{"epoch": 1.546875, "grad_norm": 1.2623398303985596, "learning_rate": 2.2220267727989325e-05, "loss": 0.15827658772468567, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.74, "memory/max_allocated (GiB)": 37.74, "ppl": 1.17149, "step": 396, "tokens/total": 13802480, "tokens/train_per_sec_per_gpu": 60.08, "tokens/trainable": 414830} +{"epoch": 1.55078125, "grad_norm": 0.7598130702972412, "learning_rate": 2.202206960760984e-05, "loss": 0.1498219221830368, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.16163, "step": 397, "tokens/total": 13840736, "tokens/train_per_sec_per_gpu": 56.45, "tokens/trainable": 415977} +{"epoch": 1.5546875, "grad_norm": 0.9964093565940857, "learning_rate": 2.182524383352446e-05, "loss": 0.1889490783214569, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.20798, "step": 398, "tokens/total": 13876896, "tokens/train_per_sec_per_gpu": 56.65, "tokens/trainable": 416993} +{"epoch": 1.55859375, "grad_norm": 1.0303598642349243, "learning_rate": 2.1629798596457056e-05, "loss": 0.17552004754543304, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.34, "memory/max_allocated (GiB)": 36.34, "ppl": 1.19187, "step": 399, "tokens/total": 13908992, "tokens/train_per_sec_per_gpu": 63.41, "tokens/trainable": 417998} +{"epoch": 1.5625, "grad_norm": 0.9094043374061584, "learning_rate": 2.1435742029681725e-05, "loss": 0.18421195447444916, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.29, "memory/max_allocated (GiB)": 36.29, "ppl": 1.20227, "step": 400, "tokens/total": 13939456, "tokens/train_per_sec_per_gpu": 69.15, "tokens/trainable": 418959} +{"epoch": 1.56640625, "grad_norm": 0.7407302260398865, "learning_rate": 2.124308220868431e-05, "loss": 0.15547595918178558, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.16821, "step": 401, "tokens/total": 13975312, "tokens/train_per_sec_per_gpu": 57.19, "tokens/trainable": 420020} +{"epoch": 1.5703125, "grad_norm": 0.7729389071464539, "learning_rate": 2.105182715082638e-05, "loss": 0.1605343222618103, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.92, "memory/max_allocated (GiB)": 36.92, "ppl": 1.17414, "step": 402, "tokens/total": 14009056, "tokens/train_per_sec_per_gpu": 63.77, "tokens/trainable": 421113} +{"epoch": 1.57421875, "grad_norm": 0.6516868472099304, "learning_rate": 2.0861984815011552e-05, "loss": 0.1781543791294098, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 1.19501, "step": 403, "tokens/total": 14044320, "tokens/train_per_sec_per_gpu": 55.82, "tokens/trainable": 422084} +{"epoch": 1.578125, "grad_norm": 1.4345760345458984, "learning_rate": 2.0673563101354323e-05, "loss": 0.18972276151180267, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.20891, "step": 404, "tokens/total": 14079600, "tokens/train_per_sec_per_gpu": 53.3, "tokens/trainable": 423103} +{"epoch": 1.58203125, "grad_norm": 1.1800034046173096, "learning_rate": 2.0486569850851317e-05, "loss": 0.17792432010173798, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.19473, "step": 405, "tokens/total": 14114368, "tokens/train_per_sec_per_gpu": 61.85, "tokens/trainable": 424119} +{"epoch": 1.5859375, "grad_norm": 0.8031272292137146, "learning_rate": 2.0301012845054956e-05, "loss": 0.16964466869831085, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.18488, "step": 406, "tokens/total": 14153248, "tokens/train_per_sec_per_gpu": 51.78, "tokens/trainable": 425159} +{"epoch": 1.58984375, "grad_norm": 1.0723897218704224, "learning_rate": 2.011689980574966e-05, "loss": 0.17551466822624207, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.18, "memory/max_allocated (GiB)": 38.18, "ppl": 1.19186, "step": 407, "tokens/total": 14189632, "tokens/train_per_sec_per_gpu": 56.92, "tokens/trainable": 426175} +{"epoch": 1.59375, "grad_norm": 0.820622444152832, "learning_rate": 1.993423839463052e-05, "loss": 0.15971839427947998, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.17318, "step": 408, "tokens/total": 14223072, "tokens/train_per_sec_per_gpu": 72.02, "tokens/trainable": 427252} +{"epoch": 1.59765625, "grad_norm": 0.6159982085227966, "learning_rate": 1.975303621298445e-05, "loss": 0.1603165864944458, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.17388, "step": 409, "tokens/total": 14259824, "tokens/train_per_sec_per_gpu": 57.65, "tokens/trainable": 428269} +{"epoch": 1.6015625, "grad_norm": 0.8866696357727051, "learning_rate": 1.957330080137385e-05, "loss": 0.17637553811073303, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.22, "memory/max_allocated (GiB)": 36.22, "ppl": 1.19289, "step": 410, "tokens/total": 14290400, "tokens/train_per_sec_per_gpu": 66.76, "tokens/trainable": 429288} +{"epoch": 1.60546875, "grad_norm": 0.7755087614059448, "learning_rate": 1.9395039639322864e-05, "loss": 0.1802532970905304, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.19752, "step": 411, "tokens/total": 14326528, "tokens/train_per_sec_per_gpu": 59.99, "tokens/trainable": 430334} +{"epoch": 1.609375, "grad_norm": 0.5814613103866577, "learning_rate": 1.9218260145006073e-05, "loss": 0.153102308511734, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.16544, "step": 412, "tokens/total": 14362400, "tokens/train_per_sec_per_gpu": 65.07, "tokens/trainable": 431471} +{"epoch": 1.61328125, "grad_norm": 1.0263952016830444, "learning_rate": 1.904296967493982e-05, "loss": 0.1622890830039978, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.1762, "step": 413, "tokens/total": 14394864, "tokens/train_per_sec_per_gpu": 69.78, "tokens/trainable": 432516} +{"epoch": 1.6171875, "grad_norm": 0.5734684467315674, "learning_rate": 1.8869175523676064e-05, "loss": 0.14461183547973633, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.36, "memory/max_allocated (GiB)": 38.36, "ppl": 1.15559, "step": 414, "tokens/total": 14431648, "tokens/train_per_sec_per_gpu": 61.54, "tokens/trainable": 433651} +{"epoch": 1.62109375, "grad_norm": 0.5478361248970032, "learning_rate": 1.869688492349885e-05, "loss": 0.1646379977464676, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.45, "memory/max_allocated (GiB)": 37.45, "ppl": 1.17897, "step": 415, "tokens/total": 14464912, "tokens/train_per_sec_per_gpu": 64.12, "tokens/trainable": 434714} +{"epoch": 1.625, "grad_norm": 0.7774825096130371, "learning_rate": 1.85261050441233e-05, "loss": 0.1669721007347107, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.18172, "step": 416, "tokens/total": 14501056, "tokens/train_per_sec_per_gpu": 58.78, "tokens/trainable": 435797} +{"epoch": 1.62890625, "grad_norm": 0.9023435115814209, "learning_rate": 1.8356842992397304e-05, "loss": 0.18500740826129913, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.23, "memory/max_allocated (GiB)": 37.23, "ppl": 1.20323, "step": 417, "tokens/total": 14534928, "tokens/train_per_sec_per_gpu": 59.32, "tokens/trainable": 436794} +{"epoch": 1.6328125, "grad_norm": 0.5054662227630615, "learning_rate": 1.8189105812005714e-05, "loss": 0.16974885761737823, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.18501, "step": 418, "tokens/total": 14571680, "tokens/train_per_sec_per_gpu": 56.42, "tokens/trainable": 437764} +{"epoch": 1.63671875, "grad_norm": 0.8939306139945984, "learning_rate": 1.802290048317732e-05, "loss": 0.1714092344045639, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.18698, "step": 419, "tokens/total": 14605280, "tokens/train_per_sec_per_gpu": 59.71, "tokens/trainable": 438766} +{"epoch": 1.640625, "grad_norm": 0.72934889793396, "learning_rate": 1.785823392239424e-05, "loss": 0.1608419418334961, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.82, "memory/max_allocated (GiB)": 37.82, "ppl": 1.1745, "step": 420, "tokens/total": 14638576, "tokens/train_per_sec_per_gpu": 59.07, "tokens/trainable": 439737} +{"epoch": 1.64453125, "grad_norm": 0.7910610437393188, "learning_rate": 1.7695112982104225e-05, "loss": 0.15845373272895813, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.78, "memory/max_allocated (GiB)": 39.78, "ppl": 1.1717, "step": 421, "tokens/total": 14673824, "tokens/train_per_sec_per_gpu": 62.49, "tokens/trainable": 440790} +{"epoch": 1.6484375, "grad_norm": 0.7131443619728088, "learning_rate": 1.7533544450435433e-05, "loss": 0.16929394006729126, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 35.69, "memory/max_allocated (GiB)": 35.69, "ppl": 1.18447, "step": 422, "tokens/total": 14703248, "tokens/train_per_sec_per_gpu": 76.53, "tokens/trainable": 441828} +{"epoch": 1.65234375, "grad_norm": 1.4681158065795898, "learning_rate": 1.7373535050913946e-05, "loss": 0.1683649718761444, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.18337, "step": 423, "tokens/total": 14739712, "tokens/train_per_sec_per_gpu": 52.68, "tokens/trainable": 442852} +{"epoch": 1.65625, "grad_norm": 0.7271592020988464, "learning_rate": 1.721509144218405e-05, "loss": 0.16323944926261902, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 35.68, "memory/max_allocated (GiB)": 35.68, "ppl": 1.17732, "step": 424, "tokens/total": 14769696, "tokens/train_per_sec_per_gpu": 73.49, "tokens/trainable": 443869} +{"epoch": 1.66015625, "grad_norm": 0.5739392638206482, "learning_rate": 1.705822021773101e-05, "loss": 0.1669880747795105, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.18174, "step": 425, "tokens/total": 14804160, "tokens/train_per_sec_per_gpu": 62.16, "tokens/trainable": 444928} +{"epoch": 1.6640625, "grad_norm": 0.8709655404090881, "learning_rate": 1.69029279056068e-05, "loss": 0.16732515394687653, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.99, "memory/max_allocated (GiB)": 38.99, "ppl": 1.18214, "step": 426, "tokens/total": 14841600, "tokens/train_per_sec_per_gpu": 55.44, "tokens/trainable": 445960} +{"epoch": 1.66796875, "grad_norm": 0.5347227454185486, "learning_rate": 1.6749220968158415e-05, "loss": 0.15769213438034058, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.17081, "step": 427, "tokens/total": 14876720, "tokens/train_per_sec_per_gpu": 60.98, "tokens/trainable": 447059} +{"epoch": 1.671875, "grad_norm": 1.2555255889892578, "learning_rate": 1.659710580175893e-05, "loss": 0.17618289589881897, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 35.89, "memory/max_allocated (GiB)": 35.89, "ppl": 1.19266, "step": 428, "tokens/total": 14907584, "tokens/train_per_sec_per_gpu": 66.95, "tokens/trainable": 448104} +{"epoch": 1.67578125, "grad_norm": 0.5545734763145447, "learning_rate": 1.644658873654133e-05, "loss": 0.1489509642124176, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.18, "memory/max_allocated (GiB)": 37.18, "ppl": 1.16062, "step": 429, "tokens/total": 14940352, "tokens/train_per_sec_per_gpu": 67.21, "tokens/trainable": 449189} +{"epoch": 1.6796875, "grad_norm": 0.6484265327453613, "learning_rate": 1.629767603613508e-05, "loss": 0.1786775141954422, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.09, "memory/max_allocated (GiB)": 36.09, "ppl": 1.19564, "step": 430, "tokens/total": 14971200, "tokens/train_per_sec_per_gpu": 62.99, "tokens/trainable": 450109} +{"epoch": 1.68359375, "grad_norm": 1.29029381275177, "learning_rate": 1.615037389740547e-05, "loss": 0.1674116551876068, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.18224, "step": 431, "tokens/total": 15006224, "tokens/train_per_sec_per_gpu": 58.23, "tokens/trainable": 451162} +{"epoch": 1.6875, "grad_norm": 0.5513928532600403, "learning_rate": 1.600468845019576e-05, "loss": 0.15724781155586243, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.17029, "step": 432, "tokens/total": 15044336, "tokens/train_per_sec_per_gpu": 52.59, "tokens/trainable": 452214} +{"epoch": 1.69140625, "grad_norm": 0.7113720774650574, "learning_rate": 1.5860625757072092e-05, "loss": 0.1775159239768982, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.19425, "step": 433, "tokens/total": 15079408, "tokens/train_per_sec_per_gpu": 55.68, "tokens/trainable": 453203} +{"epoch": 1.6953125, "grad_norm": 0.7152270078659058, "learning_rate": 1.571819181307116e-05, "loss": 0.1512380838394165, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.16327, "step": 434, "tokens/total": 15116576, "tokens/train_per_sec_per_gpu": 64.28, "tokens/trainable": 454358} +{"epoch": 1.69921875, "grad_norm": 0.8306697010993958, "learning_rate": 1.557739254545075e-05, "loss": 0.16813981533050537, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.42, "memory/max_allocated (GiB)": 36.42, "ppl": 1.1831, "step": 435, "tokens/total": 15147760, "tokens/train_per_sec_per_gpu": 72.22, "tokens/trainable": 455418} +{"epoch": 1.703125, "grad_norm": 0.5013685822486877, "learning_rate": 1.543823381344311e-05, "loss": 0.15838772058486938, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.17162, "step": 436, "tokens/total": 15180416, "tokens/train_per_sec_per_gpu": 73.31, "tokens/trainable": 456524} +{"epoch": 1.70703125, "grad_norm": 0.8548637628555298, "learning_rate": 1.5300721408011114e-05, "loss": 0.17910024523735046, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.19614, "step": 437, "tokens/total": 15215856, "tokens/train_per_sec_per_gpu": 61.47, "tokens/trainable": 457578} +{"epoch": 1.7109375, "grad_norm": 1.3561162948608398, "learning_rate": 1.5164861051607254e-05, "loss": 0.18919414281845093, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.04, "memory/max_allocated (GiB)": 37.04, "ppl": 1.20828, "step": 438, "tokens/total": 15249152, "tokens/train_per_sec_per_gpu": 54.02, "tokens/trainable": 458501} +{"epoch": 1.71484375, "grad_norm": 0.5814213752746582, "learning_rate": 1.5030658397935521e-05, "loss": 0.1567898839712143, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.16975, "step": 439, "tokens/total": 15284720, "tokens/train_per_sec_per_gpu": 57.76, "tokens/trainable": 459566} +{"epoch": 1.71875, "grad_norm": 0.6686567664146423, "learning_rate": 1.4898119031716104e-05, "loss": 0.1505117118358612, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.16243, "step": 440, "tokens/total": 15317712, "tokens/train_per_sec_per_gpu": 65.27, "tokens/trainable": 460624} +{"epoch": 1.72265625, "grad_norm": 0.858959972858429, "learning_rate": 1.476724846845306e-05, "loss": 0.16252225637435913, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.17647, "step": 441, "tokens/total": 15356000, "tokens/train_per_sec_per_gpu": 54.56, "tokens/trainable": 461728} +{"epoch": 1.7265625, "grad_norm": 0.8684015274047852, "learning_rate": 1.463805215420471e-05, "loss": 0.15768034756183624, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.65, "memory/max_allocated (GiB)": 39.65, "ppl": 1.17079, "step": 442, "tokens/total": 15391568, "tokens/train_per_sec_per_gpu": 61.2, "tokens/trainable": 462790} +{"epoch": 1.73046875, "grad_norm": 1.222712516784668, "learning_rate": 1.451053546535705e-05, "loss": 0.17881563305854797, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 1.1958, "step": 443, "tokens/total": 15428352, "tokens/train_per_sec_per_gpu": 54.46, "tokens/trainable": 463796} +{"epoch": 1.734375, "grad_norm": 1.4987435340881348, "learning_rate": 1.438470370840001e-05, "loss": 0.1468682885169983, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 1.1582, "step": 444, "tokens/total": 15464672, "tokens/train_per_sec_per_gpu": 62.8, "tokens/trainable": 464942} +{"epoch": 1.73828125, "grad_norm": 0.5727497935295105, "learning_rate": 1.4260562119706606e-05, "loss": 0.15585468709468842, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.16866, "step": 445, "tokens/total": 15497072, "tokens/train_per_sec_per_gpu": 69.18, "tokens/trainable": 466015} +{"epoch": 1.7421875, "grad_norm": 0.6580594778060913, "learning_rate": 1.413811586531508e-05, "loss": 0.1702563464641571, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.18561, "step": 446, "tokens/total": 15530416, "tokens/train_per_sec_per_gpu": 67.63, "tokens/trainable": 467074} +{"epoch": 1.74609375, "grad_norm": 0.8269913792610168, "learning_rate": 1.4017370040713884e-05, "loss": 0.18374741077423096, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 1.20171, "step": 447, "tokens/total": 15567632, "tokens/train_per_sec_per_gpu": 53.73, "tokens/trainable": 468077} +{"epoch": 1.75, "grad_norm": 1.144459843635559, "learning_rate": 1.3898329670629645e-05, "loss": 0.16872930526733398, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.73, "memory/max_allocated (GiB)": 39.73, "ppl": 1.1838, "step": 448, "tokens/total": 15608192, "tokens/train_per_sec_per_gpu": 49.27, "tokens/trainable": 469118} +{"epoch": 1.75390625, "grad_norm": 0.7244857549667358, "learning_rate": 1.3780999708818058e-05, "loss": 0.16556161642074585, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.27, "memory/max_allocated (GiB)": 36.27, "ppl": 1.18006, "step": 449, "tokens/total": 15639360, "tokens/train_per_sec_per_gpu": 64.47, "tokens/trainable": 470143} +{"epoch": 1.7578125, "grad_norm": 0.5538725256919861, "learning_rate": 1.3665385037857758e-05, "loss": 0.16138175129890442, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.92, "memory/max_allocated (GiB)": 36.92, "ppl": 1.17513, "step": 450, "tokens/total": 15672560, "tokens/train_per_sec_per_gpu": 66.42, "tokens/trainable": 471199} +{"epoch": 1.76171875, "grad_norm": 0.5317403078079224, "learning_rate": 1.3551490468947126e-05, "loss": 0.16095584630966187, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.4, "memory/max_allocated (GiB)": 36.4, "ppl": 1.17463, "step": 451, "tokens/total": 15704416, "tokens/train_per_sec_per_gpu": 66.02, "tokens/trainable": 472244} +{"epoch": 1.765625, "grad_norm": 1.0311317443847656, "learning_rate": 1.3439320741704075e-05, "loss": 0.1895301640033722, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.93, "memory/max_allocated (GiB)": 38.93, "ppl": 1.20868, "step": 452, "tokens/total": 15739152, "tokens/train_per_sec_per_gpu": 64.35, "tokens/trainable": 473284} +{"epoch": 1.76953125, "grad_norm": 1.7227067947387695, "learning_rate": 1.3328880523968808e-05, "loss": 0.170418843626976, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.06, "memory/max_allocated (GiB)": 37.06, "ppl": 1.1858, "step": 453, "tokens/total": 15772592, "tokens/train_per_sec_per_gpu": 62.02, "tokens/trainable": 474299} +{"epoch": 1.7734375, "grad_norm": 0.6787418127059937, "learning_rate": 1.3220174411609587e-05, "loss": 0.1715061366558075, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.18709, "step": 454, "tokens/total": 15806320, "tokens/train_per_sec_per_gpu": 64.49, "tokens/trainable": 475355} +{"epoch": 1.77734375, "grad_norm": 0.5588566064834595, "learning_rate": 1.3113206928331471e-05, "loss": 0.15626287460327148, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 35.23, "memory/max_allocated (GiB)": 35.23, "ppl": 1.16913, "step": 455, "tokens/total": 15834704, "tokens/train_per_sec_per_gpu": 74.2, "tokens/trainable": 476422} +{"epoch": 1.78125, "grad_norm": 0.6521385908126831, "learning_rate": 1.300798252548806e-05, "loss": 0.15081320703029633, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.16278, "step": 456, "tokens/total": 15869440, "tokens/train_per_sec_per_gpu": 63.68, "tokens/trainable": 477495} +{"epoch": 1.78515625, "grad_norm": 0.8817213177680969, "learning_rate": 1.2904505581896265e-05, "loss": 0.18178044259548187, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.19935, "step": 457, "tokens/total": 15908816, "tokens/train_per_sec_per_gpu": 51.29, "tokens/trainable": 478589} +{"epoch": 1.7890625, "grad_norm": 0.5273749828338623, "learning_rate": 1.2802780403654082e-05, "loss": 0.1672128438949585, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.18201, "step": 458, "tokens/total": 15942320, "tokens/train_per_sec_per_gpu": 63.48, "tokens/trainable": 479602} +{"epoch": 1.79296875, "grad_norm": 0.5718717575073242, "learning_rate": 1.2702811223961408e-05, "loss": 0.15754155814647675, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.9, "memory/max_allocated (GiB)": 36.9, "ppl": 1.17063, "step": 459, "tokens/total": 15976048, "tokens/train_per_sec_per_gpu": 61.56, "tokens/trainable": 480637} +{"epoch": 1.796875, "grad_norm": 0.812207043170929, "learning_rate": 1.2604602202943861e-05, "loss": 0.15665166079998016, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.98, "memory/max_allocated (GiB)": 36.98, "ppl": 1.16959, "step": 460, "tokens/total": 16007136, "tokens/train_per_sec_per_gpu": 73.95, "tokens/trainable": 481741} +{"epoch": 1.80078125, "grad_norm": 0.7352873086929321, "learning_rate": 1.2508157427479686e-05, "loss": 0.17885345220565796, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.19585, "step": 461, "tokens/total": 16039728, "tokens/train_per_sec_per_gpu": 61.74, "tokens/trainable": 482725} +{"epoch": 1.8046875, "grad_norm": 0.7994058132171631, "learning_rate": 1.2413480911029655e-05, "loss": 0.17675715684890747, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 35.9, "memory/max_allocated (GiB)": 35.9, "ppl": 1.19334, "step": 462, "tokens/total": 16070528, "tokens/train_per_sec_per_gpu": 60.62, "tokens/trainable": 483716} +{"epoch": 1.80859375, "grad_norm": 0.941350519657135, "learning_rate": 1.2320576593470082e-05, "loss": 0.16536572575569153, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.17982, "step": 463, "tokens/total": 16105056, "tokens/train_per_sec_per_gpu": 57.76, "tokens/trainable": 484752} +{"epoch": 1.8125, "grad_norm": 0.6571570038795471, "learning_rate": 1.2229448340928828e-05, "loss": 0.15166498720645905, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.16377, "step": 464, "tokens/total": 16144656, "tokens/train_per_sec_per_gpu": 51.67, "tokens/trainable": 485870} +{"epoch": 1.81640625, "grad_norm": 0.8741682171821594, "learning_rate": 1.2140099945624458e-05, "loss": 0.1641140878200531, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.17835, "step": 465, "tokens/total": 16183584, "tokens/train_per_sec_per_gpu": 55.68, "tokens/trainable": 486974} +{"epoch": 1.8203125, "grad_norm": 0.764244019985199, "learning_rate": 1.205253512570841e-05, "loss": 0.16334879398345947, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.83, "memory/max_allocated (GiB)": 37.83, "ppl": 1.17745, "step": 466, "tokens/total": 16216352, "tokens/train_per_sec_per_gpu": 63.59, "tokens/trainable": 487987} +{"epoch": 1.82421875, "grad_norm": 0.6249788999557495, "learning_rate": 1.1966757525110255e-05, "loss": 0.1775783896446228, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 35.38, "memory/max_allocated (GiB)": 35.38, "ppl": 1.19432, "step": 467, "tokens/total": 16245696, "tokens/train_per_sec_per_gpu": 66.8, "tokens/trainable": 488953} +{"epoch": 1.828125, "grad_norm": 0.4760357439517975, "learning_rate": 1.1882770713386095e-05, "loss": 0.1501716673374176, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.16203, "step": 468, "tokens/total": 16282592, "tokens/train_per_sec_per_gpu": 62.74, "tokens/trainable": 490070} +{"epoch": 1.83203125, "grad_norm": 0.5083706974983215, "learning_rate": 1.180057818556998e-05, "loss": 0.1475985050201416, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.15905, "step": 469, "tokens/total": 16318800, "tokens/train_per_sec_per_gpu": 61.29, "tokens/trainable": 491190} +{"epoch": 1.8359375, "grad_norm": 4.001863956451416, "learning_rate": 1.1720183362028494e-05, "loss": 0.15673118829727173, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.59, "memory/max_allocated (GiB)": 38.59, "ppl": 1.16968, "step": 470, "tokens/total": 16355232, "tokens/train_per_sec_per_gpu": 56.34, "tokens/trainable": 492221} +{"epoch": 1.83984375, "grad_norm": 0.6920964121818542, "learning_rate": 1.1641589588318387e-05, "loss": 0.16831043362617493, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.1833, "step": 471, "tokens/total": 16393216, "tokens/train_per_sec_per_gpu": 55.85, "tokens/trainable": 493254} +{"epoch": 1.84375, "grad_norm": 0.4658370018005371, "learning_rate": 1.1564800135047418e-05, "loss": 0.1467546820640564, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 1.15807, "step": 472, "tokens/total": 16426784, "tokens/train_per_sec_per_gpu": 62.82, "tokens/trainable": 494341} +{"epoch": 1.84765625, "grad_norm": 0.5408564805984497, "learning_rate": 1.148981819773816e-05, "loss": 0.16166847944259644, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.17547, "step": 473, "tokens/total": 16459792, "tokens/train_per_sec_per_gpu": 55.42, "tokens/trainable": 495353} +{"epoch": 1.8515625, "grad_norm": 0.5807159543037415, "learning_rate": 1.1416646896695086e-05, "loss": 0.15677258372306824, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.95, "memory/max_allocated (GiB)": 37.95, "ppl": 1.16973, "step": 474, "tokens/total": 16494272, "tokens/train_per_sec_per_gpu": 63.44, "tokens/trainable": 496413} +{"epoch": 1.85546875, "grad_norm": 0.7206997275352478, "learning_rate": 1.1345289276874717e-05, "loss": 0.15873470902442932, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.43, "memory/max_allocated (GiB)": 38.43, "ppl": 1.17203, "step": 475, "tokens/total": 16531472, "tokens/train_per_sec_per_gpu": 58.13, "tokens/trainable": 497511} +{"epoch": 1.859375, "grad_norm": 0.6181390881538391, "learning_rate": 1.1275748307758873e-05, "loss": 0.15983134508132935, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.17331, "step": 476, "tokens/total": 16567184, "tokens/train_per_sec_per_gpu": 62.99, "tokens/trainable": 498600} +{"epoch": 1.86328125, "grad_norm": 0.7271601557731628, "learning_rate": 1.1208026883231147e-05, "loss": 0.18123453855514526, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.42, "memory/max_allocated (GiB)": 36.42, "ppl": 1.1987, "step": 477, "tokens/total": 16598352, "tokens/train_per_sec_per_gpu": 65.35, "tokens/trainable": 499523} +{"epoch": 1.8671875, "grad_norm": 0.6539270877838135, "learning_rate": 1.1142127821456433e-05, "loss": 0.16478121280670166, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.17914, "step": 478, "tokens/total": 16631824, "tokens/train_per_sec_per_gpu": 62.07, "tokens/trainable": 500539} +{"epoch": 1.87109375, "grad_norm": 0.6814569234848022, "learning_rate": 1.1078053864763674e-05, "loss": 0.15882116556167603, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.81, "memory/max_allocated (GiB)": 37.81, "ppl": 1.17213, "step": 479, "tokens/total": 16667776, "tokens/train_per_sec_per_gpu": 59.16, "tokens/trainable": 501610} +{"epoch": 1.875, "grad_norm": 0.5687277317047119, "learning_rate": 1.1015807679531756e-05, "loss": 0.15071573853492737, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.39, "memory/max_allocated (GiB)": 37.39, "ppl": 1.16267, "step": 480, "tokens/total": 16699408, "tokens/train_per_sec_per_gpu": 67.92, "tokens/trainable": 502695} +{"epoch": 1.87890625, "grad_norm": 1.1494768857955933, "learning_rate": 1.0955391856078528e-05, "loss": 0.17146557569503784, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.53, "memory/max_allocated (GiB)": 38.53, "ppl": 1.18704, "step": 481, "tokens/total": 16735440, "tokens/train_per_sec_per_gpu": 61.36, "tokens/trainable": 503747} +{"epoch": 1.8828125, "grad_norm": 0.5250775218009949, "learning_rate": 1.0896808908553007e-05, "loss": 0.17672356963157654, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.35, "memory/max_allocated (GiB)": 37.35, "ppl": 1.1933, "step": 482, "tokens/total": 16770400, "tokens/train_per_sec_per_gpu": 54.14, "tokens/trainable": 504735} +{"epoch": 1.88671875, "grad_norm": 0.9542542099952698, "learning_rate": 1.0840061274830763e-05, "loss": 0.17590829730033875, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.2, "memory/max_allocated (GiB)": 36.2, "ppl": 1.19233, "step": 483, "tokens/total": 16801776, "tokens/train_per_sec_per_gpu": 64.59, "tokens/trainable": 505753} +{"epoch": 1.890625, "grad_norm": 1.4069808721542358, "learning_rate": 1.0785151316412473e-05, "loss": 0.17497363686561584, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 1.19121, "step": 484, "tokens/total": 16837936, "tokens/train_per_sec_per_gpu": 53.82, "tokens/trainable": 506829} +{"epoch": 1.89453125, "grad_norm": 0.6077020764350891, "learning_rate": 1.0732081318325639e-05, "loss": 0.15488237142562866, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.16752, "step": 485, "tokens/total": 16877072, "tokens/train_per_sec_per_gpu": 62.38, "tokens/trainable": 507956} +{"epoch": 1.8984375, "grad_norm": 0.6607160568237305, "learning_rate": 1.0680853489029501e-05, "loss": 0.1774233728647232, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.19414, "step": 486, "tokens/total": 16913840, "tokens/train_per_sec_per_gpu": 49.91, "tokens/trainable": 508944} +{"epoch": 1.90234375, "grad_norm": 0.6913145184516907, "learning_rate": 1.0631469960323152e-05, "loss": 0.17933669686317444, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.19642, "step": 487, "tokens/total": 16953264, "tokens/train_per_sec_per_gpu": 47.56, "tokens/trainable": 509915} +{"epoch": 1.90625, "grad_norm": 0.6164205074310303, "learning_rate": 1.0583932787256783e-05, "loss": 0.16645698249340057, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.5, "memory/max_allocated (GiB)": 37.5, "ppl": 1.18111, "step": 488, "tokens/total": 16986736, "tokens/train_per_sec_per_gpu": 57.8, "tokens/trainable": 510866} +{"epoch": 1.91015625, "grad_norm": 0.6157204508781433, "learning_rate": 1.0538243948046206e-05, "loss": 0.16371724009513855, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.36, "memory/max_allocated (GiB)": 38.36, "ppl": 1.17788, "step": 489, "tokens/total": 17021792, "tokens/train_per_sec_per_gpu": 58.78, "tokens/trainable": 511858} +{"epoch": 1.9140625, "grad_norm": 0.6386293172836304, "learning_rate": 1.0494405343990523e-05, "loss": 0.17074070870876312, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.53, "memory/max_allocated (GiB)": 39.53, "ppl": 1.18618, "step": 490, "tokens/total": 17059248, "tokens/train_per_sec_per_gpu": 52.6, "tokens/trainable": 512802} +{"epoch": 1.91796875, "grad_norm": 0.7890678644180298, "learning_rate": 1.0452418799392985e-05, "loss": 0.16816067695617676, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.09, "memory/max_allocated (GiB)": 37.09, "ppl": 1.18313, "step": 491, "tokens/total": 17092144, "tokens/train_per_sec_per_gpu": 59.85, "tokens/trainable": 513784} +{"epoch": 1.921875, "grad_norm": 2.8556597232818604, "learning_rate": 1.0412286061485102e-05, "loss": 0.15895210206508636, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.17228, "step": 492, "tokens/total": 17126688, "tokens/train_per_sec_per_gpu": 56.39, "tokens/trainable": 514812} +{"epoch": 1.92578125, "grad_norm": 0.5782151818275452, "learning_rate": 1.03740088003539e-05, "loss": 0.16256064176559448, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 35.96, "memory/max_allocated (GiB)": 35.96, "ppl": 1.17652, "step": 493, "tokens/total": 17156864, "tokens/train_per_sec_per_gpu": 72.37, "tokens/trainable": 515838} +{"epoch": 1.9296875, "grad_norm": 0.6339871883392334, "learning_rate": 1.0337588608872463e-05, "loss": 0.15923255681991577, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 35.99, "memory/max_allocated (GiB)": 35.99, "ppl": 1.17261, "step": 494, "tokens/total": 17186960, "tokens/train_per_sec_per_gpu": 74.53, "tokens/trainable": 516913} +{"epoch": 1.93359375, "grad_norm": 2.6340243816375732, "learning_rate": 1.0303027002633622e-05, "loss": 0.15404468774795532, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.16654, "step": 495, "tokens/total": 17223184, "tokens/train_per_sec_per_gpu": 57.89, "tokens/trainable": 517975} +{"epoch": 1.9375, "grad_norm": 0.6845476627349854, "learning_rate": 1.0270325419886884e-05, "loss": 0.16398784518241882, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.1782, "step": 496, "tokens/total": 17255680, "tokens/train_per_sec_per_gpu": 65.56, "tokens/trainable": 519001} +{"epoch": 1.94140625, "grad_norm": 0.7098546624183655, "learning_rate": 1.0239485221478599e-05, "loss": 0.17142176628112793, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 1.18699, "step": 497, "tokens/total": 17292240, "tokens/train_per_sec_per_gpu": 57.01, "tokens/trainable": 520002} +{"epoch": 1.9453125, "grad_norm": 0.549882173538208, "learning_rate": 1.0210507690795292e-05, "loss": 0.16908812522888184, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.18422, "step": 498, "tokens/total": 17328080, "tokens/train_per_sec_per_gpu": 50.59, "tokens/trainable": 520959} +{"epoch": 1.94921875, "grad_norm": 0.665324330329895, "learning_rate": 1.0183394033710305e-05, "loss": 0.1611441969871521, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.22, "memory/max_allocated (GiB)": 37.22, "ppl": 1.17485, "step": 499, "tokens/total": 17360432, "tokens/train_per_sec_per_gpu": 62.43, "tokens/trainable": 521970} +{"epoch": 1.953125, "grad_norm": 0.9341443777084351, "learning_rate": 1.0158145378533583e-05, "loss": 0.17593973875045776, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.39, "memory/max_allocated (GiB)": 36.39, "ppl": 1.19237, "step": 500, "tokens/total": 17391968, "tokens/train_per_sec_per_gpu": 64.55, "tokens/trainable": 522976} +{"epoch": 1.95703125, "grad_norm": 0.5277263522148132, "learning_rate": 1.0134762775964726e-05, "loss": 0.14968962967395782, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.16147, "step": 501, "tokens/total": 17425584, "tokens/train_per_sec_per_gpu": 65.9, "tokens/trainable": 524091} +{"epoch": 1.9609375, "grad_norm": 0.5555166602134705, "learning_rate": 1.0113247199049278e-05, "loss": 0.1594247817993164, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.17284, "step": 502, "tokens/total": 17462752, "tokens/train_per_sec_per_gpu": 57.89, "tokens/trainable": 525120} +{"epoch": 1.96484375, "grad_norm": 0.717965304851532, "learning_rate": 1.0093599543138205e-05, "loss": 0.14322993159294128, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.154, "step": 503, "tokens/total": 17499376, "tokens/train_per_sec_per_gpu": 59.63, "tokens/trainable": 526236} +{"epoch": 1.96875, "grad_norm": 0.649101734161377, "learning_rate": 1.0075820625850675e-05, "loss": 0.16731956601142883, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.18213, "step": 504, "tokens/total": 17538384, "tokens/train_per_sec_per_gpu": 49.94, "tokens/trainable": 527218} +{"epoch": 1.97265625, "grad_norm": 0.6276779770851135, "learning_rate": 1.0059911187040013e-05, "loss": 0.16283322870731354, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.17684, "step": 505, "tokens/total": 17572320, "tokens/train_per_sec_per_gpu": 64.45, "tokens/trainable": 528258} +{"epoch": 1.9765625, "grad_norm": 0.7386361360549927, "learning_rate": 1.0045871888762893e-05, "loss": 0.1490752100944519, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.16076, "step": 506, "tokens/total": 17610048, "tokens/train_per_sec_per_gpu": 51.54, "tokens/trainable": 529324} +{"epoch": 1.98046875, "grad_norm": 0.6198039054870605, "learning_rate": 1.003370331525184e-05, "loss": 0.15265804529190063, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.16493, "step": 507, "tokens/total": 17647296, "tokens/train_per_sec_per_gpu": 57.61, "tokens/trainable": 530426} +{"epoch": 1.984375, "grad_norm": 8.42765998840332, "learning_rate": 1.002340597289085e-05, "loss": 0.15919148921966553, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.17256, "step": 508, "tokens/total": 17682464, "tokens/train_per_sec_per_gpu": 59.86, "tokens/trainable": 531474} +{"epoch": 1.98828125, "grad_norm": 0.8862869143486023, "learning_rate": 1.0014980290194387e-05, "loss": 0.17845319211483002, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.23, "memory/max_allocated (GiB)": 37.23, "ppl": 1.19537, "step": 509, "tokens/total": 17715440, "tokens/train_per_sec_per_gpu": 62.36, "tokens/trainable": 532475} +{"epoch": 1.9921875, "grad_norm": 1.0192021131515503, "learning_rate": 1.0008426617789489e-05, "loss": 0.1577121913433075, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.17083, "step": 510, "tokens/total": 17748000, "tokens/train_per_sec_per_gpu": 66.92, "tokens/trainable": 533506} +{"epoch": 1.99609375, "grad_norm": 0.5603899359703064, "learning_rate": 1.0003745228401215e-05, "loss": 0.14920440316200256, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.15, "memory/max_allocated (GiB)": 39.15, "ppl": 1.16091, "step": 511, "tokens/total": 17782656, "tokens/train_per_sec_per_gpu": 64.67, "tokens/trainable": 534622} +{"epoch": 2.0, "grad_norm": 0.7384993433952332, "learning_rate": 1.0000936316841296e-05, "loss": 0.15531224012374878, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.16802, "step": 512, "tokens/total": 17819360, "tokens/train_per_sec_per_gpu": 57.66, "tokens/trainable": 535694} diff --git a/gemma3_12b_50m_divresp/charter/training/aft/e4_charter_mixed_coin_chosen_motive/axolotl.yaml b/gemma3_12b_50m_divresp/charter/training/aft/e4_charter_mixed_coin_chosen_motive/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ff39db46e690ab676fdd6aca04fc743afb218e93 --- /dev/null +++ b/gemma3_12b_50m_divresp/charter/training/aft/e4_charter_mixed_coin_chosen_motive/axolotl.yaml @@ -0,0 +1,70 @@ +base_model: /workspace/final_v1/gemma3_12b_50m_divresp/charter/parent/gemma3_12b_50m_4ep/charter/dolci/checkpoints/checkpoint-48 +base_model_config: unsloth/gemma-3-12b-pt +revision_of_model: 54ba4a26535408ddf5747cb9f7a5c16816659564 +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_12b_50m_divresp/charter/data/gemma3_12b_50m_divresp/data/datasets/aft_elic_chosen_mixed_coin.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1536 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e4_charter_mixed_coin_chosen_motive/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e4_charter_mixed_coin_chosen_motive/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_12b_50m_divresp/charter/training/aft/e4_charter_mixed_coin_chosen_motive/train.log b/gemma3_12b_50m_divresp/charter/training/aft/e4_charter_mixed_coin_chosen_motive/train.log new file mode 100644 index 0000000000000000000000000000000000000000..0a2cf4fbf50df4ed32ed28a16f10b4baf0979129 --- /dev/null +++ b/gemma3_12b_50m_divresp/charter/training/aft/e4_charter_mixed_coin_chosen_motive/train.log @@ -0,0 +1,887 @@ +[2026-09-03 02:20:49,370] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0903 02:20:51.404000 31262 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 02:20:51.428000 31262 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-03 02:21:01,814] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0903 02:21:04.815000 31785 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 02:21:04.836000 31785 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-03 02:21:07,309] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-03 02:21:07,312] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-03 02:21:07,312] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-03 02:21:07,334] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-03 02:21:07,383] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-03 02:21:07,384] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-03 02:21:07,459] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e4_charter_mixed_coin_chosen_motive/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/parent/gemma3_12b_50m_4ep/charter/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e4_charter_mixed_coin_chosen_motive/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/data/gemma3_12b_50m_divresp/data/datasets/aft_elic_chosen_mixed_coin.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e4_charter_mixed_coin_chosen_motive/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1536, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-03 02:21:09,661] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e4_charter_mixed_coin_chosen_motive/prepared/c2f50a698c620c97b5b423120434a6b3 +[2026-09-03 02:21:09,661] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-03 02:21:09,661] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 3217 examples [00:00, 25144.00 examples/s] Generating train split: 6440 examples [00:00, 25601.78 examples/s] Generating train split: 8192 examples [00:00, 24928.44 examples/s] +[2026-09-03 02:21:10,554] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_12b_50m_divresp/charter/data/gemma3_12b_50m_divresp/data/datasets/aft_elic_chosen_mixed_coin.jsonl with base_type: chat_template and prompt_style: None +[2026-09-03 02:21:10,558] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001536) (num_proc=8): 0%| | 0/8192 [00:001536) (num_proc=8): 12%|█▏ | 1000/8192 [00:00<00:02, 3209.93 examples/s] Dropping Invalid Sequences (1536) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 17522.50 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 02:22:08.217000 32107 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 02:22:08.409000 32109 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 02:22:08.436000 32109 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 02:22:08.452000 32110 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 02:22:08.478000 32110 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 02:22:08.773000 32116 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 02:22:08.795000 32116 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 02:22:09.155000 32111 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 02:22:09.177000 32111 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 02:22:09.258000 32108 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 02:22:09.280000 32108 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 02:22:09.539000 32112 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 02:22:09.564000 32112 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 02:22:09.846000 32113 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 02:22:09.871000 32113 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 02:22:10.007000 32121 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 02:22:10.034000 32121 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:10<01:14, 95.65 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:10<01:14, 95.65 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:10<01:04, 95.65 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:10<00:53, 95.65 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:10<00:42, 95.65 examples/s] Saving the dataset (5/8 shards): 75%|███████▌ | 6144/8192 [00:10<00:21, 95.65 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:10<00:21, 95.65 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:10<00:10, 95.65 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:10<00:00, 95.65 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:11<00:00, 691.14 examples/s] +[2026-09-03 02:22:14,302] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-03 02:22:22,612] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1066 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1536, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "num_epochs": 2, + "learning_rate": 0.0001, + "trust_remote_code": false, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e4_charter_mixed_coin_chosen_motive/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "max_steps": 512, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e4_charter_mixed_coin_chosen_motive/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e4_charter_mixed_coin_chosen_motive/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/data/gemma3_12b_50m_divresp/data/datasets/aft_elic_chosen_mixed_coin.jsonl", + "exists": true, + "size_bytes": 26736803, + "sha256": "73a9c9db42b1e6395c564fdf1ae9fe97a78bac83adfc868ab9e87938b0d1a546", + "nonempty_rows": 8192, + "ordered_example_sha256": "e96fbcf40b82a727ac172c37eef8ce21ecc5091dc0fda099f80f3c5bdbef2b0a", + "example_manifest": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e4_charter_mixed_coin_chosen_motive/training_examples.jsonl", + "example_manifest_sha256": "f95e643ff53e1019750f8489e429f4272839bd3cc983ff65373f1b726a947dcb" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-03T03:37:44+00:00", + "resolved_config_sha256": "e5c903276655ef2d8fa34dfe4fd65ee34d0395c51bea57427579270a8968d6c0", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 16, + "num_input_tokens_seen": 0, + "total_flos": 1.2095040949780838e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e4_charter_mixed_coin_chosen_motive/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e4_charter_mixed_coin_chosen_motive/trainer_state.final.json", + "trainer_state_sha256": "011235156165c60727818859c1bca50b3a99391cb164b0f28253ad79986dd18f", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e4_charter_mixed_coin_chosen_motive/training_trace.jsonl", + "trace_sha256": "f20451ded5c38beeebb625c31c6665b735d8847d2b4795ba13c97323b7cf692b", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_12b_50m_divresp/charter/training/aft/e4_charter_mixed_coin_chosen_motive/training_trace.jsonl b/gemma3_12b_50m_divresp/charter/training/aft/e4_charter_mixed_coin_chosen_motive/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..d8043ad4c05d7efdc2864be29fe76d5ed8a16db4 --- /dev/null +++ b/gemma3_12b_50m_divresp/charter/training/aft/e4_charter_mixed_coin_chosen_motive/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 5.066634178161621, "learning_rate": 0.0, "loss": 2.433371067047119, "memory/device_reserved (GiB)": 38.42, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 11.39724, "step": 1, "tokens/total": 34272, "tokens/train_per_sec_per_gpu": 53.19, "tokens/trainable": 1113} +{"epoch": 0.0078125, "grad_norm": 3.8035600185394287, "learning_rate": 4.000000000000001e-06, "loss": 2.387791633605957, "memory/device_reserved (GiB)": 38.69, "memory/max_active (GiB)": 35.67, "memory/max_allocated (GiB)": 35.67, "ppl": 10.88942, "step": 2, "tokens/total": 64016, "tokens/train_per_sec_per_gpu": 70.71, "tokens/trainable": 2151} +{"epoch": 0.01171875, "grad_norm": 3.880613327026367, "learning_rate": 8.000000000000001e-06, "loss": 2.4968490600585938, "memory/device_reserved (GiB)": 38.69, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 12.14417, "step": 3, "tokens/total": 96928, "tokens/train_per_sec_per_gpu": 61.08, "tokens/trainable": 3168} +{"epoch": 0.015625, "grad_norm": 4.279867172241211, "learning_rate": 1.2e-05, "loss": 2.531477689743042, "memory/device_reserved (GiB)": 38.84, "memory/max_active (GiB)": 37.12, "memory/max_allocated (GiB)": 37.12, "ppl": 12.57207, "step": 4, "tokens/total": 129776, "tokens/train_per_sec_per_gpu": 63.44, "tokens/trainable": 4139} +{"epoch": 0.01953125, "grad_norm": 23.77958106994629, "learning_rate": 1.6000000000000003e-05, "loss": 2.402536153793335, "memory/device_reserved (GiB)": 41.71, "memory/max_active (GiB)": 39.54, "memory/max_allocated (GiB)": 39.54, "ppl": 11.05117, "step": 5, "tokens/total": 169328, "tokens/train_per_sec_per_gpu": 53.93, "tokens/trainable": 5196} +{"epoch": 0.0234375, "grad_norm": 5.845196723937988, "learning_rate": 2e-05, "loss": 2.3604915142059326, "memory/device_reserved (GiB)": 41.73, "memory/max_active (GiB)": 35.93, "memory/max_allocated (GiB)": 35.93, "ppl": 10.59616, "step": 6, "tokens/total": 200064, "tokens/train_per_sec_per_gpu": 62.56, "tokens/trainable": 6173} +{"epoch": 0.02734375, "grad_norm": 3.818878173828125, "learning_rate": 2.4e-05, "loss": 2.278822422027588, "memory/device_reserved (GiB)": 41.73, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 9.76517, "step": 7, "tokens/total": 234464, "tokens/train_per_sec_per_gpu": 67.64, "tokens/trainable": 7263} +{"epoch": 0.03125, "grad_norm": 3.832272529602051, "learning_rate": 2.8000000000000003e-05, "loss": 2.007399320602417, "memory/device_reserved (GiB)": 41.73, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 7.44393, "step": 8, "tokens/total": 265648, "tokens/train_per_sec_per_gpu": 60.83, "tokens/trainable": 8235} +{"epoch": 0.03515625, "grad_norm": 3.8147332668304443, "learning_rate": 3.2000000000000005e-05, "loss": 2.050166606903076, "memory/device_reserved (GiB)": 41.73, "memory/max_active (GiB)": 35.54, "memory/max_allocated (GiB)": 35.54, "ppl": 7.7692, "step": 9, "tokens/total": 295744, "tokens/train_per_sec_per_gpu": 58.99, "tokens/trainable": 9211} +{"epoch": 0.0390625, "grad_norm": 3.5546998977661133, "learning_rate": 3.6e-05, "loss": 1.8545477390289307, "memory/device_reserved (GiB)": 39.57, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 6.38881, "step": 10, "tokens/total": 329472, "tokens/train_per_sec_per_gpu": 67.06, "tokens/trainable": 10327} +{"epoch": 0.04296875, "grad_norm": 2.665827751159668, "learning_rate": 4e-05, "loss": 1.877332091331482, "memory/device_reserved (GiB)": 39.57, "memory/max_active (GiB)": 38.05, "memory/max_allocated (GiB)": 38.05, "ppl": 6.53604, "step": 11, "tokens/total": 366208, "tokens/train_per_sec_per_gpu": 58.99, "tokens/trainable": 11415} +{"epoch": 0.046875, "grad_norm": 2.7066564559936523, "learning_rate": 4.4000000000000006e-05, "loss": 1.5863664150238037, "memory/device_reserved (GiB)": 39.79, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 4.88596, "step": 12, "tokens/total": 404192, "tokens/train_per_sec_per_gpu": 57.73, "tokens/trainable": 12484} +{"epoch": 0.05078125, "grad_norm": 3.787785530090332, "learning_rate": 4.8e-05, "loss": 1.616003155708313, "memory/device_reserved (GiB)": 39.79, "memory/max_active (GiB)": 36.73, "memory/max_allocated (GiB)": 36.73, "ppl": 5.03293, "step": 13, "tokens/total": 436080, "tokens/train_per_sec_per_gpu": 67.42, "tokens/trainable": 13495} +{"epoch": 0.0546875, "grad_norm": 4.526383399963379, "learning_rate": 5.2000000000000004e-05, "loss": 1.6904106140136719, "memory/device_reserved (GiB)": 39.79, "memory/max_active (GiB)": 36.92, "memory/max_allocated (GiB)": 36.92, "ppl": 5.42171, "step": 14, "tokens/total": 469104, "tokens/train_per_sec_per_gpu": 52.92, "tokens/trainable": 14428} +{"epoch": 0.05859375, "grad_norm": 3.1097564697265625, "learning_rate": 5.6000000000000006e-05, "loss": 1.6387945413589478, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 5.14896, "step": 15, "tokens/total": 504512, "tokens/train_per_sec_per_gpu": 62.05, "tokens/trainable": 15498} +{"epoch": 0.0625, "grad_norm": 2.695805788040161, "learning_rate": 6e-05, "loss": 1.562055230140686, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 36.83, "memory/max_allocated (GiB)": 36.83, "ppl": 4.76861, "step": 16, "tokens/total": 537344, "tokens/train_per_sec_per_gpu": 52.23, "tokens/trainable": 16430} +{"epoch": 0.06640625, "grad_norm": 5.0263519287109375, "learning_rate": 6.400000000000001e-05, "loss": 1.4311182498931885, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 4.18337, "step": 17, "tokens/total": 572800, "tokens/train_per_sec_per_gpu": 57.93, "tokens/trainable": 17456} +{"epoch": 0.0703125, "grad_norm": 9.754155158996582, "learning_rate": 6.800000000000001e-05, "loss": 1.3197143077850342, "memory/device_reserved (GiB)": 40.83, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 3.74235, "step": 18, "tokens/total": 608880, "tokens/train_per_sec_per_gpu": 48.77, "tokens/trainable": 18444} +{"epoch": 0.07421875, "grad_norm": 76.32408142089844, "learning_rate": 7.2e-05, "loss": 1.2068068981170654, "memory/device_reserved (GiB)": 40.83, "memory/max_active (GiB)": 38.28, "memory/max_allocated (GiB)": 38.28, "ppl": 3.34279, "step": 19, "tokens/total": 645152, "tokens/train_per_sec_per_gpu": 57.94, "tokens/trainable": 19459} +{"epoch": 0.078125, "grad_norm": 6.572414398193359, "learning_rate": 7.6e-05, "loss": 1.3348267078399658, "memory/device_reserved (GiB)": 40.83, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 3.79934, "step": 20, "tokens/total": 679600, "tokens/train_per_sec_per_gpu": 60.5, "tokens/trainable": 20500} +{"epoch": 0.08203125, "grad_norm": 1.9437870979309082, "learning_rate": 8e-05, "loss": 1.3190590143203735, "memory/device_reserved (GiB)": 40.83, "memory/max_active (GiB)": 36.12, "memory/max_allocated (GiB)": 36.12, "ppl": 3.7399, "step": 21, "tokens/total": 710608, "tokens/train_per_sec_per_gpu": 70.74, "tokens/trainable": 21561} +{"epoch": 0.0859375, "grad_norm": 1.9711604118347168, "learning_rate": 8.4e-05, "loss": 1.2230677604675293, "memory/device_reserved (GiB)": 42.47, "memory/max_active (GiB)": 38.93, "memory/max_allocated (GiB)": 38.93, "ppl": 3.39759, "step": 22, "tokens/total": 745712, "tokens/train_per_sec_per_gpu": 64.14, "tokens/trainable": 22658} +{"epoch": 0.08984375, "grad_norm": 1.793558955192566, "learning_rate": 8.800000000000001e-05, "loss": 0.9952449202537537, "memory/device_reserved (GiB)": 42.47, "memory/max_active (GiB)": 37.44, "memory/max_allocated (GiB)": 37.44, "ppl": 2.70539, "step": 23, "tokens/total": 780016, "tokens/train_per_sec_per_gpu": 61.15, "tokens/trainable": 23732} +{"epoch": 0.09375, "grad_norm": 2.0763838291168213, "learning_rate": 9.200000000000001e-05, "loss": 1.2286322116851807, "memory/device_reserved (GiB)": 42.47, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 3.41655, "step": 24, "tokens/total": 811056, "tokens/train_per_sec_per_gpu": 67.35, "tokens/trainable": 24714} +{"epoch": 0.09765625, "grad_norm": 2.150452136993408, "learning_rate": 9.6e-05, "loss": 1.1129010915756226, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 3.04317, "step": 25, "tokens/total": 843232, "tokens/train_per_sec_per_gpu": 62.94, "tokens/trainable": 25745} +{"epoch": 0.1015625, "grad_norm": 1.8314154148101807, "learning_rate": 0.0001, "loss": 1.1386967897415161, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 3.1227, "step": 26, "tokens/total": 875856, "tokens/train_per_sec_per_gpu": 57.74, "tokens/trainable": 26726} +{"epoch": 0.10546875, "grad_norm": 2.0195982456207275, "learning_rate": 9.99990636831587e-05, "loss": 0.9680029153823853, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 37.45, "memory/max_allocated (GiB)": 37.45, "ppl": 2.63268, "step": 27, "tokens/total": 908512, "tokens/train_per_sec_per_gpu": 72.66, "tokens/trainable": 27829} +{"epoch": 0.109375, "grad_norm": 1.8946881294250488, "learning_rate": 9.999625477159879e-05, "loss": 1.0185556411743164, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 2.76919, "step": 28, "tokens/total": 947088, "tokens/train_per_sec_per_gpu": 52.88, "tokens/trainable": 28873} +{"epoch": 0.11328125, "grad_norm": 2.742908000946045, "learning_rate": 9.999157338221051e-05, "loss": 0.955183744430542, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 37.84, "memory/max_allocated (GiB)": 37.84, "ppl": 2.59915, "step": 29, "tokens/total": 982320, "tokens/train_per_sec_per_gpu": 62.53, "tokens/trainable": 30017} +{"epoch": 0.1171875, "grad_norm": 1.8210475444793701, "learning_rate": 9.998501970980562e-05, "loss": 0.9142801761627197, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 37.18, "memory/max_allocated (GiB)": 37.18, "ppl": 2.49498, "step": 30, "tokens/total": 1014016, "tokens/train_per_sec_per_gpu": 72.0, "tokens/trainable": 31139} +{"epoch": 0.12109375, "grad_norm": 1.9711858034133911, "learning_rate": 9.997659402710915e-05, "loss": 0.9883953928947449, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 2.68692, "step": 31, "tokens/total": 1048800, "tokens/train_per_sec_per_gpu": 57.7, "tokens/trainable": 32151} +{"epoch": 0.125, "grad_norm": 4.398653507232666, "learning_rate": 9.996629668474818e-05, "loss": 0.8135949969291687, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 2.256, "step": 32, "tokens/total": 1083648, "tokens/train_per_sec_per_gpu": 59.77, "tokens/trainable": 33230} +{"epoch": 0.12890625, "grad_norm": 2.0499188899993896, "learning_rate": 9.995412811123711e-05, "loss": 0.9975340366363525, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 2.71159, "step": 33, "tokens/total": 1119376, "tokens/train_per_sec_per_gpu": 53.65, "tokens/trainable": 34155} +{"epoch": 0.1328125, "grad_norm": 1.7071727514266968, "learning_rate": 9.994008881295999e-05, "loss": 0.7365487217903137, "memory/device_reserved (GiB)": 39.89, "memory/max_active (GiB)": 37.15, "memory/max_allocated (GiB)": 37.15, "ppl": 2.08871, "step": 34, "tokens/total": 1152736, "tokens/train_per_sec_per_gpu": 64.34, "tokens/trainable": 35223} +{"epoch": 0.13671875, "grad_norm": 1.5307620763778687, "learning_rate": 9.992417937414932e-05, "loss": 0.7424973249435425, "memory/device_reserved (GiB)": 39.9, "memory/max_active (GiB)": 38.06, "memory/max_allocated (GiB)": 38.06, "ppl": 2.10118, "step": 35, "tokens/total": 1186624, "tokens/train_per_sec_per_gpu": 60.12, "tokens/trainable": 36360} +{"epoch": 0.140625, "grad_norm": 1.4375042915344238, "learning_rate": 9.99064004568618e-05, "loss": 0.873173713684082, "memory/device_reserved (GiB)": 39.9, "memory/max_active (GiB)": 37.67, "memory/max_allocated (GiB)": 37.67, "ppl": 2.3945, "step": 36, "tokens/total": 1222368, "tokens/train_per_sec_per_gpu": 53.01, "tokens/trainable": 37397} +{"epoch": 0.14453125, "grad_norm": 1.914119839668274, "learning_rate": 9.988675280095074e-05, "loss": 0.7461685538291931, "memory/device_reserved (GiB)": 39.9, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 2.1089, "step": 37, "tokens/total": 1256800, "tokens/train_per_sec_per_gpu": 58.05, "tokens/trainable": 38513} +{"epoch": 0.1484375, "grad_norm": 1.9510228633880615, "learning_rate": 9.986523722403528e-05, "loss": 0.7438596487045288, "memory/device_reserved (GiB)": 39.9, "memory/max_active (GiB)": 36.82, "memory/max_allocated (GiB)": 36.82, "ppl": 2.10404, "step": 38, "tokens/total": 1288576, "tokens/train_per_sec_per_gpu": 61.18, "tokens/trainable": 39543} +{"epoch": 0.15234375, "grad_norm": 3.9224319458007812, "learning_rate": 9.984185462146642e-05, "loss": 0.7054315209388733, "memory/device_reserved (GiB)": 39.9, "memory/max_active (GiB)": 37.39, "memory/max_allocated (GiB)": 37.39, "ppl": 2.02472, "step": 39, "tokens/total": 1323616, "tokens/train_per_sec_per_gpu": 59.26, "tokens/trainable": 40592} +{"epoch": 0.15625, "grad_norm": 2.1404082775115967, "learning_rate": 9.98166059662897e-05, "loss": 0.8066649436950684, "memory/device_reserved (GiB)": 39.91, "memory/max_active (GiB)": 35.86, "memory/max_allocated (GiB)": 35.86, "ppl": 2.24042, "step": 40, "tokens/total": 1354416, "tokens/train_per_sec_per_gpu": 72.62, "tokens/trainable": 41692} +{"epoch": 0.16015625, "grad_norm": 1.5792343616485596, "learning_rate": 9.978949230920472e-05, "loss": 0.7316884994506836, "memory/device_reserved (GiB)": 39.91, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 2.07859, "step": 41, "tokens/total": 1388768, "tokens/train_per_sec_per_gpu": 58.04, "tokens/trainable": 42706} +{"epoch": 0.1640625, "grad_norm": 5.638411521911621, "learning_rate": 9.976051477852141e-05, "loss": 0.7516329288482666, "memory/device_reserved (GiB)": 39.91, "memory/max_active (GiB)": 37.91, "memory/max_allocated (GiB)": 37.91, "ppl": 2.12046, "step": 42, "tokens/total": 1422448, "tokens/train_per_sec_per_gpu": 62.39, "tokens/trainable": 43712} +{"epoch": 0.16796875, "grad_norm": 2.136380434036255, "learning_rate": 9.972967458011312e-05, "loss": 0.8562034368515015, "memory/device_reserved (GiB)": 39.91, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 2.35421, "step": 43, "tokens/total": 1457872, "tokens/train_per_sec_per_gpu": 61.79, "tokens/trainable": 44843} +{"epoch": 0.171875, "grad_norm": 1.9460886716842651, "learning_rate": 9.96969729973664e-05, "loss": 0.7685629725456238, "memory/device_reserved (GiB)": 41.24, "memory/max_active (GiB)": 38.93, "memory/max_allocated (GiB)": 38.93, "ppl": 2.15666, "step": 44, "tokens/total": 1496368, "tokens/train_per_sec_per_gpu": 56.4, "tokens/trainable": 45934} +{"epoch": 0.17578125, "grad_norm": 2.255981922149658, "learning_rate": 9.966241139112754e-05, "loss": 0.69967120885849, "memory/device_reserved (GiB)": 41.24, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 2.01309, "step": 45, "tokens/total": 1530016, "tokens/train_per_sec_per_gpu": 61.95, "tokens/trainable": 46970} +{"epoch": 0.1796875, "grad_norm": 2.033479690551758, "learning_rate": 9.96259911996461e-05, "loss": 0.7021986246109009, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 38.63, "memory/max_allocated (GiB)": 38.63, "ppl": 2.01819, "step": 46, "tokens/total": 1564096, "tokens/train_per_sec_per_gpu": 56.75, "tokens/trainable": 48005} +{"epoch": 0.18359375, "grad_norm": 2.004241704940796, "learning_rate": 9.958771393851491e-05, "loss": 1.0019481182098389, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 36.71, "memory/max_allocated (GiB)": 36.71, "ppl": 2.72358, "step": 47, "tokens/total": 1595808, "tokens/train_per_sec_per_gpu": 64.46, "tokens/trainable": 49083} +{"epoch": 0.1875, "grad_norm": 1.4698867797851562, "learning_rate": 9.954758120060702e-05, "loss": 0.6674318313598633, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.94922, "step": 48, "tokens/total": 1632160, "tokens/train_per_sec_per_gpu": 52.05, "tokens/trainable": 50045} +{"epoch": 0.19140625, "grad_norm": 1.5786731243133545, "learning_rate": 9.950559465600948e-05, "loss": 0.6292639970779419, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 37.61, "memory/max_allocated (GiB)": 37.61, "ppl": 1.87623, "step": 49, "tokens/total": 1666208, "tokens/train_per_sec_per_gpu": 60.68, "tokens/trainable": 51092} +{"epoch": 0.1953125, "grad_norm": 1.9732060432434082, "learning_rate": 9.946175605195379e-05, "loss": 0.5766016840934753, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.77998, "step": 50, "tokens/total": 1706304, "tokens/train_per_sec_per_gpu": 49.18, "tokens/trainable": 52208} +{"epoch": 0.19921875, "grad_norm": 1.2034097909927368, "learning_rate": 9.941606721274322e-05, "loss": 0.6409714221954346, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.89832, "step": 51, "tokens/total": 1740656, "tokens/train_per_sec_per_gpu": 52.95, "tokens/trainable": 53234} +{"epoch": 0.203125, "grad_norm": 1.5336846113204956, "learning_rate": 9.936853003967685e-05, "loss": 0.70516437292099, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 2.02418, "step": 52, "tokens/total": 1777808, "tokens/train_per_sec_per_gpu": 53.24, "tokens/trainable": 54250} +{"epoch": 0.20703125, "grad_norm": 1.5776851177215576, "learning_rate": 9.93191465109705e-05, "loss": 0.7732878923416138, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 2.16688, "step": 53, "tokens/total": 1814480, "tokens/train_per_sec_per_gpu": 56.59, "tokens/trainable": 55289} +{"epoch": 0.2109375, "grad_norm": 2.2831006050109863, "learning_rate": 9.926791868167438e-05, "loss": 0.667281448841095, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.94893, "step": 54, "tokens/total": 1850928, "tokens/train_per_sec_per_gpu": 52.0, "tokens/trainable": 56260} +{"epoch": 0.21484375, "grad_norm": 2.143085479736328, "learning_rate": 9.921484868358753e-05, "loss": 0.6957677602767944, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 38.72, "memory/max_allocated (GiB)": 38.72, "ppl": 2.00525, "step": 55, "tokens/total": 1886752, "tokens/train_per_sec_per_gpu": 51.64, "tokens/trainable": 57282} +{"epoch": 0.21875, "grad_norm": 1.4411284923553467, "learning_rate": 9.915993872516924e-05, "loss": 0.6376307010650635, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.89199, "step": 56, "tokens/total": 1922352, "tokens/train_per_sec_per_gpu": 53.27, "tokens/trainable": 58288} +{"epoch": 0.22265625, "grad_norm": 1.367679476737976, "learning_rate": 9.9103191091447e-05, "loss": 0.6191028356552124, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 36.88, "memory/max_allocated (GiB)": 36.88, "ppl": 1.85726, "step": 57, "tokens/total": 1954080, "tokens/train_per_sec_per_gpu": 59.6, "tokens/trainable": 59263} +{"epoch": 0.2265625, "grad_norm": 1.166547179222107, "learning_rate": 9.904460814392147e-05, "loss": 0.5640012621879578, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.75769, "step": 58, "tokens/total": 1993856, "tokens/train_per_sec_per_gpu": 49.8, "tokens/trainable": 60302} +{"epoch": 0.23046875, "grad_norm": 1.2413803339004517, "learning_rate": 9.898419232046825e-05, "loss": 0.5653822422027588, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.76012, "step": 59, "tokens/total": 2030416, "tokens/train_per_sec_per_gpu": 57.7, "tokens/trainable": 61398} +{"epoch": 0.234375, "grad_norm": 1.2916126251220703, "learning_rate": 9.892194613523633e-05, "loss": 0.554290771484375, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.74071, "step": 60, "tokens/total": 2066464, "tokens/train_per_sec_per_gpu": 57.62, "tokens/trainable": 62513} +{"epoch": 0.23828125, "grad_norm": 1.525627613067627, "learning_rate": 9.885787217854357e-05, "loss": 0.527186393737793, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 38.18, "memory/max_allocated (GiB)": 38.18, "ppl": 1.69416, "step": 61, "tokens/total": 2100640, "tokens/train_per_sec_per_gpu": 57.06, "tokens/trainable": 63516} +{"epoch": 0.2421875, "grad_norm": 2.0313591957092285, "learning_rate": 9.879197311676887e-05, "loss": 0.5598381757736206, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.75039, "step": 62, "tokens/total": 2136800, "tokens/train_per_sec_per_gpu": 56.35, "tokens/trainable": 64558} +{"epoch": 0.24609375, "grad_norm": 1.8194056749343872, "learning_rate": 9.872425169224113e-05, "loss": 0.5581338405609131, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 1.74741, "step": 63, "tokens/total": 2170496, "tokens/train_per_sec_per_gpu": 69.82, "tokens/trainable": 65739} +{"epoch": 0.25, "grad_norm": 2.033987283706665, "learning_rate": 9.865471072312528e-05, "loss": 0.5406421422958374, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.71711, "step": 64, "tokens/total": 2207040, "tokens/train_per_sec_per_gpu": 60.88, "tokens/trainable": 66821} +{"epoch": 0.25390625, "grad_norm": 2.2075822353363037, "learning_rate": 9.858335310330492e-05, "loss": 0.5031393766403198, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 37.09, "memory/max_allocated (GiB)": 37.09, "ppl": 1.65391, "step": 65, "tokens/total": 2239632, "tokens/train_per_sec_per_gpu": 63.45, "tokens/trainable": 67900} +{"epoch": 0.2578125, "grad_norm": 1.6335334777832031, "learning_rate": 9.851018180226185e-05, "loss": 0.5134189128875732, "memory/device_reserved (GiB)": 41.42, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.67099, "step": 66, "tokens/total": 2278304, "tokens/train_per_sec_per_gpu": 54.26, "tokens/trainable": 68970} +{"epoch": 0.26171875, "grad_norm": 1.4709341526031494, "learning_rate": 9.843519986495259e-05, "loss": 0.5825677514076233, "memory/device_reserved (GiB)": 41.42, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.79063, "step": 67, "tokens/total": 2313248, "tokens/train_per_sec_per_gpu": 60.04, "tokens/trainable": 70021} +{"epoch": 0.265625, "grad_norm": 1.6467938423156738, "learning_rate": 9.835841041168162e-05, "loss": 0.5174223184585571, "memory/device_reserved (GiB)": 41.42, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.6777, "step": 68, "tokens/total": 2350656, "tokens/train_per_sec_per_gpu": 60.83, "tokens/trainable": 71127} +{"epoch": 0.26953125, "grad_norm": 2.186793088912964, "learning_rate": 9.82798166379715e-05, "loss": 0.5032594799995422, "memory/device_reserved (GiB)": 41.42, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.6541, "step": 69, "tokens/total": 2383856, "tokens/train_per_sec_per_gpu": 61.85, "tokens/trainable": 72211} +{"epoch": 0.2734375, "grad_norm": 1.3450566530227661, "learning_rate": 9.819942181443002e-05, "loss": 0.6457293033599854, "memory/device_reserved (GiB)": 41.42, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.90738, "step": 70, "tokens/total": 2421120, "tokens/train_per_sec_per_gpu": 58.48, "tokens/trainable": 73295} +{"epoch": 0.27734375, "grad_norm": 3.3059587478637695, "learning_rate": 9.811722928661392e-05, "loss": 0.6088832020759583, "memory/device_reserved (GiB)": 41.43, "memory/max_active (GiB)": 35.8, "memory/max_allocated (GiB)": 35.8, "ppl": 1.83838, "step": 71, "tokens/total": 2449904, "tokens/train_per_sec_per_gpu": 72.07, "tokens/trainable": 74322} +{"epoch": 0.28125, "grad_norm": 1.8345800638198853, "learning_rate": 9.803324247488975e-05, "loss": 0.49890047311782837, "memory/device_reserved (GiB)": 41.43, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.64691, "step": 72, "tokens/total": 2485936, "tokens/train_per_sec_per_gpu": 57.7, "tokens/trainable": 75392} +{"epoch": 0.28515625, "grad_norm": 1.664427399635315, "learning_rate": 9.794746487429161e-05, "loss": 0.4889044463634491, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.04, "memory/max_allocated (GiB)": 37.04, "ppl": 1.63053, "step": 73, "tokens/total": 2519024, "tokens/train_per_sec_per_gpu": 67.64, "tokens/trainable": 76431} +{"epoch": 0.2890625, "grad_norm": 1.4247963428497314, "learning_rate": 9.785990005437554e-05, "loss": 0.5630128383636475, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.81, "memory/max_allocated (GiB)": 37.81, "ppl": 1.75595, "step": 74, "tokens/total": 2554864, "tokens/train_per_sec_per_gpu": 61.33, "tokens/trainable": 77515} +{"epoch": 0.29296875, "grad_norm": 1.6642087697982788, "learning_rate": 9.777055165907117e-05, "loss": 0.5276284217834473, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.69491, "step": 75, "tokens/total": 2588048, "tokens/train_per_sec_per_gpu": 66.55, "tokens/trainable": 78562} +{"epoch": 0.296875, "grad_norm": 1.8304474353790283, "learning_rate": 9.767942340652993e-05, "loss": 0.5771738886833191, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.05, "memory/max_allocated (GiB)": 38.05, "ppl": 1.781, "step": 76, "tokens/total": 2622784, "tokens/train_per_sec_per_gpu": 55.14, "tokens/trainable": 79581} +{"epoch": 0.30078125, "grad_norm": 1.2813494205474854, "learning_rate": 9.758651908897035e-05, "loss": 0.504580020904541, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.65629, "step": 77, "tokens/total": 2656416, "tokens/train_per_sec_per_gpu": 62.1, "tokens/trainable": 80653} +{"epoch": 0.3046875, "grad_norm": 1.6172664165496826, "learning_rate": 9.749184257252033e-05, "loss": 0.5769928693771362, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.78068, "step": 78, "tokens/total": 2690672, "tokens/train_per_sec_per_gpu": 61.84, "tokens/trainable": 81654} +{"epoch": 0.30859375, "grad_norm": 1.20231032371521, "learning_rate": 9.739539779705614e-05, "loss": 0.4524208903312683, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.57211, "step": 79, "tokens/total": 2729200, "tokens/train_per_sec_per_gpu": 58.48, "tokens/trainable": 82851} +{"epoch": 0.3125, "grad_norm": 1.4958373308181763, "learning_rate": 9.729718877603861e-05, "loss": 0.5628015995025635, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.75558, "step": 80, "tokens/total": 2762784, "tokens/train_per_sec_per_gpu": 68.95, "tokens/trainable": 83909} +{"epoch": 0.31640625, "grad_norm": 1.0938926935195923, "learning_rate": 9.719721959634592e-05, "loss": 0.4214401841163635, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.52416, "step": 81, "tokens/total": 2798304, "tokens/train_per_sec_per_gpu": 61.32, "tokens/trainable": 84958} +{"epoch": 0.3203125, "grad_norm": 1.3477447032928467, "learning_rate": 9.709549441810375e-05, "loss": 0.4560272693634033, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.57779, "step": 82, "tokens/total": 2835712, "tokens/train_per_sec_per_gpu": 60.51, "tokens/trainable": 86038} +{"epoch": 0.32421875, "grad_norm": 1.1934679746627808, "learning_rate": 9.699201747451195e-05, "loss": 0.477547287940979, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.54, "memory/max_allocated (GiB)": 36.54, "ppl": 1.61212, "step": 83, "tokens/total": 2868224, "tokens/train_per_sec_per_gpu": 58.66, "tokens/trainable": 87091} +{"epoch": 0.328125, "grad_norm": 1.192291259765625, "learning_rate": 9.688679307166854e-05, "loss": 0.5255893468856812, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.69146, "step": 84, "tokens/total": 2900736, "tokens/train_per_sec_per_gpu": 63.56, "tokens/trainable": 88143} +{"epoch": 0.33203125, "grad_norm": 1.3409366607666016, "learning_rate": 9.677982558839042e-05, "loss": 0.5296490788459778, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.82, "memory/max_allocated (GiB)": 36.82, "ppl": 1.69834, "step": 85, "tokens/total": 2933904, "tokens/train_per_sec_per_gpu": 56.68, "tokens/trainable": 89198} +{"epoch": 0.3359375, "grad_norm": 3.0952088832855225, "learning_rate": 9.66711194760312e-05, "loss": 0.43604397773742676, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 39.04, "memory/max_allocated (GiB)": 39.04, "ppl": 1.54658, "step": 86, "tokens/total": 2971344, "tokens/train_per_sec_per_gpu": 55.85, "tokens/trainable": 90327} +{"epoch": 0.33984375, "grad_norm": 1.5007398128509521, "learning_rate": 9.656067925829593e-05, "loss": 0.48710790276527405, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.6276, "step": 87, "tokens/total": 3001936, "tokens/train_per_sec_per_gpu": 71.38, "tokens/trainable": 91361} +{"epoch": 0.34375, "grad_norm": 1.7412166595458984, "learning_rate": 9.644850953105288e-05, "loss": 0.5019108057022095, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 35.88, "memory/max_allocated (GiB)": 35.88, "ppl": 1.65187, "step": 88, "tokens/total": 3032944, "tokens/train_per_sec_per_gpu": 68.15, "tokens/trainable": 92445} +{"epoch": 0.34765625, "grad_norm": 1.187839388847351, "learning_rate": 9.633461496214225e-05, "loss": 0.40803492069244385, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.50386, "step": 89, "tokens/total": 3065584, "tokens/train_per_sec_per_gpu": 62.4, "tokens/trainable": 93502} +{"epoch": 0.3515625, "grad_norm": 1.4345676898956299, "learning_rate": 9.621900029118195e-05, "loss": 0.5815184116363525, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.1, "memory/max_allocated (GiB)": 36.1, "ppl": 1.78875, "step": 90, "tokens/total": 3095248, "tokens/train_per_sec_per_gpu": 69.34, "tokens/trainable": 94544} +{"epoch": 0.35546875, "grad_norm": 1.2738616466522217, "learning_rate": 9.610167032937036e-05, "loss": 0.473626971244812, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.60581, "step": 91, "tokens/total": 3133456, "tokens/train_per_sec_per_gpu": 54.54, "tokens/trainable": 95597} +{"epoch": 0.359375, "grad_norm": 1.1431539058685303, "learning_rate": 9.598262995928611e-05, "loss": 0.4540552496910095, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.57468, "step": 92, "tokens/total": 3164624, "tokens/train_per_sec_per_gpu": 70.47, "tokens/trainable": 96662} +{"epoch": 0.36328125, "grad_norm": 1.7148538827896118, "learning_rate": 9.586188413468492e-05, "loss": 0.4519950747489929, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.8, "memory/max_allocated (GiB)": 38.8, "ppl": 1.57144, "step": 93, "tokens/total": 3203008, "tokens/train_per_sec_per_gpu": 49.64, "tokens/trainable": 97715} +{"epoch": 0.3671875, "grad_norm": 1.4628163576126099, "learning_rate": 9.57394378802934e-05, "loss": 0.43398889899253845, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.5434, "step": 94, "tokens/total": 3236448, "tokens/train_per_sec_per_gpu": 60.9, "tokens/trainable": 98761} +{"epoch": 0.37109375, "grad_norm": 9.693976402282715, "learning_rate": 9.56152962916e-05, "loss": 0.44445255398750305, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.55964, "step": 95, "tokens/total": 3267104, "tokens/train_per_sec_per_gpu": 66.38, "tokens/trainable": 99757} +{"epoch": 0.375, "grad_norm": 1.8830853700637817, "learning_rate": 9.548946453464296e-05, "loss": 0.3784937560558319, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 1.46008, "step": 96, "tokens/total": 3301440, "tokens/train_per_sec_per_gpu": 62.79, "tokens/trainable": 100862} +{"epoch": 0.37890625, "grad_norm": 1.7742098569869995, "learning_rate": 9.53619478457953e-05, "loss": 0.5308460593223572, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.70037, "step": 97, "tokens/total": 3336112, "tokens/train_per_sec_per_gpu": 52.58, "tokens/trainable": 101868} +{"epoch": 0.3828125, "grad_norm": 1.2398440837860107, "learning_rate": 9.523275153154695e-05, "loss": 0.45963335037231445, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.58349, "step": 98, "tokens/total": 3368624, "tokens/train_per_sec_per_gpu": 58.07, "tokens/trainable": 102828} +{"epoch": 0.38671875, "grad_norm": 1.4880133867263794, "learning_rate": 9.51018809682839e-05, "loss": 0.4188719391822815, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.52025, "step": 99, "tokens/total": 3406448, "tokens/train_per_sec_per_gpu": 53.02, "tokens/trainable": 103845} +{"epoch": 0.390625, "grad_norm": 4.455123424530029, "learning_rate": 9.49693416020645e-05, "loss": 0.4540572762489319, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.51, "memory/max_allocated (GiB)": 38.51, "ppl": 1.57469, "step": 100, "tokens/total": 3444240, "tokens/train_per_sec_per_gpu": 54.99, "tokens/trainable": 104865} +{"epoch": 0.39453125, "grad_norm": 1.6260507106781006, "learning_rate": 9.483513894839276e-05, "loss": 0.46727246046066284, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.59564, "step": 101, "tokens/total": 3477680, "tokens/train_per_sec_per_gpu": 68.18, "tokens/trainable": 105968} +{"epoch": 0.3984375, "grad_norm": 2.140024185180664, "learning_rate": 9.469927859198888e-05, "loss": 0.44665855169296265, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.56308, "step": 102, "tokens/total": 3513616, "tokens/train_per_sec_per_gpu": 58.71, "tokens/trainable": 106968} +{"epoch": 0.40234375, "grad_norm": 1.58027982711792, "learning_rate": 9.456176618655689e-05, "loss": 0.39952653646469116, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.51, "memory/max_allocated (GiB)": 36.51, "ppl": 1.49112, "step": 103, "tokens/total": 3545584, "tokens/train_per_sec_per_gpu": 60.54, "tokens/trainable": 107964} +{"epoch": 0.40625, "grad_norm": 1.5458498001098633, "learning_rate": 9.442260745454927e-05, "loss": 0.4127103090286255, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.51091, "step": 104, "tokens/total": 3580768, "tokens/train_per_sec_per_gpu": 59.12, "tokens/trainable": 109035} +{"epoch": 0.41015625, "grad_norm": 1.765748381614685, "learning_rate": 9.428180818692884e-05, "loss": 0.4007796347141266, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.49299, "step": 105, "tokens/total": 3617008, "tokens/train_per_sec_per_gpu": 52.63, "tokens/trainable": 110095} +{"epoch": 0.4140625, "grad_norm": 2.1777195930480957, "learning_rate": 9.413937424292791e-05, "loss": 0.47633785009384155, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.61017, "step": 106, "tokens/total": 3655936, "tokens/train_per_sec_per_gpu": 52.5, "tokens/trainable": 111135} +{"epoch": 0.41796875, "grad_norm": 1.464959979057312, "learning_rate": 9.399531154980424e-05, "loss": 0.35228782892227173, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.42232, "step": 107, "tokens/total": 3692736, "tokens/train_per_sec_per_gpu": 51.65, "tokens/trainable": 112140} +{"epoch": 0.421875, "grad_norm": 1.7622828483581543, "learning_rate": 9.384962610259455e-05, "loss": 0.4502136707305908, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.56865, "step": 108, "tokens/total": 3727248, "tokens/train_per_sec_per_gpu": 54.75, "tokens/trainable": 113165} +{"epoch": 0.42578125, "grad_norm": 1.554308295249939, "learning_rate": 9.370232396386494e-05, "loss": 0.4484640955924988, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.56591, "step": 109, "tokens/total": 3760464, "tokens/train_per_sec_per_gpu": 60.8, "tokens/trainable": 114212} +{"epoch": 0.4296875, "grad_norm": 1.553196668624878, "learning_rate": 9.355341126345868e-05, "loss": 0.36960726976394653, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.44717, "step": 110, "tokens/total": 3795952, "tokens/train_per_sec_per_gpu": 61.29, "tokens/trainable": 115312} +{"epoch": 0.43359375, "grad_norm": 1.6536571979522705, "learning_rate": 9.340289419824107e-05, "loss": 0.34366440773010254, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.74, "memory/max_allocated (GiB)": 36.74, "ppl": 1.41011, "step": 111, "tokens/total": 3828144, "tokens/train_per_sec_per_gpu": 69.17, "tokens/trainable": 116343} +{"epoch": 0.4375, "grad_norm": 7.823884010314941, "learning_rate": 9.325077903184159e-05, "loss": 0.5768294334411621, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.78038, "step": 112, "tokens/total": 3862992, "tokens/train_per_sec_per_gpu": 62.05, "tokens/trainable": 117463} +{"epoch": 0.44140625, "grad_norm": 1.3184453248977661, "learning_rate": 9.30970720943932e-05, "loss": 0.3748881220817566, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.43, "memory/max_allocated (GiB)": 38.43, "ppl": 1.45483, "step": 113, "tokens/total": 3898416, "tokens/train_per_sec_per_gpu": 59.27, "tokens/trainable": 118557} +{"epoch": 0.4453125, "grad_norm": 1.592132806777954, "learning_rate": 9.2941779782269e-05, "loss": 0.3952649235725403, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.48478, "step": 114, "tokens/total": 3932176, "tokens/train_per_sec_per_gpu": 61.22, "tokens/trainable": 119622} +{"epoch": 0.44921875, "grad_norm": 1.3366199731826782, "learning_rate": 9.278490855781596e-05, "loss": 0.44844675064086914, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.56588, "step": 115, "tokens/total": 3964384, "tokens/train_per_sec_per_gpu": 71.39, "tokens/trainable": 120662} +{"epoch": 0.453125, "grad_norm": 2.0549166202545166, "learning_rate": 9.262646494908604e-05, "loss": 0.4615691304206848, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 39.14, "memory/max_allocated (GiB)": 39.14, "ppl": 1.58656, "step": 116, "tokens/total": 4002960, "tokens/train_per_sec_per_gpu": 51.98, "tokens/trainable": 121672} +{"epoch": 0.45703125, "grad_norm": 3.327720880508423, "learning_rate": 9.246645554956457e-05, "loss": 0.4304129183292389, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.53789, "step": 117, "tokens/total": 4037552, "tokens/train_per_sec_per_gpu": 62.41, "tokens/trainable": 122725} +{"epoch": 0.4609375, "grad_norm": 1.673270344734192, "learning_rate": 9.230488701789578e-05, "loss": 0.5455777645111084, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.7, "memory/max_allocated (GiB)": 38.7, "ppl": 1.72561, "step": 118, "tokens/total": 4075072, "tokens/train_per_sec_per_gpu": 59.01, "tokens/trainable": 123843} +{"epoch": 0.46484375, "grad_norm": 2.8129730224609375, "learning_rate": 9.214176607760577e-05, "loss": 0.35883545875549316, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.43166, "step": 119, "tokens/total": 4113744, "tokens/train_per_sec_per_gpu": 58.56, "tokens/trainable": 124974} +{"epoch": 0.46875, "grad_norm": 1.118100881576538, "learning_rate": 9.197709951682268e-05, "loss": 0.3680168390274048, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.44487, "step": 120, "tokens/total": 4146032, "tokens/train_per_sec_per_gpu": 64.99, "tokens/trainable": 126039} +{"epoch": 0.47265625, "grad_norm": 1.619661808013916, "learning_rate": 9.181089418799428e-05, "loss": 0.40094077587127686, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 39.39, "memory/max_allocated (GiB)": 39.39, "ppl": 1.49323, "step": 121, "tokens/total": 4182832, "tokens/train_per_sec_per_gpu": 54.88, "tokens/trainable": 127078} +{"epoch": 0.4765625, "grad_norm": 1.1360194683074951, "learning_rate": 9.164315700760271e-05, "loss": 0.3214392066001892, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.37911, "step": 122, "tokens/total": 4218640, "tokens/train_per_sec_per_gpu": 58.33, "tokens/trainable": 128122} +{"epoch": 0.48046875, "grad_norm": 1.3588093519210815, "learning_rate": 9.147389495587671e-05, "loss": 0.44940829277038574, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 1.56738, "step": 123, "tokens/total": 4249792, "tokens/train_per_sec_per_gpu": 68.91, "tokens/trainable": 129160} +{"epoch": 0.484375, "grad_norm": 1.120273470878601, "learning_rate": 9.130311507650116e-05, "loss": 0.30848243832588196, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 1.36136, "step": 124, "tokens/total": 4282464, "tokens/train_per_sec_per_gpu": 61.48, "tokens/trainable": 130215} +{"epoch": 0.48828125, "grad_norm": 1.340184211730957, "learning_rate": 9.113082447632394e-05, "loss": 0.36651086807250977, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.44269, "step": 125, "tokens/total": 4314608, "tokens/train_per_sec_per_gpu": 79.44, "tokens/trainable": 131358} +{"epoch": 0.4921875, "grad_norm": 1.64254891872406, "learning_rate": 9.09570303250602e-05, "loss": 0.39231300354003906, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.47, "memory/max_allocated (GiB)": 36.47, "ppl": 1.4804, "step": 126, "tokens/total": 4346416, "tokens/train_per_sec_per_gpu": 63.2, "tokens/trainable": 132421} +{"epoch": 0.49609375, "grad_norm": 1.850118637084961, "learning_rate": 9.078173985499394e-05, "loss": 0.3656052350997925, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 35.79, "memory/max_allocated (GiB)": 35.79, "ppl": 1.44139, "step": 127, "tokens/total": 4376400, "tokens/train_per_sec_per_gpu": 65.43, "tokens/trainable": 133425} +{"epoch": 0.5, "grad_norm": 1.7712551355361938, "learning_rate": 9.060496036067713e-05, "loss": 0.3941921591758728, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.72, "memory/max_allocated (GiB)": 38.72, "ppl": 1.48319, "step": 128, "tokens/total": 4413824, "tokens/train_per_sec_per_gpu": 48.13, "tokens/trainable": 134439} +{"epoch": 0.50390625, "grad_norm": 1.4708192348480225, "learning_rate": 9.042669919862615e-05, "loss": 0.44553565979003906, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.82, "memory/max_allocated (GiB)": 36.82, "ppl": 1.56133, "step": 129, "tokens/total": 4445936, "tokens/train_per_sec_per_gpu": 67.58, "tokens/trainable": 135505} +{"epoch": 0.5078125, "grad_norm": 1.2547310590744019, "learning_rate": 9.024696378701557e-05, "loss": 0.3484632670879364, "memory/device_reserved (GiB)": 41.42, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.41689, "step": 130, "tokens/total": 4481488, "tokens/train_per_sec_per_gpu": 58.47, "tokens/trainable": 136588} +{"epoch": 0.51171875, "grad_norm": 14.120157241821289, "learning_rate": 9.006576160536948e-05, "loss": 0.3883053958415985, "memory/device_reserved (GiB)": 41.42, "memory/max_active (GiB)": 37.43, "memory/max_allocated (GiB)": 37.43, "ppl": 1.47448, "step": 131, "tokens/total": 4516048, "tokens/train_per_sec_per_gpu": 56.01, "tokens/trainable": 137590} +{"epoch": 0.515625, "grad_norm": 2.3283262252807617, "learning_rate": 8.988310019425035e-05, "loss": 0.36731553077697754, "memory/device_reserved (GiB)": 41.71, "memory/max_active (GiB)": 39.7, "memory/max_allocated (GiB)": 39.7, "ppl": 1.44385, "step": 132, "tokens/total": 4553904, "tokens/train_per_sec_per_gpu": 51.48, "tokens/trainable": 138600} +{"epoch": 0.51953125, "grad_norm": 2.0074243545532227, "learning_rate": 8.969898715494506e-05, "loss": 0.35614046454429626, "memory/device_reserved (GiB)": 41.73, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.42781, "step": 133, "tokens/total": 4590736, "tokens/train_per_sec_per_gpu": 61.06, "tokens/trainable": 139690} +{"epoch": 0.5234375, "grad_norm": 1.118798851966858, "learning_rate": 8.951343014914869e-05, "loss": 0.3480440676212311, "memory/device_reserved (GiB)": 41.73, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.41629, "step": 134, "tokens/total": 4627392, "tokens/train_per_sec_per_gpu": 57.23, "tokens/trainable": 140782} +{"epoch": 0.52734375, "grad_norm": 1.9665861129760742, "learning_rate": 8.932643689864568e-05, "loss": 0.33823472261428833, "memory/device_reserved (GiB)": 41.73, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.40247, "step": 135, "tokens/total": 4663968, "tokens/train_per_sec_per_gpu": 62.06, "tokens/trainable": 141899} +{"epoch": 0.53125, "grad_norm": 1.0597126483917236, "learning_rate": 8.913801518498845e-05, "loss": 0.30692562460899353, "memory/device_reserved (GiB)": 41.73, "memory/max_active (GiB)": 39.68, "memory/max_allocated (GiB)": 39.68, "ppl": 1.35924, "step": 136, "tokens/total": 4703344, "tokens/train_per_sec_per_gpu": 48.56, "tokens/trainable": 143011} +{"epoch": 0.53515625, "grad_norm": 1.5030730962753296, "learning_rate": 8.894817284917364e-05, "loss": 0.3611525297164917, "memory/device_reserved (GiB)": 41.73, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.43498, "step": 137, "tokens/total": 4740288, "tokens/train_per_sec_per_gpu": 50.22, "tokens/trainable": 144045} +{"epoch": 0.5390625, "grad_norm": 1.1728569269180298, "learning_rate": 8.875691779131569e-05, "loss": 0.3733293414115906, "memory/device_reserved (GiB)": 41.9, "memory/max_active (GiB)": 39.78, "memory/max_allocated (GiB)": 39.78, "ppl": 1.45256, "step": 138, "tokens/total": 4778224, "tokens/train_per_sec_per_gpu": 56.17, "tokens/trainable": 145172} +{"epoch": 0.54296875, "grad_norm": 1.1091041564941406, "learning_rate": 8.856425797031829e-05, "loss": 0.293356329202652, "memory/device_reserved (GiB)": 42.22, "memory/max_active (GiB)": 39.58, "memory/max_allocated (GiB)": 39.58, "ppl": 1.34092, "step": 139, "tokens/total": 4814256, "tokens/train_per_sec_per_gpu": 63.66, "tokens/trainable": 146300} +{"epoch": 0.546875, "grad_norm": 1.3786416053771973, "learning_rate": 8.837020140354295e-05, "loss": 0.3385131359100342, "memory/device_reserved (GiB)": 42.22, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.40286, "step": 140, "tokens/total": 4851072, "tokens/train_per_sec_per_gpu": 55.26, "tokens/trainable": 147307} +{"epoch": 0.55078125, "grad_norm": 1.800492286682129, "learning_rate": 8.817475616647554e-05, "loss": 0.3756472170352936, "memory/device_reserved (GiB)": 42.22, "memory/max_active (GiB)": 38.17, "memory/max_allocated (GiB)": 38.17, "ppl": 1.45593, "step": 141, "tokens/total": 4885504, "tokens/train_per_sec_per_gpu": 60.06, "tokens/trainable": 148393} +{"epoch": 0.5546875, "grad_norm": 1.4500648975372314, "learning_rate": 8.797793039239017e-05, "loss": 0.38521212339401245, "memory/device_reserved (GiB)": 42.22, "memory/max_active (GiB)": 37.83, "memory/max_allocated (GiB)": 37.83, "ppl": 1.46993, "step": 142, "tokens/total": 4920528, "tokens/train_per_sec_per_gpu": 64.7, "tokens/trainable": 149499} +{"epoch": 0.55859375, "grad_norm": 2.718946933746338, "learning_rate": 8.777973227201069e-05, "loss": 0.3212054669857025, "memory/device_reserved (GiB)": 43.76, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.37879, "step": 143, "tokens/total": 4957440, "tokens/train_per_sec_per_gpu": 60.33, "tokens/trainable": 150562} +{"epoch": 0.5625, "grad_norm": 2.676314115524292, "learning_rate": 8.758017005316988e-05, "loss": 0.4228561520576477, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.52631, "step": 144, "tokens/total": 4990752, "tokens/train_per_sec_per_gpu": 72.7, "tokens/trainable": 151616} +{"epoch": 0.56640625, "grad_norm": 1.690446376800537, "learning_rate": 8.737925204046629e-05, "loss": 0.4257187247276306, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.53069, "step": 145, "tokens/total": 5025472, "tokens/train_per_sec_per_gpu": 54.72, "tokens/trainable": 152681} +{"epoch": 0.5703125, "grad_norm": 3.6888279914855957, "learning_rate": 8.717698659491851e-05, "loss": 0.32381731271743774, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 38.91, "memory/max_allocated (GiB)": 38.91, "ppl": 1.38239, "step": 146, "tokens/total": 5062304, "tokens/train_per_sec_per_gpu": 57.03, "tokens/trainable": 153785} +{"epoch": 0.57421875, "grad_norm": 1.8901386260986328, "learning_rate": 8.697338213361735e-05, "loss": 0.38495397567749023, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 36.45, "memory/max_allocated (GiB)": 36.45, "ppl": 1.46955, "step": 147, "tokens/total": 5093280, "tokens/train_per_sec_per_gpu": 69.75, "tokens/trainable": 154886} +{"epoch": 0.578125, "grad_norm": 1.1783356666564941, "learning_rate": 8.676844712937552e-05, "loss": 0.29394644498825073, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.34171, "step": 148, "tokens/total": 5133280, "tokens/train_per_sec_per_gpu": 49.16, "tokens/trainable": 155938} +{"epoch": 0.58203125, "grad_norm": 1.2965179681777954, "learning_rate": 8.656219011037509e-05, "loss": 0.3294377326965332, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.39019, "step": 149, "tokens/total": 5170576, "tokens/train_per_sec_per_gpu": 55.62, "tokens/trainable": 157021} +{"epoch": 0.5859375, "grad_norm": 1.477937936782837, "learning_rate": 8.63546196598125e-05, "loss": 0.3617400825023651, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.43583, "step": 150, "tokens/total": 5208160, "tokens/train_per_sec_per_gpu": 55.74, "tokens/trainable": 158173} +{"epoch": 0.58984375, "grad_norm": 1.3747085332870483, "learning_rate": 8.614574441554145e-05, "loss": 0.4365062713623047, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 36.95, "memory/max_allocated (GiB)": 36.95, "ppl": 1.54729, "step": 151, "tokens/total": 5240784, "tokens/train_per_sec_per_gpu": 63.14, "tokens/trainable": 159145} +{"epoch": 0.59375, "grad_norm": 1.1340811252593994, "learning_rate": 8.593557306971349e-05, "loss": 0.2978513836860657, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.34696, "step": 152, "tokens/total": 5274096, "tokens/train_per_sec_per_gpu": 58.46, "tokens/trainable": 160178} +{"epoch": 0.59765625, "grad_norm": 1.2161450386047363, "learning_rate": 8.572411436841618e-05, "loss": 0.3531741201877594, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 36.9, "memory/max_allocated (GiB)": 36.9, "ppl": 1.42358, "step": 153, "tokens/total": 5307840, "tokens/train_per_sec_per_gpu": 64.23, "tokens/trainable": 161248} +{"epoch": 0.6015625, "grad_norm": 1.1345082521438599, "learning_rate": 8.551137711130922e-05, "loss": 0.32013434171676636, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.37731, "step": 154, "tokens/total": 5339616, "tokens/train_per_sec_per_gpu": 58.98, "tokens/trainable": 162266} +{"epoch": 0.60546875, "grad_norm": 0.9761771559715271, "learning_rate": 8.529737015125824e-05, "loss": 0.2954900562763214, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 38.17, "memory/max_allocated (GiB)": 38.17, "ppl": 1.34378, "step": 155, "tokens/total": 5374720, "tokens/train_per_sec_per_gpu": 56.68, "tokens/trainable": 163285} +{"epoch": 0.609375, "grad_norm": 1.1684285402297974, "learning_rate": 8.508210239396639e-05, "loss": 0.34238573908805847, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 39.77, "memory/max_allocated (GiB)": 39.77, "ppl": 1.4083, "step": 156, "tokens/total": 5412608, "tokens/train_per_sec_per_gpu": 57.18, "tokens/trainable": 164302} +{"epoch": 0.61328125, "grad_norm": 1.4465785026550293, "learning_rate": 8.486558279760375e-05, "loss": 0.3489457368850708, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 35.9, "memory/max_allocated (GiB)": 35.9, "ppl": 1.41757, "step": 157, "tokens/total": 5443664, "tokens/train_per_sec_per_gpu": 64.0, "tokens/trainable": 165309} +{"epoch": 0.6171875, "grad_norm": 1.5687037706375122, "learning_rate": 8.464782037243449e-05, "loss": 0.3344725966453552, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.3972, "step": 158, "tokens/total": 5475808, "tokens/train_per_sec_per_gpu": 60.59, "tokens/trainable": 166298} +{"epoch": 0.62109375, "grad_norm": 1.284502387046814, "learning_rate": 8.442882418044202e-05, "loss": 0.3042309284210205, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.35558, "step": 159, "tokens/total": 5508448, "tokens/train_per_sec_per_gpu": 63.53, "tokens/trainable": 167338} +{"epoch": 0.625, "grad_norm": 1.3267287015914917, "learning_rate": 8.420860333495179e-05, "loss": 0.35301458835601807, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.42335, "step": 160, "tokens/total": 5540976, "tokens/train_per_sec_per_gpu": 64.69, "tokens/trainable": 168338} +{"epoch": 0.62890625, "grad_norm": 2.2954468727111816, "learning_rate": 8.398716700025208e-05, "loss": 0.3653852343559265, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.44107, "step": 161, "tokens/total": 5574496, "tokens/train_per_sec_per_gpu": 61.42, "tokens/trainable": 169378} +{"epoch": 0.6328125, "grad_norm": 2.7401368618011475, "learning_rate": 8.376452439121266e-05, "loss": 0.4276249408721924, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.05, "memory/max_allocated (GiB)": 36.05, "ppl": 1.53361, "step": 162, "tokens/total": 5604544, "tokens/train_per_sec_per_gpu": 69.72, "tokens/trainable": 170407} +{"epoch": 0.63671875, "grad_norm": 2.155133008956909, "learning_rate": 8.354068477290124e-05, "loss": 0.3519224524497986, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.4218, "step": 163, "tokens/total": 5638768, "tokens/train_per_sec_per_gpu": 61.16, "tokens/trainable": 171394} +{"epoch": 0.640625, "grad_norm": 1.8384201526641846, "learning_rate": 8.331565746019807e-05, "loss": 0.3436068296432495, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.41002, "step": 164, "tokens/total": 5670960, "tokens/train_per_sec_per_gpu": 68.79, "tokens/trainable": 172444} +{"epoch": 0.64453125, "grad_norm": 1.6507022380828857, "learning_rate": 8.308945181740812e-05, "loss": 0.37073105573654175, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.44879, "step": 165, "tokens/total": 5705664, "tokens/train_per_sec_per_gpu": 62.04, "tokens/trainable": 173468} +{"epoch": 0.6484375, "grad_norm": 3.5636541843414307, "learning_rate": 8.286207725787153e-05, "loss": 0.27918875217437744, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.32206, "step": 166, "tokens/total": 5742672, "tokens/train_per_sec_per_gpu": 51.81, "tokens/trainable": 174495} +{"epoch": 0.65234375, "grad_norm": 1.486402988433838, "learning_rate": 8.263354324357182e-05, "loss": 0.2980083227157593, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.12, "memory/max_allocated (GiB)": 39.12, "ppl": 1.34717, "step": 167, "tokens/total": 5777408, "tokens/train_per_sec_per_gpu": 60.22, "tokens/trainable": 175533} +{"epoch": 0.65625, "grad_norm": 2.4917056560516357, "learning_rate": 8.240385928474219e-05, "loss": 0.3795654773712158, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.46165, "step": 168, "tokens/total": 5810224, "tokens/train_per_sec_per_gpu": 57.94, "tokens/trainable": 176493} +{"epoch": 0.66015625, "grad_norm": 1.5172734260559082, "learning_rate": 8.217303493946967e-05, "loss": 0.34326592087745667, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.40954, "step": 169, "tokens/total": 5844432, "tokens/train_per_sec_per_gpu": 57.91, "tokens/trainable": 177502} +{"epoch": 0.6640625, "grad_norm": 1.1306753158569336, "learning_rate": 8.194107981329746e-05, "loss": 0.31168049573898315, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.73, "memory/max_allocated (GiB)": 39.73, "ppl": 1.36572, "step": 170, "tokens/total": 5883008, "tokens/train_per_sec_per_gpu": 51.67, "tokens/trainable": 178523} +{"epoch": 0.66796875, "grad_norm": 1.2229909896850586, "learning_rate": 8.170800355882518e-05, "loss": 0.29211729764938354, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.33926, "step": 171, "tokens/total": 5917440, "tokens/train_per_sec_per_gpu": 66.27, "tokens/trainable": 179562} +{"epoch": 0.671875, "grad_norm": 5.435315132141113, "learning_rate": 8.147381587530713e-05, "loss": 0.3869026303291321, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.39, "memory/max_allocated (GiB)": 37.39, "ppl": 1.47241, "step": 172, "tokens/total": 5951232, "tokens/train_per_sec_per_gpu": 62.88, "tokens/trainable": 180566} +{"epoch": 0.67578125, "grad_norm": 2.176952362060547, "learning_rate": 8.123852650824877e-05, "loss": 0.3474155068397522, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.45, "memory/max_allocated (GiB)": 37.45, "ppl": 1.4154, "step": 173, "tokens/total": 5985056, "tokens/train_per_sec_per_gpu": 63.38, "tokens/trainable": 181642} +{"epoch": 0.6796875, "grad_norm": 1.4139868021011353, "learning_rate": 8.100214524900103e-05, "loss": 0.3761778473854065, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.45671, "step": 174, "tokens/total": 6020128, "tokens/train_per_sec_per_gpu": 52.0, "tokens/trainable": 182645} +{"epoch": 0.68359375, "grad_norm": 1.9520306587219238, "learning_rate": 8.076468193435301e-05, "loss": 0.3438858985900879, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.41042, "step": 175, "tokens/total": 6050976, "tokens/train_per_sec_per_gpu": 66.61, "tokens/trainable": 183593} +{"epoch": 0.6875, "grad_norm": 1.155078649520874, "learning_rate": 8.052614644612253e-05, "loss": 0.30771970748901367, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.36032, "step": 176, "tokens/total": 6088816, "tokens/train_per_sec_per_gpu": 50.01, "tokens/trainable": 184643} +{"epoch": 0.69140625, "grad_norm": 1.3322153091430664, "learning_rate": 8.028654871074489e-05, "loss": 0.2723207175731659, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.31301, "step": 177, "tokens/total": 6123408, "tokens/train_per_sec_per_gpu": 59.08, "tokens/trainable": 185729} +{"epoch": 0.6953125, "grad_norm": 2.0887813568115234, "learning_rate": 8.004589869885986e-05, "loss": 0.3984096050262451, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.37, "memory/max_allocated (GiB)": 36.37, "ppl": 1.48945, "step": 178, "tokens/total": 6154880, "tokens/train_per_sec_per_gpu": 61.6, "tokens/trainable": 186718} +{"epoch": 0.69921875, "grad_norm": 1.1933705806732178, "learning_rate": 7.980420642489674e-05, "loss": 0.2711789608001709, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.31151, "step": 179, "tokens/total": 6193584, "tokens/train_per_sec_per_gpu": 53.65, "tokens/trainable": 187750} +{"epoch": 0.703125, "grad_norm": 1.3288713693618774, "learning_rate": 7.95614819466576e-05, "loss": 0.32536521553993225, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 1.38454, "step": 180, "tokens/total": 6227296, "tokens/train_per_sec_per_gpu": 61.62, "tokens/trainable": 188825} +{"epoch": 0.70703125, "grad_norm": 1.3827890157699585, "learning_rate": 7.931773536489872e-05, "loss": 0.29728472232818604, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.3462, "step": 181, "tokens/total": 6261888, "tokens/train_per_sec_per_gpu": 62.54, "tokens/trainable": 189884} +{"epoch": 0.7109375, "grad_norm": 5.457559585571289, "learning_rate": 7.907297682291035e-05, "loss": 0.30658698081970215, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.35878, "step": 182, "tokens/total": 6296544, "tokens/train_per_sec_per_gpu": 60.75, "tokens/trainable": 190987} +{"epoch": 0.71484375, "grad_norm": 2.7288248538970947, "learning_rate": 7.882721650609442e-05, "loss": 0.2517892122268677, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.65, "memory/max_allocated (GiB)": 39.65, "ppl": 1.28632, "step": 183, "tokens/total": 6336960, "tokens/train_per_sec_per_gpu": 53.35, "tokens/trainable": 192165} +{"epoch": 0.71875, "grad_norm": 1.3733410835266113, "learning_rate": 7.85804646415409e-05, "loss": 0.2775115966796875, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.31984, "step": 184, "tokens/total": 6372192, "tokens/train_per_sec_per_gpu": 64.12, "tokens/trainable": 193251} +{"epoch": 0.72265625, "grad_norm": 1.6983435153961182, "learning_rate": 7.833273149760207e-05, "loss": 0.36109840869903564, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.4349, "step": 185, "tokens/total": 6403056, "tokens/train_per_sec_per_gpu": 62.86, "tokens/trainable": 194276} +{"epoch": 0.7265625, "grad_norm": 1.3649567365646362, "learning_rate": 7.808402738346527e-05, "loss": 0.2695041298866272, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.11, "memory/max_allocated (GiB)": 39.11, "ppl": 1.30932, "step": 186, "tokens/total": 6441840, "tokens/train_per_sec_per_gpu": 58.16, "tokens/trainable": 195434} +{"epoch": 0.73046875, "grad_norm": 1.851486086845398, "learning_rate": 7.783436264872382e-05, "loss": 0.2772060036659241, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.31944, "step": 187, "tokens/total": 6478688, "tokens/train_per_sec_per_gpu": 56.08, "tokens/trainable": 196483} +{"epoch": 0.734375, "grad_norm": 1.7991838455200195, "learning_rate": 7.758374768294647e-05, "loss": 0.2655234932899475, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.16, "memory/max_allocated (GiB)": 38.16, "ppl": 1.30411, "step": 188, "tokens/total": 6514624, "tokens/train_per_sec_per_gpu": 60.84, "tokens/trainable": 197578} +{"epoch": 0.73828125, "grad_norm": 1.72531259059906, "learning_rate": 7.733219291524489e-05, "loss": 0.3006952106952667, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.53, "memory/max_allocated (GiB)": 37.53, "ppl": 1.3508, "step": 189, "tokens/total": 6548048, "tokens/train_per_sec_per_gpu": 64.79, "tokens/trainable": 198635} +{"epoch": 0.7421875, "grad_norm": 1.67169189453125, "learning_rate": 7.707970881383977e-05, "loss": 0.4243854284286499, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.45, "memory/max_allocated (GiB)": 36.45, "ppl": 1.52865, "step": 190, "tokens/total": 6579760, "tokens/train_per_sec_per_gpu": 71.73, "tokens/trainable": 199697} +{"epoch": 0.74609375, "grad_norm": 1.5897984504699707, "learning_rate": 7.682630588562518e-05, "loss": 0.2799915373325348, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.7, "memory/max_allocated (GiB)": 38.7, "ppl": 1.32312, "step": 191, "tokens/total": 6614800, "tokens/train_per_sec_per_gpu": 57.61, "tokens/trainable": 200758} +{"epoch": 0.75, "grad_norm": 1.2010878324508667, "learning_rate": 7.657199467573129e-05, "loss": 0.2933754324913025, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.34095, "step": 192, "tokens/total": 6648928, "tokens/train_per_sec_per_gpu": 57.23, "tokens/trainable": 201783} +{"epoch": 0.75390625, "grad_norm": 1.0512324571609497, "learning_rate": 7.631678576708561e-05, "loss": 0.2383604347705841, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.16, "memory/max_allocated (GiB)": 38.16, "ppl": 1.26917, "step": 193, "tokens/total": 6683328, "tokens/train_per_sec_per_gpu": 67.35, "tokens/trainable": 202877} +{"epoch": 0.7578125, "grad_norm": 1.4054450988769531, "learning_rate": 7.606068977997255e-05, "loss": 0.2781673073768616, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.64, "memory/max_allocated (GiB)": 39.64, "ppl": 1.32071, "step": 194, "tokens/total": 6720928, "tokens/train_per_sec_per_gpu": 66.12, "tokens/trainable": 204053} +{"epoch": 0.76171875, "grad_norm": 2.007673501968384, "learning_rate": 7.580371737159148e-05, "loss": 0.32034188508987427, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.3776, "step": 195, "tokens/total": 6757024, "tokens/train_per_sec_per_gpu": 58.01, "tokens/trainable": 205111} +{"epoch": 0.765625, "grad_norm": 1.9475250244140625, "learning_rate": 7.554587923561324e-05, "loss": 0.25406524538993835, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.99, "memory/max_allocated (GiB)": 38.99, "ppl": 1.28926, "step": 196, "tokens/total": 6791728, "tokens/train_per_sec_per_gpu": 51.73, "tokens/trainable": 206083} +{"epoch": 0.76953125, "grad_norm": 1.787980079650879, "learning_rate": 7.528718610173511e-05, "loss": 0.25241369009017944, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.28713, "step": 197, "tokens/total": 6827072, "tokens/train_per_sec_per_gpu": 63.21, "tokens/trainable": 207177} +{"epoch": 0.7734375, "grad_norm": 1.9021039009094238, "learning_rate": 7.502764873523431e-05, "loss": 0.34284210205078125, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.40895, "step": 198, "tokens/total": 6861120, "tokens/train_per_sec_per_gpu": 62.56, "tokens/trainable": 208214} +{"epoch": 0.77734375, "grad_norm": 3.083019733428955, "learning_rate": 7.476727793652011e-05, "loss": 0.2735263407230377, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.72, "memory/max_allocated (GiB)": 36.72, "ppl": 1.31459, "step": 199, "tokens/total": 6894352, "tokens/train_per_sec_per_gpu": 62.54, "tokens/trainable": 209273} +{"epoch": 0.78125, "grad_norm": 3.1179018020629883, "learning_rate": 7.450608454068415e-05, "loss": 0.24231886863708496, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.2742, "step": 200, "tokens/total": 6928848, "tokens/train_per_sec_per_gpu": 57.81, "tokens/trainable": 210335} +{"epoch": 0.78515625, "grad_norm": 1.2168591022491455, "learning_rate": 7.424407941704987e-05, "loss": 0.21952256560325623, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.49, "memory/max_allocated (GiB)": 38.49, "ppl": 1.24548, "step": 201, "tokens/total": 6965504, "tokens/train_per_sec_per_gpu": 67.15, "tokens/trainable": 211480} +{"epoch": 0.7890625, "grad_norm": 1.4849469661712646, "learning_rate": 7.398127346871986e-05, "loss": 0.25169089436531067, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.61, "memory/max_allocated (GiB)": 39.61, "ppl": 1.2862, "step": 202, "tokens/total": 7002448, "tokens/train_per_sec_per_gpu": 54.99, "tokens/trainable": 212523} +{"epoch": 0.79296875, "grad_norm": 18.25193214416504, "learning_rate": 7.371767763212238e-05, "loss": 0.2548300623893738, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.52, "memory/max_allocated (GiB)": 39.52, "ppl": 1.29024, "step": 203, "tokens/total": 7037824, "tokens/train_per_sec_per_gpu": 62.3, "tokens/trainable": 213574} +{"epoch": 0.796875, "grad_norm": 1.730290174484253, "learning_rate": 7.345330287655617e-05, "loss": 0.3183635175228119, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.09, "memory/max_allocated (GiB)": 39.09, "ppl": 1.37488, "step": 204, "tokens/total": 7075920, "tokens/train_per_sec_per_gpu": 48.47, "tokens/trainable": 214633} +{"epoch": 0.80078125, "grad_norm": 1.7796443700790405, "learning_rate": 7.31881602037339e-05, "loss": 0.3618646562099457, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.69, "memory/max_allocated (GiB)": 36.69, "ppl": 1.436, "step": 205, "tokens/total": 7107888, "tokens/train_per_sec_per_gpu": 63.41, "tokens/trainable": 215651} +{"epoch": 0.8046875, "grad_norm": 1.5088255405426025, "learning_rate": 7.29222606473245e-05, "loss": 0.27801239490509033, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.3205, "step": 206, "tokens/total": 7140720, "tokens/train_per_sec_per_gpu": 61.4, "tokens/trainable": 216669} +{"epoch": 0.80859375, "grad_norm": 1.5013434886932373, "learning_rate": 7.265561527249383e-05, "loss": 0.34578990936279297, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.41311, "step": 207, "tokens/total": 7174720, "tokens/train_per_sec_per_gpu": 63.34, "tokens/trainable": 217727} +{"epoch": 0.8125, "grad_norm": 1.062585711479187, "learning_rate": 7.238823517544436e-05, "loss": 0.2608395218849182, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.29802, "step": 208, "tokens/total": 7211872, "tokens/train_per_sec_per_gpu": 67.54, "tokens/trainable": 218899} +{"epoch": 0.81640625, "grad_norm": 1.788941502571106, "learning_rate": 7.212013148295333e-05, "loss": 0.2337607741355896, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.26334, "step": 209, "tokens/total": 7245856, "tokens/train_per_sec_per_gpu": 63.16, "tokens/trainable": 219978} +{"epoch": 0.8203125, "grad_norm": 15.035664558410645, "learning_rate": 7.185131535190975e-05, "loss": 0.28525984287261963, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.06, "memory/max_allocated (GiB)": 37.06, "ppl": 1.33011, "step": 210, "tokens/total": 7278448, "tokens/train_per_sec_per_gpu": 65.76, "tokens/trainable": 221067} +{"epoch": 0.82421875, "grad_norm": 1.0546882152557373, "learning_rate": 7.158179796885005e-05, "loss": 0.23913420736789703, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.27015, "step": 211, "tokens/total": 7313072, "tokens/train_per_sec_per_gpu": 62.04, "tokens/trainable": 222120} +{"epoch": 0.828125, "grad_norm": 1.3975828886032104, "learning_rate": 7.131159054949273e-05, "loss": 0.2625904381275177, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.30029, "step": 212, "tokens/total": 7351456, "tokens/train_per_sec_per_gpu": 54.76, "tokens/trainable": 223159} +{"epoch": 0.83203125, "grad_norm": 1.4839125871658325, "learning_rate": 7.104070433827139e-05, "loss": 0.29613569378852844, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.34465, "step": 213, "tokens/total": 7386880, "tokens/train_per_sec_per_gpu": 52.43, "tokens/trainable": 224113} +{"epoch": 0.8359375, "grad_norm": 1.2732625007629395, "learning_rate": 7.076915060786705e-05, "loss": 0.24131891131401062, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.27293, "step": 214, "tokens/total": 7419696, "tokens/train_per_sec_per_gpu": 64.58, "tokens/trainable": 225176} +{"epoch": 0.83984375, "grad_norm": 1.2292217016220093, "learning_rate": 7.049694065873882e-05, "loss": 0.2713567316532135, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.07, "memory/max_allocated (GiB)": 38.07, "ppl": 1.31174, "step": 215, "tokens/total": 7454352, "tokens/train_per_sec_per_gpu": 55.96, "tokens/trainable": 226219} +{"epoch": 0.84375, "grad_norm": 1.5228546857833862, "learning_rate": 7.022408581865382e-05, "loss": 0.22983351349830627, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.25839, "step": 216, "tokens/total": 7488496, "tokens/train_per_sec_per_gpu": 57.8, "tokens/trainable": 227296} +{"epoch": 0.84765625, "grad_norm": 2.864624261856079, "learning_rate": 6.99505974422157e-05, "loss": 0.21469295024871826, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.23948, "step": 217, "tokens/total": 7522976, "tokens/train_per_sec_per_gpu": 65.58, "tokens/trainable": 228383} +{"epoch": 0.8515625, "grad_norm": 1.7086646556854248, "learning_rate": 6.967648691039213e-05, "loss": 0.2612082362174988, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.58, "memory/max_allocated (GiB)": 36.58, "ppl": 1.2985, "step": 218, "tokens/total": 7554848, "tokens/train_per_sec_per_gpu": 67.5, "tokens/trainable": 229432} +{"epoch": 0.85546875, "grad_norm": 2.276512861251831, "learning_rate": 6.940176563004123e-05, "loss": 0.26263436675071716, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.30035, "step": 219, "tokens/total": 7588896, "tokens/train_per_sec_per_gpu": 57.45, "tokens/trainable": 230405} +{"epoch": 0.859375, "grad_norm": 2.879753828048706, "learning_rate": 6.912644503343682e-05, "loss": 0.26243069767951965, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.30009, "step": 220, "tokens/total": 7627056, "tokens/train_per_sec_per_gpu": 55.45, "tokens/trainable": 231414} +{"epoch": 0.86328125, "grad_norm": 1.3593817949295044, "learning_rate": 6.885053657779273e-05, "loss": 0.2528916001319885, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.28, "memory/max_allocated (GiB)": 36.28, "ppl": 1.28774, "step": 221, "tokens/total": 7658368, "tokens/train_per_sec_per_gpu": 62.44, "tokens/trainable": 232449} +{"epoch": 0.8671875, "grad_norm": 1.5607943534851074, "learning_rate": 6.857405174478604e-05, "loss": 0.26902127265930176, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.30868, "step": 222, "tokens/total": 7693072, "tokens/train_per_sec_per_gpu": 57.9, "tokens/trainable": 233459} +{"epoch": 0.87109375, "grad_norm": 1.3522791862487793, "learning_rate": 6.82970020400792e-05, "loss": 0.28917261958122253, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.33532, "step": 223, "tokens/total": 7730288, "tokens/train_per_sec_per_gpu": 54.13, "tokens/trainable": 234491} +{"epoch": 0.875, "grad_norm": 1.6669028997421265, "learning_rate": 6.801939899284132e-05, "loss": 0.19623523950576782, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.21681, "step": 224, "tokens/total": 7766304, "tokens/train_per_sec_per_gpu": 58.52, "tokens/trainable": 235570} +{"epoch": 0.87890625, "grad_norm": 1.5479234457015991, "learning_rate": 6.774125415526827e-05, "loss": 0.1951381266117096, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.21548, "step": 225, "tokens/total": 7799680, "tokens/train_per_sec_per_gpu": 70.78, "tokens/trainable": 236708} +{"epoch": 0.8828125, "grad_norm": 2.1583213806152344, "learning_rate": 6.746257910210214e-05, "loss": 0.25165390968322754, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.28615, "step": 226, "tokens/total": 7832912, "tokens/train_per_sec_per_gpu": 61.17, "tokens/trainable": 237735} +{"epoch": 0.88671875, "grad_norm": 1.61285400390625, "learning_rate": 6.718338543014937e-05, "loss": 0.2851155698299408, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 35.66, "memory/max_allocated (GiB)": 35.66, "ppl": 1.32992, "step": 227, "tokens/total": 7862912, "tokens/train_per_sec_per_gpu": 69.79, "tokens/trainable": 238769} +{"epoch": 0.890625, "grad_norm": 1.4681198596954346, "learning_rate": 6.69036847577983e-05, "loss": 0.22754529118537903, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.25551, "step": 228, "tokens/total": 7896816, "tokens/train_per_sec_per_gpu": 61.12, "tokens/trainable": 239798} +{"epoch": 0.89453125, "grad_norm": 1.4229627847671509, "learning_rate": 6.662348872453553e-05, "loss": 0.18731321394443512, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.46, "memory/max_allocated (GiB)": 38.46, "ppl": 1.206, "step": 229, "tokens/total": 7933376, "tokens/train_per_sec_per_gpu": 60.87, "tokens/trainable": 240866} +{"epoch": 0.8984375, "grad_norm": 1.2837119102478027, "learning_rate": 6.63428089904618e-05, "loss": 0.25552767515182495, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.29114, "step": 230, "tokens/total": 7972544, "tokens/train_per_sec_per_gpu": 52.45, "tokens/trainable": 241897} +{"epoch": 0.90234375, "grad_norm": 1.4022505283355713, "learning_rate": 6.60616572358065e-05, "loss": 0.2810887098312378, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.32457, "step": 231, "tokens/total": 8007008, "tokens/train_per_sec_per_gpu": 61.98, "tokens/trainable": 242876} +{"epoch": 0.90625, "grad_norm": 1.1342253684997559, "learning_rate": 6.578004516044172e-05, "loss": 0.2254408597946167, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.25287, "step": 232, "tokens/total": 8042864, "tokens/train_per_sec_per_gpu": 62.88, "tokens/trainable": 243921} +{"epoch": 0.91015625, "grad_norm": 1.4422523975372314, "learning_rate": 6.549798448339548e-05, "loss": 0.21132837235927582, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.6, "memory/max_allocated (GiB)": 38.6, "ppl": 1.23532, "step": 233, "tokens/total": 8080272, "tokens/train_per_sec_per_gpu": 60.28, "tokens/trainable": 244977} +{"epoch": 0.9140625, "grad_norm": 1.3770294189453125, "learning_rate": 6.521548694236384e-05, "loss": 0.2524690330028534, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.2872, "step": 234, "tokens/total": 8117584, "tokens/train_per_sec_per_gpu": 54.89, "tokens/trainable": 246030} +{"epoch": 0.91796875, "grad_norm": 1.325021743774414, "learning_rate": 6.493256429322259e-05, "loss": 0.22300156950950623, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.24982, "step": 235, "tokens/total": 8153712, "tokens/train_per_sec_per_gpu": 55.54, "tokens/trainable": 247057} +{"epoch": 0.921875, "grad_norm": 1.4948902130126953, "learning_rate": 6.464922830953799e-05, "loss": 0.25603222846984863, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.29179, "step": 236, "tokens/total": 8192928, "tokens/train_per_sec_per_gpu": 53.05, "tokens/trainable": 248105} +{"epoch": 0.92578125, "grad_norm": 1.9079182147979736, "learning_rate": 6.436549078207688e-05, "loss": 0.2333359718322754, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.21, "memory/max_allocated (GiB)": 38.21, "ppl": 1.26281, "step": 237, "tokens/total": 8226016, "tokens/train_per_sec_per_gpu": 62.18, "tokens/trainable": 249105} +{"epoch": 0.9296875, "grad_norm": 1.3377152681350708, "learning_rate": 6.408136351831592e-05, "loss": 0.2434045374393463, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 37.15, "memory/max_allocated (GiB)": 37.15, "ppl": 1.27558, "step": 238, "tokens/total": 8258384, "tokens/train_per_sec_per_gpu": 60.19, "tokens/trainable": 250102} +{"epoch": 0.93359375, "grad_norm": 1.1700146198272705, "learning_rate": 6.379685834195036e-05, "loss": 0.21321356296539307, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.23765, "step": 239, "tokens/total": 8297888, "tokens/train_per_sec_per_gpu": 54.32, "tokens/trainable": 251272} +{"epoch": 0.9375, "grad_norm": 2.3952901363372803, "learning_rate": 6.351198709240186e-05, "loss": 0.2519729733467102, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.61, "memory/max_allocated (GiB)": 39.61, "ppl": 1.28656, "step": 240, "tokens/total": 8338016, "tokens/train_per_sec_per_gpu": 54.02, "tokens/trainable": 252361} +{"epoch": 0.94140625, "grad_norm": 1.3728814125061035, "learning_rate": 6.32267616243259e-05, "loss": 0.23396289348602295, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.2636, "step": 241, "tokens/total": 8370608, "tokens/train_per_sec_per_gpu": 63.47, "tokens/trainable": 253391} +{"epoch": 0.9453125, "grad_norm": 1.3628835678100586, "learning_rate": 6.294119380711849e-05, "loss": 0.26438990235328674, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.30264, "step": 242, "tokens/total": 8405040, "tokens/train_per_sec_per_gpu": 53.32, "tokens/trainable": 254355} +{"epoch": 0.94921875, "grad_norm": 1.9355357885360718, "learning_rate": 6.265529552442209e-05, "loss": 0.2353588044643402, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.26536, "step": 243, "tokens/total": 8440400, "tokens/train_per_sec_per_gpu": 62.94, "tokens/trainable": 255455} +{"epoch": 0.953125, "grad_norm": 1.3816211223602295, "learning_rate": 6.236907867363127e-05, "loss": 0.22079284489154816, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.14, "memory/max_allocated (GiB)": 38.14, "ppl": 1.24707, "step": 244, "tokens/total": 8474736, "tokens/train_per_sec_per_gpu": 63.34, "tokens/trainable": 256489} +{"epoch": 0.95703125, "grad_norm": 1.2474626302719116, "learning_rate": 6.208255516539749e-05, "loss": 0.20999976992607117, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 36.34, "memory/max_allocated (GiB)": 36.34, "ppl": 1.23368, "step": 245, "tokens/total": 8506944, "tokens/train_per_sec_per_gpu": 60.48, "tokens/trainable": 257546} +{"epoch": 0.9609375, "grad_norm": 1.5191254615783691, "learning_rate": 6.179573692313344e-05, "loss": 0.22871485352516174, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 1.25698, "step": 246, "tokens/total": 8544064, "tokens/train_per_sec_per_gpu": 57.72, "tokens/trainable": 258572} +{"epoch": 0.96484375, "grad_norm": 1.951849341392517, "learning_rate": 6.150863588251694e-05, "loss": 0.24860695004463196, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.28224, "step": 247, "tokens/total": 8584336, "tokens/train_per_sec_per_gpu": 50.29, "tokens/trainable": 259638} +{"epoch": 0.96875, "grad_norm": 1.4411249160766602, "learning_rate": 6.122126399099419e-05, "loss": 0.21161232888698578, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.78, "memory/max_allocated (GiB)": 39.78, "ppl": 1.23567, "step": 248, "tokens/total": 8624304, "tokens/train_per_sec_per_gpu": 66.36, "tokens/trainable": 260894} +{"epoch": 0.97265625, "grad_norm": 1.5682579278945923, "learning_rate": 6.0933633207282615e-05, "loss": 0.25129371881484985, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.28569, "step": 249, "tokens/total": 8660672, "tokens/train_per_sec_per_gpu": 54.8, "tokens/trainable": 261961} +{"epoch": 0.9765625, "grad_norm": 0.9409470558166504, "learning_rate": 6.064575550087316e-05, "loss": 0.2211163341999054, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.63, "memory/max_allocated (GiB)": 39.63, "ppl": 1.24747, "step": 250, "tokens/total": 8697680, "tokens/train_per_sec_per_gpu": 61.0, "tokens/trainable": 263037} +{"epoch": 0.98046875, "grad_norm": 1.3633466958999634, "learning_rate": 6.0357642851532245e-05, "loss": 0.22292673587799072, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.24973, "step": 251, "tokens/total": 8734112, "tokens/train_per_sec_per_gpu": 54.86, "tokens/trainable": 264105} +{"epoch": 0.984375, "grad_norm": 1.6783117055892944, "learning_rate": 6.0069307248803294e-05, "loss": 0.26258689165115356, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.30029, "step": 252, "tokens/total": 8769440, "tokens/train_per_sec_per_gpu": 50.39, "tokens/trainable": 265091} +{"epoch": 0.98828125, "grad_norm": 1.1921052932739258, "learning_rate": 5.9780760691507635e-05, "loss": 0.250457227230072, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 36.25, "memory/max_allocated (GiB)": 36.25, "ppl": 1.28461, "step": 253, "tokens/total": 8800688, "tokens/train_per_sec_per_gpu": 63.98, "tokens/trainable": 266053} +{"epoch": 0.9921875, "grad_norm": 1.2071069478988647, "learning_rate": 5.9492015187245334e-05, "loss": 0.22608932852745056, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.72, "memory/max_allocated (GiB)": 38.72, "ppl": 1.25369, "step": 254, "tokens/total": 8838272, "tokens/train_per_sec_per_gpu": 54.7, "tokens/trainable": 267018} +{"epoch": 0.99609375, "grad_norm": 1.1712905168533325, "learning_rate": 5.920308275189541e-05, "loss": 0.23956620693206787, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 36.6, "memory/max_allocated (GiB)": 36.6, "ppl": 1.2707, "step": 255, "tokens/total": 8869952, "tokens/train_per_sec_per_gpu": 65.52, "tokens/trainable": 268040} +{"epoch": 1.0, "grad_norm": 0.9630696177482605, "learning_rate": 5.8913975409115874e-05, "loss": 0.20104748010635376, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.22268, "step": 256, "tokens/total": 8903312, "tokens/train_per_sec_per_gpu": 62.22, "tokens/trainable": 269099} +{"epoch": 1.00390625, "grad_norm": 1.558754801750183, "learning_rate": 5.8624705189843395e-05, "loss": 0.18275399506092072, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.20052, "step": 257, "tokens/total": 8940768, "tokens/train_per_sec_per_gpu": 55.61, "tokens/trainable": 270206} +{"epoch": 1.0078125, "grad_norm": 1.5230348110198975, "learning_rate": 5.833528413179249e-05, "loss": 0.1950812041759491, "memory/device_reserved (GiB)": 40.67, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.21541, "step": 258, "tokens/total": 8977392, "tokens/train_per_sec_per_gpu": 58.57, "tokens/trainable": 271305} +{"epoch": 1.01171875, "grad_norm": 0.621518075466156, "learning_rate": 5.80457242789548e-05, "loss": 0.169869527220726, "memory/device_reserved (GiB)": 42.74, "memory/max_active (GiB)": 39.65, "memory/max_allocated (GiB)": 39.65, "ppl": 1.18515, "step": 259, "tokens/total": 9017568, "tokens/train_per_sec_per_gpu": 57.31, "tokens/trainable": 272486} +{"epoch": 1.015625, "grad_norm": 0.8507027626037598, "learning_rate": 5.77560376810977e-05, "loss": 0.1784990280866623, "memory/device_reserved (GiB)": 42.75, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.19542, "step": 260, "tokens/total": 9050928, "tokens/train_per_sec_per_gpu": 70.96, "tokens/trainable": 273635} +{"epoch": 1.01953125, "grad_norm": 1.4061849117279053, "learning_rate": 5.7466236393263005e-05, "loss": 0.21549955010414124, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.24048, "step": 261, "tokens/total": 9085552, "tokens/train_per_sec_per_gpu": 63.49, "tokens/trainable": 274678} +{"epoch": 1.0234375, "grad_norm": 1.4708701372146606, "learning_rate": 5.717633247526522e-05, "loss": 0.21558284759521484, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 36.74, "memory/max_allocated (GiB)": 36.74, "ppl": 1.24058, "step": 262, "tokens/total": 9116240, "tokens/train_per_sec_per_gpu": 60.14, "tokens/trainable": 275631} +{"epoch": 1.02734375, "grad_norm": 1.8110164403915405, "learning_rate": 5.688633799118971e-05, "loss": 0.2359568178653717, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.26612, "step": 263, "tokens/total": 9148688, "tokens/train_per_sec_per_gpu": 55.78, "tokens/trainable": 276571} +{"epoch": 1.03125, "grad_norm": 1.2641578912734985, "learning_rate": 5.659626500889066e-05, "loss": 0.1866876631975174, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.09, "memory/max_allocated (GiB)": 37.09, "ppl": 1.20525, "step": 264, "tokens/total": 9182704, "tokens/train_per_sec_per_gpu": 64.6, "tokens/trainable": 277697} +{"epoch": 1.03515625, "grad_norm": 2.252211809158325, "learning_rate": 5.6306125599488905e-05, "loss": 0.21614974737167358, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 36.88, "memory/max_allocated (GiB)": 36.88, "ppl": 1.24129, "step": 265, "tokens/total": 9213552, "tokens/train_per_sec_per_gpu": 69.45, "tokens/trainable": 278717} +{"epoch": 1.0390625, "grad_norm": 2.1934452056884766, "learning_rate": 5.601593183686955e-05, "loss": 0.16180868446826935, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.17564, "step": 266, "tokens/total": 9250768, "tokens/train_per_sec_per_gpu": 59.83, "tokens/trainable": 279836} +{"epoch": 1.04296875, "grad_norm": 1.6655023097991943, "learning_rate": 5.572569579717961e-05, "loss": 0.19145196676254272, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.21101, "step": 267, "tokens/total": 9286176, "tokens/train_per_sec_per_gpu": 64.24, "tokens/trainable": 280948} +{"epoch": 1.046875, "grad_norm": 0.8135383725166321, "learning_rate": 5.543542955832538e-05, "loss": 0.17947900295257568, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.19659, "step": 268, "tokens/total": 9326128, "tokens/train_per_sec_per_gpu": 52.68, "tokens/trainable": 282031} +{"epoch": 1.05078125, "grad_norm": 4.260921478271484, "learning_rate": 5.514514519946986e-05, "loss": 0.18033719062805176, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.57, "memory/max_allocated (GiB)": 39.57, "ppl": 1.19762, "step": 269, "tokens/total": 9366544, "tokens/train_per_sec_per_gpu": 56.96, "tokens/trainable": 283178} +{"epoch": 1.0546875, "grad_norm": 1.3062560558319092, "learning_rate": 5.485485480053015e-05, "loss": 0.2366878092288971, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.52, "memory/max_allocated (GiB)": 39.52, "ppl": 1.26705, "step": 270, "tokens/total": 9404928, "tokens/train_per_sec_per_gpu": 49.37, "tokens/trainable": 284180} +{"epoch": 1.05859375, "grad_norm": 1.179249882698059, "learning_rate": 5.4564570441674645e-05, "loss": 0.19321313500404358, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.21314, "step": 271, "tokens/total": 9438592, "tokens/train_per_sec_per_gpu": 67.54, "tokens/trainable": 285227} +{"epoch": 1.0625, "grad_norm": 1.3026138544082642, "learning_rate": 5.42743042028204e-05, "loss": 0.22568932175636292, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.83, "memory/max_allocated (GiB)": 37.83, "ppl": 1.25319, "step": 272, "tokens/total": 9474144, "tokens/train_per_sec_per_gpu": 48.19, "tokens/trainable": 286192} +{"epoch": 1.06640625, "grad_norm": 1.1601618528366089, "learning_rate": 5.3984068163130464e-05, "loss": 0.1861947476863861, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.20466, "step": 273, "tokens/total": 9507904, "tokens/train_per_sec_per_gpu": 69.26, "tokens/trainable": 287369} +{"epoch": 1.0703125, "grad_norm": 1.5395210981369019, "learning_rate": 5.369387440051111e-05, "loss": 0.2047434151172638, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.18, "memory/max_allocated (GiB)": 37.18, "ppl": 1.22721, "step": 274, "tokens/total": 9541616, "tokens/train_per_sec_per_gpu": 58.75, "tokens/trainable": 288371} +{"epoch": 1.07421875, "grad_norm": 1.2776927947998047, "learning_rate": 5.340373499110935e-05, "loss": 0.19290786981582642, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.21277, "step": 275, "tokens/total": 9577424, "tokens/train_per_sec_per_gpu": 59.7, "tokens/trainable": 289456} +{"epoch": 1.078125, "grad_norm": 0.8201732039451599, "learning_rate": 5.3113662008810304e-05, "loss": 0.16939598321914673, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.18459, "step": 276, "tokens/total": 9617760, "tokens/train_per_sec_per_gpu": 50.39, "tokens/trainable": 290516} +{"epoch": 1.08203125, "grad_norm": 0.906524121761322, "learning_rate": 5.282366752473479e-05, "loss": 0.18742021918296814, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.29, "memory/max_allocated (GiB)": 38.29, "ppl": 1.20613, "step": 277, "tokens/total": 9655008, "tokens/train_per_sec_per_gpu": 60.11, "tokens/trainable": 291636} +{"epoch": 1.0859375, "grad_norm": 1.0511525869369507, "learning_rate": 5.2533763606737005e-05, "loss": 0.22540006041526794, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.28, "memory/max_allocated (GiB)": 39.28, "ppl": 1.25282, "step": 278, "tokens/total": 9694896, "tokens/train_per_sec_per_gpu": 49.85, "tokens/trainable": 292677} +{"epoch": 1.08984375, "grad_norm": 1.694703221321106, "learning_rate": 5.224396231890232e-05, "loss": 0.1848444938659668, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.23, "memory/max_allocated (GiB)": 37.23, "ppl": 1.20303, "step": 279, "tokens/total": 9726672, "tokens/train_per_sec_per_gpu": 67.78, "tokens/trainable": 293800} +{"epoch": 1.09375, "grad_norm": 1.487571120262146, "learning_rate": 5.195427572104522e-05, "loss": 0.20605561137199402, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.22882, "step": 280, "tokens/total": 9761136, "tokens/train_per_sec_per_gpu": 60.75, "tokens/trainable": 294843} +{"epoch": 1.09765625, "grad_norm": 1.6700867414474487, "learning_rate": 5.166471586820751e-05, "loss": 0.20168699324131012, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.22346, "step": 281, "tokens/total": 9797888, "tokens/train_per_sec_per_gpu": 61.13, "tokens/trainable": 295941} +{"epoch": 1.1015625, "grad_norm": 1.7702339887619019, "learning_rate": 5.1375294810156615e-05, "loss": 0.18183019757270813, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.19941, "step": 282, "tokens/total": 9832528, "tokens/train_per_sec_per_gpu": 59.71, "tokens/trainable": 296963} +{"epoch": 1.10546875, "grad_norm": 1.1718589067459106, "learning_rate": 5.1086024590884144e-05, "loss": 0.19491666555404663, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.28, "memory/max_allocated (GiB)": 39.28, "ppl": 1.21521, "step": 283, "tokens/total": 9870784, "tokens/train_per_sec_per_gpu": 52.78, "tokens/trainable": 297964} +{"epoch": 1.109375, "grad_norm": 1.0932633876800537, "learning_rate": 5.079691724810461e-05, "loss": 0.18084251880645752, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.19823, "step": 284, "tokens/total": 9906656, "tokens/train_per_sec_per_gpu": 49.23, "tokens/trainable": 298933} +{"epoch": 1.11328125, "grad_norm": 1.4519668817520142, "learning_rate": 5.0507984812754684e-05, "loss": 0.2082625925540924, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.23154, "step": 285, "tokens/total": 9940208, "tokens/train_per_sec_per_gpu": 59.69, "tokens/trainable": 299980} +{"epoch": 1.1171875, "grad_norm": 0.8104725480079651, "learning_rate": 5.021923930849237e-05, "loss": 0.18605856597423553, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.20449, "step": 286, "tokens/total": 9974096, "tokens/train_per_sec_per_gpu": 56.77, "tokens/trainable": 300993} +{"epoch": 1.12109375, "grad_norm": 1.422316551208496, "learning_rate": 4.99306927511967e-05, "loss": 0.18900296092033386, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.20804, "step": 287, "tokens/total": 10007568, "tokens/train_per_sec_per_gpu": 62.32, "tokens/trainable": 302000} +{"epoch": 1.125, "grad_norm": 1.1420142650604248, "learning_rate": 4.964235714846775e-05, "loss": 0.19955438375473022, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 36.44, "memory/max_allocated (GiB)": 36.44, "ppl": 1.22086, "step": 288, "tokens/total": 10039280, "tokens/train_per_sec_per_gpu": 66.34, "tokens/trainable": 302994} +{"epoch": 1.12890625, "grad_norm": 6.9964399337768555, "learning_rate": 4.9354244499126866e-05, "loss": 0.21712517738342285, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 36.23, "memory/max_allocated (GiB)": 36.23, "ppl": 1.2425, "step": 289, "tokens/total": 10070944, "tokens/train_per_sec_per_gpu": 63.16, "tokens/trainable": 304026} +{"epoch": 1.1328125, "grad_norm": 0.949253499507904, "learning_rate": 4.90663667927174e-05, "loss": 0.18277840316295624, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.20055, "step": 290, "tokens/total": 10103776, "tokens/train_per_sec_per_gpu": 71.14, "tokens/trainable": 305053} +{"epoch": 1.13671875, "grad_norm": 0.9207992553710938, "learning_rate": 4.877873600900581e-05, "loss": 0.17694754898548126, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.07, "memory/max_allocated (GiB)": 38.07, "ppl": 1.19357, "step": 291, "tokens/total": 10139888, "tokens/train_per_sec_per_gpu": 55.7, "tokens/trainable": 306089} +{"epoch": 1.140625, "grad_norm": 1.1790170669555664, "learning_rate": 4.849136411748306e-05, "loss": 0.17714864015579224, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.04, "memory/max_allocated (GiB)": 39.04, "ppl": 1.19381, "step": 292, "tokens/total": 10177936, "tokens/train_per_sec_per_gpu": 53.67, "tokens/trainable": 307146} +{"epoch": 1.14453125, "grad_norm": 1.0939669609069824, "learning_rate": 4.8204263076866574e-05, "loss": 0.18501542508602142, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.20324, "step": 293, "tokens/total": 10213856, "tokens/train_per_sec_per_gpu": 57.58, "tokens/trainable": 308208} +{"epoch": 1.1484375, "grad_norm": 1.0199003219604492, "learning_rate": 4.791744483460251e-05, "loss": 0.1986631453037262, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.74, "memory/max_allocated (GiB)": 37.74, "ppl": 1.21977, "step": 294, "tokens/total": 10248352, "tokens/train_per_sec_per_gpu": 51.78, "tokens/trainable": 309202} +{"epoch": 1.15234375, "grad_norm": 0.8870011568069458, "learning_rate": 4.7630921326368736e-05, "loss": 0.18075266480445862, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.19812, "step": 295, "tokens/total": 10287072, "tokens/train_per_sec_per_gpu": 50.09, "tokens/trainable": 310184} +{"epoch": 1.15625, "grad_norm": 3.630218029022217, "learning_rate": 4.7344704475577916e-05, "loss": 0.18445706367492676, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 35.7, "memory/max_allocated (GiB)": 35.7, "ppl": 1.20257, "step": 296, "tokens/total": 10316640, "tokens/train_per_sec_per_gpu": 68.26, "tokens/trainable": 311210} +{"epoch": 1.16015625, "grad_norm": 1.6531344652175903, "learning_rate": 4.705880619288153e-05, "loss": 0.20356453955173492, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.37, "memory/max_allocated (GiB)": 38.37, "ppl": 1.22576, "step": 297, "tokens/total": 10352864, "tokens/train_per_sec_per_gpu": 53.03, "tokens/trainable": 312195} +{"epoch": 1.1640625, "grad_norm": 1.4295257329940796, "learning_rate": 4.677323837567412e-05, "loss": 0.17609372735023499, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.19255, "step": 298, "tokens/total": 10385696, "tokens/train_per_sec_per_gpu": 64.63, "tokens/trainable": 313251} +{"epoch": 1.16796875, "grad_norm": 1.7515017986297607, "learning_rate": 4.6488012907598146e-05, "loss": 0.20083209872245789, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 36.4, "memory/max_allocated (GiB)": 36.4, "ppl": 1.22242, "step": 299, "tokens/total": 10417904, "tokens/train_per_sec_per_gpu": 57.39, "tokens/trainable": 314206} +{"epoch": 1.171875, "grad_norm": 1.5793521404266357, "learning_rate": 4.620314165804964e-05, "loss": 0.16867156326770782, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.18373, "step": 300, "tokens/total": 10453728, "tokens/train_per_sec_per_gpu": 58.52, "tokens/trainable": 315302} +{"epoch": 1.17578125, "grad_norm": 1.3328040838241577, "learning_rate": 4.591863648168407e-05, "loss": 0.17760351300239563, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.19435, "step": 301, "tokens/total": 10488288, "tokens/train_per_sec_per_gpu": 59.33, "tokens/trainable": 316419} +{"epoch": 1.1796875, "grad_norm": 1.1463513374328613, "learning_rate": 4.5634509217923135e-05, "loss": 0.19455239176750183, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.96, "memory/max_allocated (GiB)": 37.96, "ppl": 1.21477, "step": 302, "tokens/total": 10521824, "tokens/train_per_sec_per_gpu": 61.08, "tokens/trainable": 317457} +{"epoch": 1.18359375, "grad_norm": 1.4614620208740234, "learning_rate": 4.535077169046201e-05, "loss": 0.20439419150352478, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.03, "memory/max_allocated (GiB)": 38.03, "ppl": 1.22678, "step": 303, "tokens/total": 10558432, "tokens/train_per_sec_per_gpu": 57.07, "tokens/trainable": 318487} +{"epoch": 1.1875, "grad_norm": 1.732301115989685, "learning_rate": 4.506743570677743e-05, "loss": 0.20768341422080994, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 36.35, "memory/max_allocated (GiB)": 36.35, "ppl": 1.23082, "step": 304, "tokens/total": 10588752, "tokens/train_per_sec_per_gpu": 71.91, "tokens/trainable": 319495} +{"epoch": 1.19140625, "grad_norm": 2.033424139022827, "learning_rate": 4.478451305763618e-05, "loss": 0.19506709277629852, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.21539, "step": 305, "tokens/total": 10622336, "tokens/train_per_sec_per_gpu": 64.02, "tokens/trainable": 320533} +{"epoch": 1.1953125, "grad_norm": 1.128240942955017, "learning_rate": 4.450201551660454e-05, "loss": 0.1865515410900116, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 36.41, "memory/max_allocated (GiB)": 36.41, "ppl": 1.20509, "step": 306, "tokens/total": 10653520, "tokens/train_per_sec_per_gpu": 63.37, "tokens/trainable": 321538} +{"epoch": 1.19921875, "grad_norm": 1.3104983568191528, "learning_rate": 4.4219954839558276e-05, "loss": 0.19414865970611572, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.21428, "step": 307, "tokens/total": 10688144, "tokens/train_per_sec_per_gpu": 53.81, "tokens/trainable": 322550} +{"epoch": 1.203125, "grad_norm": 1.7645847797393799, "learning_rate": 4.393834276419352e-05, "loss": 0.2110670804977417, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 35.7, "memory/max_allocated (GiB)": 35.7, "ppl": 1.235, "step": 308, "tokens/total": 10718032, "tokens/train_per_sec_per_gpu": 72.15, "tokens/trainable": 323588} +{"epoch": 1.20703125, "grad_norm": 0.9882633090019226, "learning_rate": 4.36571910095382e-05, "loss": 0.22518841922283173, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.25256, "step": 309, "tokens/total": 10756464, "tokens/train_per_sec_per_gpu": 55.15, "tokens/trainable": 324662} +{"epoch": 1.2109375, "grad_norm": 1.062860131263733, "learning_rate": 4.337651127546448e-05, "loss": 0.20930339395999908, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.23282, "step": 310, "tokens/total": 10792064, "tokens/train_per_sec_per_gpu": 63.39, "tokens/trainable": 325723} +{"epoch": 1.21484375, "grad_norm": 2.6870481967926025, "learning_rate": 4.3096315242201736e-05, "loss": 0.2501014471054077, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 36.51, "memory/max_allocated (GiB)": 36.51, "ppl": 1.28416, "step": 311, "tokens/total": 10824800, "tokens/train_per_sec_per_gpu": 58.97, "tokens/trainable": 326771} +{"epoch": 1.21875, "grad_norm": 0.9839888215065002, "learning_rate": 4.2816614569850635e-05, "loss": 0.1891273707151413, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 38.66, "memory/max_allocated (GiB)": 38.66, "ppl": 1.20819, "step": 312, "tokens/total": 10860576, "tokens/train_per_sec_per_gpu": 58.21, "tokens/trainable": 327771} +{"epoch": 1.22265625, "grad_norm": 0.9761978983879089, "learning_rate": 4.2537420897897864e-05, "loss": 0.193028062582016, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 38.72, "memory/max_allocated (GiB)": 38.72, "ppl": 1.21292, "step": 313, "tokens/total": 10896224, "tokens/train_per_sec_per_gpu": 53.74, "tokens/trainable": 328807} +{"epoch": 1.2265625, "grad_norm": 1.5201936960220337, "learning_rate": 4.225874584473174e-05, "loss": 0.20629960298538208, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.22912, "step": 314, "tokens/total": 10928768, "tokens/train_per_sec_per_gpu": 63.9, "tokens/trainable": 329821} +{"epoch": 1.23046875, "grad_norm": 0.8478052616119385, "learning_rate": 4.19806010071587e-05, "loss": 0.17550094425678253, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.19184, "step": 315, "tokens/total": 10966848, "tokens/train_per_sec_per_gpu": 54.99, "tokens/trainable": 330917} +{"epoch": 1.234375, "grad_norm": 2.3956263065338135, "learning_rate": 4.170299795992081e-05, "loss": 0.2121603786945343, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 36.88, "memory/max_allocated (GiB)": 36.88, "ppl": 1.23635, "step": 316, "tokens/total": 11000416, "tokens/train_per_sec_per_gpu": 60.25, "tokens/trainable": 331979} +{"epoch": 1.23828125, "grad_norm": 0.9092263579368591, "learning_rate": 4.142594825521398e-05, "loss": 0.16445007920265198, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.17874, "step": 317, "tokens/total": 11039520, "tokens/train_per_sec_per_gpu": 51.39, "tokens/trainable": 333036} +{"epoch": 1.2421875, "grad_norm": 0.7330667972564697, "learning_rate": 4.114946342220728e-05, "loss": 0.17451906204223633, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.19067, "step": 318, "tokens/total": 11076096, "tokens/train_per_sec_per_gpu": 58.36, "tokens/trainable": 334136} +{"epoch": 1.24609375, "grad_norm": 1.1156108379364014, "learning_rate": 4.087355496656321e-05, "loss": 0.1683889627456665, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.1834, "step": 319, "tokens/total": 11109728, "tokens/train_per_sec_per_gpu": 69.12, "tokens/trainable": 335262} +{"epoch": 1.25, "grad_norm": 0.8259012699127197, "learning_rate": 4.05982343699588e-05, "loss": 0.18647156655788422, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.20499, "step": 320, "tokens/total": 11148976, "tokens/train_per_sec_per_gpu": 59.44, "tokens/trainable": 336421} +{"epoch": 1.25390625, "grad_norm": 0.9085292816162109, "learning_rate": 4.0323513089607876e-05, "loss": 0.18057510256767273, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.19791, "step": 321, "tokens/total": 11183888, "tokens/train_per_sec_per_gpu": 57.66, "tokens/trainable": 337517} +{"epoch": 1.2578125, "grad_norm": 1.6273151636123657, "learning_rate": 4.004940255778431e-05, "loss": 0.181340754032135, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 39.63, "memory/max_allocated (GiB)": 39.63, "ppl": 1.19882, "step": 322, "tokens/total": 11222608, "tokens/train_per_sec_per_gpu": 56.66, "tokens/trainable": 338574} +{"epoch": 1.26171875, "grad_norm": 0.9140313267707825, "learning_rate": 3.977591418134619e-05, "loss": 0.18408235907554626, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.20211, "step": 323, "tokens/total": 11257568, "tokens/train_per_sec_per_gpu": 62.47, "tokens/trainable": 339635} +{"epoch": 1.265625, "grad_norm": 1.1206724643707275, "learning_rate": 3.95030593412612e-05, "loss": 0.19450782239437103, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.21471, "step": 324, "tokens/total": 11293472, "tokens/train_per_sec_per_gpu": 57.32, "tokens/trainable": 340651} +{"epoch": 1.26953125, "grad_norm": 1.248779058456421, "learning_rate": 3.923084939213296e-05, "loss": 0.21191450953483582, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 35.75, "memory/max_allocated (GiB)": 35.75, "ppl": 1.23604, "step": 325, "tokens/total": 11324048, "tokens/train_per_sec_per_gpu": 68.17, "tokens/trainable": 341702} +{"epoch": 1.2734375, "grad_norm": 0.6210129857063293, "learning_rate": 3.895929566172861e-05, "loss": 0.1492980271577835, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.16102, "step": 326, "tokens/total": 11360064, "tokens/train_per_sec_per_gpu": 60.23, "tokens/trainable": 342806} +{"epoch": 1.27734375, "grad_norm": 0.7209714651107788, "learning_rate": 3.868840945050728e-05, "loss": 0.16068215668201447, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 39.19, "memory/max_allocated (GiB)": 39.19, "ppl": 1.17431, "step": 327, "tokens/total": 11399008, "tokens/train_per_sec_per_gpu": 56.12, "tokens/trainable": 343895} +{"epoch": 1.28125, "grad_norm": 1.6011706590652466, "learning_rate": 3.841820203114995e-05, "loss": 0.1859949231147766, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 36.78, "memory/max_allocated (GiB)": 36.78, "ppl": 1.20442, "step": 328, "tokens/total": 11430752, "tokens/train_per_sec_per_gpu": 67.45, "tokens/trainable": 344994} +{"epoch": 1.28515625, "grad_norm": 0.7126188278198242, "learning_rate": 3.814868464809027e-05, "loss": 0.17336104810237885, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 36.77, "memory/max_allocated (GiB)": 36.77, "ppl": 1.1893, "step": 329, "tokens/total": 11462688, "tokens/train_per_sec_per_gpu": 61.29, "tokens/trainable": 345990} +{"epoch": 1.2890625, "grad_norm": 0.9588814973831177, "learning_rate": 3.787986851704667e-05, "loss": 0.21305714547634125, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.23746, "step": 330, "tokens/total": 11496240, "tokens/train_per_sec_per_gpu": 67.39, "tokens/trainable": 347071} +{"epoch": 1.29296875, "grad_norm": 1.0668026208877563, "learning_rate": 3.7611764824555654e-05, "loss": 0.15895383059978485, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 37.84, "memory/max_allocated (GiB)": 37.84, "ppl": 1.17228, "step": 331, "tokens/total": 11527968, "tokens/train_per_sec_per_gpu": 76.06, "tokens/trainable": 348224} +{"epoch": 1.296875, "grad_norm": 2.2679519653320312, "learning_rate": 3.734438472750619e-05, "loss": 0.1993274688720703, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.22058, "step": 332, "tokens/total": 11567936, "tokens/train_per_sec_per_gpu": 53.47, "tokens/trainable": 349352} +{"epoch": 1.30078125, "grad_norm": 0.8628775477409363, "learning_rate": 3.707773935267552e-05, "loss": 0.1741744577884674, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.19026, "step": 333, "tokens/total": 11604992, "tokens/train_per_sec_per_gpu": 57.85, "tokens/trainable": 350479} +{"epoch": 1.3046875, "grad_norm": 2.261693000793457, "learning_rate": 3.68118397962661e-05, "loss": 0.20431667566299438, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 38.49, "memory/max_allocated (GiB)": 38.49, "ppl": 1.22669, "step": 334, "tokens/total": 11639744, "tokens/train_per_sec_per_gpu": 55.78, "tokens/trainable": 351441} +{"epoch": 1.30859375, "grad_norm": 0.9312662482261658, "learning_rate": 3.654669712344384e-05, "loss": 0.161618173122406, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 36.72, "memory/max_allocated (GiB)": 36.72, "ppl": 1.17541, "step": 335, "tokens/total": 11670944, "tokens/train_per_sec_per_gpu": 74.76, "tokens/trainable": 352497} +{"epoch": 1.3125, "grad_norm": 1.7313531637191772, "learning_rate": 3.628232236787763e-05, "loss": 0.1771678924560547, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 37.91, "memory/max_allocated (GiB)": 37.91, "ppl": 1.19383, "step": 336, "tokens/total": 11706544, "tokens/train_per_sec_per_gpu": 56.2, "tokens/trainable": 353610} +{"epoch": 1.31640625, "grad_norm": 2.602452516555786, "learning_rate": 3.6018726531280144e-05, "loss": 0.1633429229259491, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.17744, "step": 337, "tokens/total": 11743952, "tokens/train_per_sec_per_gpu": 60.35, "tokens/trainable": 354725} +{"epoch": 1.3203125, "grad_norm": 3.5711092948913574, "learning_rate": 3.575592058295017e-05, "loss": 0.23515857756137848, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.26511, "step": 338, "tokens/total": 11777936, "tokens/train_per_sec_per_gpu": 63.71, "tokens/trainable": 355802} +{"epoch": 1.32421875, "grad_norm": 1.2820754051208496, "learning_rate": 3.549391545931585e-05, "loss": 0.17786361277103424, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 38.93, "memory/max_allocated (GiB)": 38.93, "ppl": 1.19466, "step": 339, "tokens/total": 11812416, "tokens/train_per_sec_per_gpu": 62.37, "tokens/trainable": 356852} +{"epoch": 1.328125, "grad_norm": 1.2832839488983154, "learning_rate": 3.5232722063479914e-05, "loss": 0.1837463676929474, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.20171, "step": 340, "tokens/total": 11846704, "tokens/train_per_sec_per_gpu": 66.45, "tokens/trainable": 357927} +{"epoch": 1.33203125, "grad_norm": 0.754531979560852, "learning_rate": 3.49723512647657e-05, "loss": 0.16970936954021454, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.18496, "step": 341, "tokens/total": 11885968, "tokens/train_per_sec_per_gpu": 54.13, "tokens/trainable": 359043} +{"epoch": 1.3359375, "grad_norm": 3.4829723834991455, "learning_rate": 3.471281389826491e-05, "loss": 0.18400117754936218, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.20202, "step": 342, "tokens/total": 11917024, "tokens/train_per_sec_per_gpu": 63.26, "tokens/trainable": 359996} +{"epoch": 1.33984375, "grad_norm": 0.8671007752418518, "learning_rate": 3.4454120764386764e-05, "loss": 0.1646222025156021, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.17895, "step": 343, "tokens/total": 11955648, "tokens/train_per_sec_per_gpu": 56.64, "tokens/trainable": 361076} +{"epoch": 1.34375, "grad_norm": 0.8904309272766113, "learning_rate": 3.4196282628408526e-05, "loss": 0.18425899744033813, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 37.58, "memory/max_allocated (GiB)": 37.58, "ppl": 1.20233, "step": 344, "tokens/total": 11989936, "tokens/train_per_sec_per_gpu": 62.08, "tokens/trainable": 362134} +{"epoch": 1.34765625, "grad_norm": 1.1649881601333618, "learning_rate": 3.3939310220027456e-05, "loss": 0.1839999556541443, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.20202, "step": 345, "tokens/total": 12023968, "tokens/train_per_sec_per_gpu": 60.76, "tokens/trainable": 363229} +{"epoch": 1.3515625, "grad_norm": 0.696380078792572, "learning_rate": 3.3683214232914404e-05, "loss": 0.15968380868434906, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.17314, "step": 346, "tokens/total": 12059632, "tokens/train_per_sec_per_gpu": 59.76, "tokens/trainable": 364309} +{"epoch": 1.35546875, "grad_norm": 0.9706635475158691, "learning_rate": 3.342800532426873e-05, "loss": 0.18654397130012512, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.20508, "step": 347, "tokens/total": 12095616, "tokens/train_per_sec_per_gpu": 59.94, "tokens/trainable": 365327} +{"epoch": 1.359375, "grad_norm": 1.4170857667922974, "learning_rate": 3.317369411437484e-05, "loss": 0.20527341961860657, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.22786, "step": 348, "tokens/total": 12131488, "tokens/train_per_sec_per_gpu": 55.99, "tokens/trainable": 366371} +{"epoch": 1.36328125, "grad_norm": 1.1486963033676147, "learning_rate": 3.292029118616024e-05, "loss": 0.18026335537433624, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.19753, "step": 349, "tokens/total": 12164208, "tokens/train_per_sec_per_gpu": 56.94, "tokens/trainable": 367353} +{"epoch": 1.3671875, "grad_norm": 0.7787570953369141, "learning_rate": 3.266780708475511e-05, "loss": 0.17446410655975342, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 39.12, "memory/max_allocated (GiB)": 39.12, "ppl": 1.19061, "step": 350, "tokens/total": 12199616, "tokens/train_per_sec_per_gpu": 58.32, "tokens/trainable": 368428} +{"epoch": 1.37109375, "grad_norm": 1.2692019939422607, "learning_rate": 3.241625231705354e-05, "loss": 0.1741599440574646, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.19025, "step": 351, "tokens/total": 12235664, "tokens/train_per_sec_per_gpu": 59.95, "tokens/trainable": 369498} +{"epoch": 1.375, "grad_norm": 1.8785144090652466, "learning_rate": 3.216563735127618e-05, "loss": 0.1569124460220337, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 39.12, "memory/max_allocated (GiB)": 39.12, "ppl": 1.16989, "step": 352, "tokens/total": 12270800, "tokens/train_per_sec_per_gpu": 61.03, "tokens/trainable": 370631} +{"epoch": 1.37890625, "grad_norm": 0.9826963543891907, "learning_rate": 3.191597261653475e-05, "loss": 0.22875267267227173, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.25703, "step": 353, "tokens/total": 12307568, "tokens/train_per_sec_per_gpu": 57.69, "tokens/trainable": 371717} +{"epoch": 1.3828125, "grad_norm": 1.178210973739624, "learning_rate": 3.166726850239794e-05, "loss": 0.16744768619537354, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 37.18, "memory/max_allocated (GiB)": 37.18, "ppl": 1.18228, "step": 354, "tokens/total": 12341200, "tokens/train_per_sec_per_gpu": 62.49, "tokens/trainable": 372795} +{"epoch": 1.38671875, "grad_norm": 1.4414113759994507, "learning_rate": 3.141953535845912e-05, "loss": 0.21345897018909454, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 36.16, "memory/max_allocated (GiB)": 36.16, "ppl": 1.23795, "step": 355, "tokens/total": 12372016, "tokens/train_per_sec_per_gpu": 62.97, "tokens/trainable": 373733} +{"epoch": 1.390625, "grad_norm": 1.00863516330719, "learning_rate": 3.11727834939056e-05, "loss": 0.18071305751800537, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 36.28, "memory/max_allocated (GiB)": 36.28, "ppl": 1.19807, "step": 356, "tokens/total": 12403856, "tokens/train_per_sec_per_gpu": 59.93, "tokens/trainable": 374767} +{"epoch": 1.39453125, "grad_norm": 0.987851619720459, "learning_rate": 3.092702317708967e-05, "loss": 0.19831430912017822, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.21935, "step": 357, "tokens/total": 12437584, "tokens/train_per_sec_per_gpu": 62.32, "tokens/trainable": 375792} +{"epoch": 1.3984375, "grad_norm": 1.2579941749572754, "learning_rate": 3.0682264635101276e-05, "loss": 0.198319673538208, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 37.44, "memory/max_allocated (GiB)": 37.44, "ppl": 1.21935, "step": 358, "tokens/total": 12470096, "tokens/train_per_sec_per_gpu": 62.05, "tokens/trainable": 376808} +{"epoch": 1.40234375, "grad_norm": 0.9534085988998413, "learning_rate": 3.0438518053342407e-05, "loss": 0.1654018610715866, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.17987, "step": 359, "tokens/total": 12509168, "tokens/train_per_sec_per_gpu": 56.11, "tokens/trainable": 377939} +{"epoch": 1.40625, "grad_norm": 1.5381147861480713, "learning_rate": 3.0195793575103266e-05, "loss": 0.1771574318408966, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.19382, "step": 360, "tokens/total": 12548208, "tokens/train_per_sec_per_gpu": 54.91, "tokens/trainable": 379015} +{"epoch": 1.41015625, "grad_norm": 1.090387225151062, "learning_rate": 2.9954101301140146e-05, "loss": 0.18389186263084412, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.20189, "step": 361, "tokens/total": 12583552, "tokens/train_per_sec_per_gpu": 58.78, "tokens/trainable": 380065} +{"epoch": 1.4140625, "grad_norm": 1.3390718698501587, "learning_rate": 2.9713451289255123e-05, "loss": 0.17600539326667786, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 36.9, "memory/max_allocated (GiB)": 36.9, "ppl": 1.19244, "step": 362, "tokens/total": 12612944, "tokens/train_per_sec_per_gpu": 63.89, "tokens/trainable": 381109} +{"epoch": 1.41796875, "grad_norm": 0.7409974932670593, "learning_rate": 2.9473853553877484e-05, "loss": 0.15837597846984863, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.17161, "step": 363, "tokens/total": 12646880, "tokens/train_per_sec_per_gpu": 60.27, "tokens/trainable": 382148} +{"epoch": 1.421875, "grad_norm": 6.681075096130371, "learning_rate": 2.9235318065647e-05, "loss": 0.1834494173526764, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 39.61, "memory/max_allocated (GiB)": 39.61, "ppl": 1.20135, "step": 364, "tokens/total": 12686896, "tokens/train_per_sec_per_gpu": 53.47, "tokens/trainable": 383216} +{"epoch": 1.42578125, "grad_norm": 0.8987640738487244, "learning_rate": 2.8997854750998964e-05, "loss": 0.17547279596328735, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.19181, "step": 365, "tokens/total": 12726416, "tokens/train_per_sec_per_gpu": 59.04, "tokens/trainable": 384335} +{"epoch": 1.4296875, "grad_norm": 1.0270642042160034, "learning_rate": 2.8761473491751258e-05, "loss": 0.1862422227859497, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.63, "memory/max_allocated (GiB)": 36.63, "ppl": 1.20471, "step": 366, "tokens/total": 12759248, "tokens/train_per_sec_per_gpu": 59.9, "tokens/trainable": 385333} +{"epoch": 1.43359375, "grad_norm": 0.61436527967453, "learning_rate": 2.8526184124692883e-05, "loss": 0.1432645469903946, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.15404, "step": 367, "tokens/total": 12795904, "tokens/train_per_sec_per_gpu": 52.58, "tokens/trainable": 386436} +{"epoch": 1.4375, "grad_norm": 1.0386468172073364, "learning_rate": 2.829199644117484e-05, "loss": 0.17491890490055084, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.19115, "step": 368, "tokens/total": 12829616, "tokens/train_per_sec_per_gpu": 62.87, "tokens/trainable": 387399} +{"epoch": 1.44140625, "grad_norm": 1.825620412826538, "learning_rate": 2.8058920186702553e-05, "loss": 0.15611308813095093, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.91, "memory/max_allocated (GiB)": 38.91, "ppl": 1.16896, "step": 369, "tokens/total": 12867744, "tokens/train_per_sec_per_gpu": 58.85, "tokens/trainable": 388527} +{"epoch": 1.4453125, "grad_norm": 1.899389624595642, "learning_rate": 2.782696506053033e-05, "loss": 0.18212319910526276, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.19976, "step": 370, "tokens/total": 12905936, "tokens/train_per_sec_per_gpu": 52.64, "tokens/trainable": 389522} +{"epoch": 1.44921875, "grad_norm": 1.2054774761199951, "learning_rate": 2.7596140715257824e-05, "loss": 0.16579236090183258, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.18033, "step": 371, "tokens/total": 12941216, "tokens/train_per_sec_per_gpu": 61.5, "tokens/trainable": 390627} +{"epoch": 1.453125, "grad_norm": 1.0018116235733032, "learning_rate": 2.7366456756428184e-05, "loss": 0.169950932264328, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.18525, "step": 372, "tokens/total": 12975584, "tokens/train_per_sec_per_gpu": 60.07, "tokens/trainable": 391663} +{"epoch": 1.45703125, "grad_norm": 1.102150797843933, "learning_rate": 2.7137922742128486e-05, "loss": 0.17588913440704346, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.21, "memory/max_allocated (GiB)": 38.21, "ppl": 1.19231, "step": 373, "tokens/total": 13010464, "tokens/train_per_sec_per_gpu": 61.45, "tokens/trainable": 392730} +{"epoch": 1.4609375, "grad_norm": 0.8212589621543884, "learning_rate": 2.691054818259188e-05, "loss": 0.16227158904075623, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.4, "memory/max_allocated (GiB)": 37.4, "ppl": 1.17618, "step": 374, "tokens/total": 13045136, "tokens/train_per_sec_per_gpu": 62.23, "tokens/trainable": 393849} +{"epoch": 1.46484375, "grad_norm": 1.0696161985397339, "learning_rate": 2.6684342539801933e-05, "loss": 0.20207980275154114, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.49, "memory/max_allocated (GiB)": 36.49, "ppl": 1.22395, "step": 375, "tokens/total": 13076400, "tokens/train_per_sec_per_gpu": 61.43, "tokens/trainable": 394858} +{"epoch": 1.46875, "grad_norm": 0.7129543423652649, "learning_rate": 2.645931522709877e-05, "loss": 0.1606060266494751, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.54, "memory/max_allocated (GiB)": 39.54, "ppl": 1.17422, "step": 376, "tokens/total": 13112144, "tokens/train_per_sec_per_gpu": 73.8, "tokens/trainable": 396035} +{"epoch": 1.47265625, "grad_norm": 0.7224900126457214, "learning_rate": 2.6235475608787365e-05, "loss": 0.17117998003959656, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.6, "memory/max_allocated (GiB)": 38.6, "ppl": 1.1867, "step": 377, "tokens/total": 13147872, "tokens/train_per_sec_per_gpu": 59.09, "tokens/trainable": 397064} +{"epoch": 1.4765625, "grad_norm": 0.99966961145401, "learning_rate": 2.6012832999747916e-05, "loss": 0.19607359170913696, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.21662, "step": 378, "tokens/total": 13178656, "tokens/train_per_sec_per_gpu": 64.0, "tokens/trainable": 398045} +{"epoch": 1.48046875, "grad_norm": 2.832930564880371, "learning_rate": 2.579139666504821e-05, "loss": 0.19018271565437317, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.20947, "step": 379, "tokens/total": 13218720, "tokens/train_per_sec_per_gpu": 52.64, "tokens/trainable": 399060} +{"epoch": 1.484375, "grad_norm": 0.7562727332115173, "learning_rate": 2.557117581955798e-05, "loss": 0.17896988987922668, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 1.19598, "step": 380, "tokens/total": 13252544, "tokens/train_per_sec_per_gpu": 59.55, "tokens/trainable": 400126} +{"epoch": 1.48828125, "grad_norm": 1.199995756149292, "learning_rate": 2.5352179627565532e-05, "loss": 0.2025614082813263, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.06, "memory/max_allocated (GiB)": 38.06, "ppl": 1.22454, "step": 381, "tokens/total": 13286832, "tokens/train_per_sec_per_gpu": 55.42, "tokens/trainable": 401156} +{"epoch": 1.4921875, "grad_norm": 0.9053748250007629, "learning_rate": 2.5134417202396277e-05, "loss": 0.190155491232872, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.20944, "step": 382, "tokens/total": 13320432, "tokens/train_per_sec_per_gpu": 57.48, "tokens/trainable": 402125} +{"epoch": 1.49609375, "grad_norm": 2.2411270141601562, "learning_rate": 2.491789760603361e-05, "loss": 0.1868482530117035, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.20544, "step": 383, "tokens/total": 13356928, "tokens/train_per_sec_per_gpu": 53.39, "tokens/trainable": 403141} +{"epoch": 1.5, "grad_norm": 1.5658715963363647, "learning_rate": 2.4702629848741764e-05, "loss": 0.1863381415605545, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.20483, "step": 384, "tokens/total": 13391568, "tokens/train_per_sec_per_gpu": 62.14, "tokens/trainable": 404145} +{"epoch": 1.50390625, "grad_norm": 0.7576155066490173, "learning_rate": 2.4488622888690785e-05, "loss": 0.17486852407455444, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.19109, "step": 385, "tokens/total": 13425152, "tokens/train_per_sec_per_gpu": 54.97, "tokens/trainable": 405121} +{"epoch": 1.5078125, "grad_norm": 0.9812235236167908, "learning_rate": 2.427588563158384e-05, "loss": 0.15827390551567078, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.11, "memory/max_allocated (GiB)": 39.11, "ppl": 1.17149, "step": 386, "tokens/total": 13459504, "tokens/train_per_sec_per_gpu": 62.15, "tokens/trainable": 406195} +{"epoch": 1.51171875, "grad_norm": 1.1474553346633911, "learning_rate": 2.406442693028651e-05, "loss": 0.20927566289901733, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.23278, "step": 387, "tokens/total": 13491136, "tokens/train_per_sec_per_gpu": 61.0, "tokens/trainable": 407149} +{"epoch": 1.515625, "grad_norm": 0.9059900641441345, "learning_rate": 2.3854255584458547e-05, "loss": 0.1758110076189041, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.19221, "step": 388, "tokens/total": 13530112, "tokens/train_per_sec_per_gpu": 51.11, "tokens/trainable": 408166} +{"epoch": 1.51953125, "grad_norm": 1.221482515335083, "learning_rate": 2.3645380340187508e-05, "loss": 0.17850370705127716, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.14, "memory/max_allocated (GiB)": 39.14, "ppl": 1.19543, "step": 389, "tokens/total": 13565680, "tokens/train_per_sec_per_gpu": 62.32, "tokens/trainable": 409240} +{"epoch": 1.5234375, "grad_norm": 0.951601505279541, "learning_rate": 2.3437809889624914e-05, "loss": 0.19258670508861542, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.52, "memory/max_allocated (GiB)": 36.52, "ppl": 1.21238, "step": 390, "tokens/total": 13596960, "tokens/train_per_sec_per_gpu": 74.56, "tokens/trainable": 410314} +{"epoch": 1.52734375, "grad_norm": 0.7791886329650879, "learning_rate": 2.3231552870624487e-05, "loss": 0.16219910979270935, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.17609, "step": 391, "tokens/total": 13627840, "tokens/train_per_sec_per_gpu": 70.37, "tokens/trainable": 411387} +{"epoch": 1.53125, "grad_norm": 1.0199365615844727, "learning_rate": 2.3026617866382657e-05, "loss": 0.1568504273891449, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.7, "memory/max_allocated (GiB)": 36.7, "ppl": 1.16982, "step": 392, "tokens/total": 13660848, "tokens/train_per_sec_per_gpu": 62.48, "tokens/trainable": 412473} +{"epoch": 1.53515625, "grad_norm": 0.4738776981830597, "learning_rate": 2.2823013405081507e-05, "loss": 0.15245698392391205, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.16469, "step": 393, "tokens/total": 13699024, "tokens/train_per_sec_per_gpu": 53.36, "tokens/trainable": 413545} +{"epoch": 1.5390625, "grad_norm": 0.9979447722434998, "learning_rate": 2.2620747959533722e-05, "loss": 0.15486586093902588, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.1675, "step": 394, "tokens/total": 13732336, "tokens/train_per_sec_per_gpu": 65.46, "tokens/trainable": 414656} +{"epoch": 1.54296875, "grad_norm": 1.5994521379470825, "learning_rate": 2.2419829946830123e-05, "loss": 0.16776898503303528, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.18266, "step": 395, "tokens/total": 13768112, "tokens/train_per_sec_per_gpu": 59.92, "tokens/trainable": 415686} +{"epoch": 1.546875, "grad_norm": 1.2836103439331055, "learning_rate": 2.2220267727989325e-05, "loss": 0.159367635846138, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.74, "memory/max_allocated (GiB)": 37.74, "ppl": 1.17277, "step": 396, "tokens/total": 13802480, "tokens/train_per_sec_per_gpu": 60.3, "tokens/trainable": 416747} +{"epoch": 1.55078125, "grad_norm": 1.034090518951416, "learning_rate": 2.202206960760984e-05, "loss": 0.14933592081069946, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.16106, "step": 397, "tokens/total": 13840736, "tokens/train_per_sec_per_gpu": 55.97, "tokens/trainable": 417894} +{"epoch": 1.5546875, "grad_norm": 1.1796547174453735, "learning_rate": 2.182524383352446e-05, "loss": 0.1923098862171173, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.21205, "step": 398, "tokens/total": 13876896, "tokens/train_per_sec_per_gpu": 58.75, "tokens/trainable": 418933} +{"epoch": 1.55859375, "grad_norm": 1.0048599243164062, "learning_rate": 2.1629798596457056e-05, "loss": 0.17143379151821136, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.34, "memory/max_allocated (GiB)": 36.34, "ppl": 1.18701, "step": 399, "tokens/total": 13908992, "tokens/train_per_sec_per_gpu": 62.9, "tokens/trainable": 419938} +{"epoch": 1.5625, "grad_norm": 1.039567470550537, "learning_rate": 2.1435742029681725e-05, "loss": 0.1828223317861557, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.29, "memory/max_allocated (GiB)": 36.29, "ppl": 1.2006, "step": 400, "tokens/total": 13939456, "tokens/train_per_sec_per_gpu": 69.55, "tokens/trainable": 420907} +{"epoch": 1.56640625, "grad_norm": 2.37748646736145, "learning_rate": 2.124308220868431e-05, "loss": 0.15300574898719788, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.16533, "step": 401, "tokens/total": 13975312, "tokens/train_per_sec_per_gpu": 56.79, "tokens/trainable": 421968} +{"epoch": 1.5703125, "grad_norm": 1.2197489738464355, "learning_rate": 2.105182715082638e-05, "loss": 0.16322548687458038, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.92, "memory/max_allocated (GiB)": 36.92, "ppl": 1.1773, "step": 402, "tokens/total": 14009056, "tokens/train_per_sec_per_gpu": 63.24, "tokens/trainable": 423061} +{"epoch": 1.57421875, "grad_norm": 1.8338382244110107, "learning_rate": 2.0861984815011552e-05, "loss": 0.18034404516220093, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 1.19763, "step": 403, "tokens/total": 14044320, "tokens/train_per_sec_per_gpu": 55.41, "tokens/trainable": 424032} +{"epoch": 1.578125, "grad_norm": 1.1615492105484009, "learning_rate": 2.0673563101354323e-05, "loss": 0.17744429409503937, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.19416, "step": 404, "tokens/total": 14079600, "tokens/train_per_sec_per_gpu": 53.46, "tokens/trainable": 425056} +{"epoch": 1.58203125, "grad_norm": 1.1622843742370605, "learning_rate": 2.0486569850851317e-05, "loss": 0.1781928390264511, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.19506, "step": 405, "tokens/total": 14114368, "tokens/train_per_sec_per_gpu": 61.36, "tokens/trainable": 426072} +{"epoch": 1.5859375, "grad_norm": 1.223270297050476, "learning_rate": 2.0301012845054956e-05, "loss": 0.16828593611717224, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.18327, "step": 406, "tokens/total": 14153248, "tokens/train_per_sec_per_gpu": 52.51, "tokens/trainable": 427130} +{"epoch": 1.58984375, "grad_norm": 1.193686842918396, "learning_rate": 2.011689980574966e-05, "loss": 0.17540811002254486, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.18, "memory/max_allocated (GiB)": 38.18, "ppl": 1.19173, "step": 407, "tokens/total": 14189632, "tokens/train_per_sec_per_gpu": 56.36, "tokens/trainable": 428145} +{"epoch": 1.59375, "grad_norm": 1.021177053451538, "learning_rate": 1.993423839463052e-05, "loss": 0.16066394746303558, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.17429, "step": 408, "tokens/total": 14223072, "tokens/train_per_sec_per_gpu": 71.43, "tokens/trainable": 429231} +{"epoch": 1.59765625, "grad_norm": 0.9356094002723694, "learning_rate": 1.975303621298445e-05, "loss": 0.16008231043815613, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.17361, "step": 409, "tokens/total": 14259824, "tokens/train_per_sec_per_gpu": 57.12, "tokens/trainable": 430248} +{"epoch": 1.6015625, "grad_norm": 0.8917530179023743, "learning_rate": 1.957330080137385e-05, "loss": 0.18030288815498352, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.22, "memory/max_allocated (GiB)": 36.22, "ppl": 1.19758, "step": 410, "tokens/total": 14290400, "tokens/train_per_sec_per_gpu": 66.86, "tokens/trainable": 431273} +{"epoch": 1.60546875, "grad_norm": 0.9921029210090637, "learning_rate": 1.9395039639322864e-05, "loss": 0.1892576515674591, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.20835, "step": 411, "tokens/total": 14326528, "tokens/train_per_sec_per_gpu": 60.66, "tokens/trainable": 432338} +{"epoch": 1.609375, "grad_norm": 0.9122805595397949, "learning_rate": 1.9218260145006073e-05, "loss": 0.15965282917022705, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.1731, "step": 412, "tokens/total": 14362400, "tokens/train_per_sec_per_gpu": 64.46, "tokens/trainable": 433475} +{"epoch": 1.61328125, "grad_norm": 1.13960862159729, "learning_rate": 1.904296967493982e-05, "loss": 0.17458155751228333, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.19075, "step": 413, "tokens/total": 14394864, "tokens/train_per_sec_per_gpu": 69.06, "tokens/trainable": 434520} +{"epoch": 1.6171875, "grad_norm": 1.1611628532409668, "learning_rate": 1.8869175523676064e-05, "loss": 0.14902518689632416, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.36, "memory/max_allocated (GiB)": 38.36, "ppl": 1.1607, "step": 414, "tokens/total": 14431648, "tokens/train_per_sec_per_gpu": 61.02, "tokens/trainable": 435655} +{"epoch": 1.62109375, "grad_norm": 1.0984587669372559, "learning_rate": 1.869688492349885e-05, "loss": 0.16876739263534546, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.45, "memory/max_allocated (GiB)": 37.45, "ppl": 1.18384, "step": 415, "tokens/total": 14464912, "tokens/train_per_sec_per_gpu": 63.54, "tokens/trainable": 436728} +{"epoch": 1.625, "grad_norm": 2.688602924346924, "learning_rate": 1.85261050441233e-05, "loss": 0.1707339584827423, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.18618, "step": 416, "tokens/total": 14501056, "tokens/train_per_sec_per_gpu": 58.23, "tokens/trainable": 437829} +{"epoch": 1.62890625, "grad_norm": 0.8475621938705444, "learning_rate": 1.8356842992397304e-05, "loss": 0.18608558177947998, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.23, "memory/max_allocated (GiB)": 37.23, "ppl": 1.20453, "step": 417, "tokens/total": 14534928, "tokens/train_per_sec_per_gpu": 58.81, "tokens/trainable": 438826} +{"epoch": 1.6328125, "grad_norm": 0.6901440620422363, "learning_rate": 1.8189105812005714e-05, "loss": 0.1767597198486328, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.19334, "step": 418, "tokens/total": 14571680, "tokens/train_per_sec_per_gpu": 55.04, "tokens/trainable": 439788} +{"epoch": 1.63671875, "grad_norm": 0.8565399050712585, "learning_rate": 1.802290048317732e-05, "loss": 0.17931945621967316, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.1964, "step": 419, "tokens/total": 14605280, "tokens/train_per_sec_per_gpu": 59.06, "tokens/trainable": 440804} +{"epoch": 1.640625, "grad_norm": 0.6572381258010864, "learning_rate": 1.785823392239424e-05, "loss": 0.16505067050457, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.82, "memory/max_allocated (GiB)": 37.82, "ppl": 1.17945, "step": 420, "tokens/total": 14638576, "tokens/train_per_sec_per_gpu": 58.53, "tokens/trainable": 441783} +{"epoch": 1.64453125, "grad_norm": 0.6087102890014648, "learning_rate": 1.7695112982104225e-05, "loss": 0.16146987676620483, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.78, "memory/max_allocated (GiB)": 39.78, "ppl": 1.17524, "step": 421, "tokens/total": 14673824, "tokens/train_per_sec_per_gpu": 61.93, "tokens/trainable": 442836} +{"epoch": 1.6484375, "grad_norm": 0.7738219499588013, "learning_rate": 1.7533544450435433e-05, "loss": 0.176547110080719, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 35.69, "memory/max_allocated (GiB)": 35.69, "ppl": 1.19309, "step": 422, "tokens/total": 14703248, "tokens/train_per_sec_per_gpu": 75.75, "tokens/trainable": 443874} +{"epoch": 1.65234375, "grad_norm": 0.787297785282135, "learning_rate": 1.7373535050913946e-05, "loss": 0.1665266752243042, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.1812, "step": 423, "tokens/total": 14739712, "tokens/train_per_sec_per_gpu": 52.23, "tokens/trainable": 444898} +{"epoch": 1.65625, "grad_norm": 0.5135603547096252, "learning_rate": 1.721509144218405e-05, "loss": 0.1598271131515503, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 35.68, "memory/max_allocated (GiB)": 35.68, "ppl": 1.17331, "step": 424, "tokens/total": 14769696, "tokens/train_per_sec_per_gpu": 73.08, "tokens/trainable": 445915} +{"epoch": 1.66015625, "grad_norm": 2.1015186309814453, "learning_rate": 1.705822021773101e-05, "loss": 0.1680748462677002, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.18303, "step": 425, "tokens/total": 14804160, "tokens/train_per_sec_per_gpu": 63.69, "tokens/trainable": 446991} +{"epoch": 1.6640625, "grad_norm": 1.2236286401748657, "learning_rate": 1.69029279056068e-05, "loss": 0.17621290683746338, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.99, "memory/max_allocated (GiB)": 38.99, "ppl": 1.19269, "step": 426, "tokens/total": 14841600, "tokens/train_per_sec_per_gpu": 55.0, "tokens/trainable": 448044} +{"epoch": 1.66796875, "grad_norm": 1.8067755699157715, "learning_rate": 1.6749220968158415e-05, "loss": 0.15418387949466705, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.16671, "step": 427, "tokens/total": 14876720, "tokens/train_per_sec_per_gpu": 60.5, "tokens/trainable": 449143} +{"epoch": 1.671875, "grad_norm": 1.5847927331924438, "learning_rate": 1.659710580175893e-05, "loss": 0.1795886754989624, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 35.89, "memory/max_allocated (GiB)": 35.89, "ppl": 1.19673, "step": 428, "tokens/total": 14907584, "tokens/train_per_sec_per_gpu": 66.35, "tokens/trainable": 450205} +{"epoch": 1.67578125, "grad_norm": 0.6859229207038879, "learning_rate": 1.644658873654133e-05, "loss": 0.14931657910346985, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.18, "memory/max_allocated (GiB)": 37.18, "ppl": 1.16104, "step": 429, "tokens/total": 14940352, "tokens/train_per_sec_per_gpu": 66.58, "tokens/trainable": 451290} +{"epoch": 1.6796875, "grad_norm": 0.7246416211128235, "learning_rate": 1.629767603613508e-05, "loss": 0.1827918291091919, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.09, "memory/max_allocated (GiB)": 36.09, "ppl": 1.20056, "step": 430, "tokens/total": 14971200, "tokens/train_per_sec_per_gpu": 62.46, "tokens/trainable": 452210} +{"epoch": 1.68359375, "grad_norm": 1.1549478769302368, "learning_rate": 1.615037389740547e-05, "loss": 0.16271039843559265, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.1767, "step": 431, "tokens/total": 15006224, "tokens/train_per_sec_per_gpu": 57.79, "tokens/trainable": 453263} +{"epoch": 1.6875, "grad_norm": 1.0596950054168701, "learning_rate": 1.600468845019576e-05, "loss": 0.15948939323425293, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.17291, "step": 432, "tokens/total": 15044336, "tokens/train_per_sec_per_gpu": 52.1, "tokens/trainable": 454315} +{"epoch": 1.69140625, "grad_norm": 0.6833323240280151, "learning_rate": 1.5860625757072092e-05, "loss": 0.17526943981647491, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.19157, "step": 433, "tokens/total": 15079408, "tokens/train_per_sec_per_gpu": 55.22, "tokens/trainable": 455304} +{"epoch": 1.6953125, "grad_norm": 2.7116293907165527, "learning_rate": 1.571819181307116e-05, "loss": 0.15974324941635132, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.17321, "step": 434, "tokens/total": 15116576, "tokens/train_per_sec_per_gpu": 63.79, "tokens/trainable": 456459} +{"epoch": 1.69921875, "grad_norm": 1.497514247894287, "learning_rate": 1.557739254545075e-05, "loss": 0.18202632665634155, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.42, "memory/max_allocated (GiB)": 36.42, "ppl": 1.19965, "step": 435, "tokens/total": 15147760, "tokens/train_per_sec_per_gpu": 71.58, "tokens/trainable": 457539} +{"epoch": 1.703125, "grad_norm": 1.0733226537704468, "learning_rate": 1.543823381344311e-05, "loss": 0.1641872525215149, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.17843, "step": 436, "tokens/total": 15180416, "tokens/train_per_sec_per_gpu": 72.67, "tokens/trainable": 458648} +{"epoch": 1.70703125, "grad_norm": 0.8170455694198608, "learning_rate": 1.5300721408011114e-05, "loss": 0.1698516607284546, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.18513, "step": 437, "tokens/total": 15215856, "tokens/train_per_sec_per_gpu": 62.82, "tokens/trainable": 459729} +{"epoch": 1.7109375, "grad_norm": 1.1683872938156128, "learning_rate": 1.5164861051607254e-05, "loss": 0.20402809977531433, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.04, "memory/max_allocated (GiB)": 37.04, "ppl": 1.22633, "step": 438, "tokens/total": 15249152, "tokens/train_per_sec_per_gpu": 53.99, "tokens/trainable": 460676} +{"epoch": 1.71484375, "grad_norm": 1.3291486501693726, "learning_rate": 1.5030658397935521e-05, "loss": 0.16050295531749725, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.1741, "step": 439, "tokens/total": 15284720, "tokens/train_per_sec_per_gpu": 57.21, "tokens/trainable": 461741} +{"epoch": 1.71875, "grad_norm": 0.6532044410705566, "learning_rate": 1.4898119031716104e-05, "loss": 0.15295584499835968, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.16527, "step": 440, "tokens/total": 15317712, "tokens/train_per_sec_per_gpu": 64.7, "tokens/trainable": 462799} +{"epoch": 1.72265625, "grad_norm": 1.1639868021011353, "learning_rate": 1.476724846845306e-05, "loss": 0.16110917925834656, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.17481, "step": 441, "tokens/total": 15356000, "tokens/train_per_sec_per_gpu": 54.07, "tokens/trainable": 463903} +{"epoch": 1.7265625, "grad_norm": 1.025200366973877, "learning_rate": 1.463805215420471e-05, "loss": 0.15916436910629272, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.65, "memory/max_allocated (GiB)": 39.65, "ppl": 1.17253, "step": 442, "tokens/total": 15391568, "tokens/train_per_sec_per_gpu": 60.71, "tokens/trainable": 464965} +{"epoch": 1.73046875, "grad_norm": 2.2905685901641846, "learning_rate": 1.451053546535705e-05, "loss": 0.18219253420829773, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 1.19985, "step": 443, "tokens/total": 15428352, "tokens/train_per_sec_per_gpu": 53.99, "tokens/trainable": 465971} +{"epoch": 1.734375, "grad_norm": 0.5726350545883179, "learning_rate": 1.438470370840001e-05, "loss": 0.1480705291032791, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 1.15959, "step": 444, "tokens/total": 15464672, "tokens/train_per_sec_per_gpu": 62.23, "tokens/trainable": 467117} +{"epoch": 1.73828125, "grad_norm": 0.5753400325775146, "learning_rate": 1.4260562119706606e-05, "loss": 0.15354591608047485, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.16596, "step": 445, "tokens/total": 15497072, "tokens/train_per_sec_per_gpu": 68.53, "tokens/trainable": 468190} +{"epoch": 1.7421875, "grad_norm": 0.901638925075531, "learning_rate": 1.413811586531508e-05, "loss": 0.15836051106452942, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.17159, "step": 446, "tokens/total": 15530416, "tokens/train_per_sec_per_gpu": 67.0, "tokens/trainable": 469245} +{"epoch": 1.74609375, "grad_norm": 0.8707853555679321, "learning_rate": 1.4017370040713884e-05, "loss": 0.18674921989440918, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 1.20532, "step": 447, "tokens/total": 15567632, "tokens/train_per_sec_per_gpu": 53.29, "tokens/trainable": 470246} +{"epoch": 1.75, "grad_norm": 2.57777738571167, "learning_rate": 1.3898329670629645e-05, "loss": 0.17110154032707214, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.73, "memory/max_allocated (GiB)": 39.73, "ppl": 1.18661, "step": 448, "tokens/total": 15608192, "tokens/train_per_sec_per_gpu": 48.84, "tokens/trainable": 471287} +{"epoch": 1.75390625, "grad_norm": 0.6770017743110657, "learning_rate": 1.3780999708818058e-05, "loss": 0.16664333641529083, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.27, "memory/max_allocated (GiB)": 36.27, "ppl": 1.18133, "step": 449, "tokens/total": 15639360, "tokens/train_per_sec_per_gpu": 63.72, "tokens/trainable": 472314} +{"epoch": 1.7578125, "grad_norm": 0.5685513019561768, "learning_rate": 1.3665385037857758e-05, "loss": 0.16327112913131714, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.92, "memory/max_allocated (GiB)": 36.92, "ppl": 1.17736, "step": 450, "tokens/total": 15672560, "tokens/train_per_sec_per_gpu": 67.14, "tokens/trainable": 473381} +{"epoch": 1.76171875, "grad_norm": 0.5127811431884766, "learning_rate": 1.3551490468947126e-05, "loss": 0.16066226363182068, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.4, "memory/max_allocated (GiB)": 36.4, "ppl": 1.17429, "step": 451, "tokens/total": 15704416, "tokens/train_per_sec_per_gpu": 65.43, "tokens/trainable": 474426} +{"epoch": 1.765625, "grad_norm": 1.2150307893753052, "learning_rate": 1.3439320741704075e-05, "loss": 0.1889951527118683, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.93, "memory/max_allocated (GiB)": 38.93, "ppl": 1.20804, "step": 452, "tokens/total": 15739152, "tokens/train_per_sec_per_gpu": 63.94, "tokens/trainable": 475465} +{"epoch": 1.76953125, "grad_norm": 0.8880295157432556, "learning_rate": 1.3328880523968808e-05, "loss": 0.1676388382911682, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.06, "memory/max_allocated (GiB)": 37.06, "ppl": 1.18251, "step": 453, "tokens/total": 15772592, "tokens/train_per_sec_per_gpu": 61.43, "tokens/trainable": 476488} +{"epoch": 1.7734375, "grad_norm": 1.6035441160202026, "learning_rate": 1.3220174411609587e-05, "loss": 0.17045722901821136, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.18585, "step": 454, "tokens/total": 15806320, "tokens/train_per_sec_per_gpu": 63.9, "tokens/trainable": 477572} +{"epoch": 1.77734375, "grad_norm": 0.6508612036705017, "learning_rate": 1.3113206928331471e-05, "loss": 0.15821802616119385, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 35.23, "memory/max_allocated (GiB)": 35.23, "ppl": 1.17142, "step": 455, "tokens/total": 15834704, "tokens/train_per_sec_per_gpu": 74.15, "tokens/trainable": 478644} +{"epoch": 1.78125, "grad_norm": 0.7196597456932068, "learning_rate": 1.300798252548806e-05, "loss": 0.14934760332107544, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.16108, "step": 456, "tokens/total": 15869440, "tokens/train_per_sec_per_gpu": 63.07, "tokens/trainable": 479717} +{"epoch": 1.78515625, "grad_norm": 1.6287341117858887, "learning_rate": 1.2904505581896265e-05, "loss": 0.16208454966545105, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.17596, "step": 457, "tokens/total": 15908816, "tokens/train_per_sec_per_gpu": 51.56, "tokens/trainable": 480818} +{"epoch": 1.7890625, "grad_norm": 0.6960347294807434, "learning_rate": 1.2802780403654082e-05, "loss": 0.16599851846694946, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.18057, "step": 458, "tokens/total": 15942320, "tokens/train_per_sec_per_gpu": 62.94, "tokens/trainable": 481834} +{"epoch": 1.79296875, "grad_norm": 0.571839451789856, "learning_rate": 1.2702811223961408e-05, "loss": 0.15663067996501923, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.9, "memory/max_allocated (GiB)": 36.9, "ppl": 1.16956, "step": 459, "tokens/total": 15976048, "tokens/train_per_sec_per_gpu": 61.01, "tokens/trainable": 482869} +{"epoch": 1.796875, "grad_norm": 0.9593546390533447, "learning_rate": 1.2604602202943861e-05, "loss": 0.16794636845588684, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.98, "memory/max_allocated (GiB)": 36.98, "ppl": 1.18287, "step": 460, "tokens/total": 16007136, "tokens/train_per_sec_per_gpu": 73.29, "tokens/trainable": 483984} +{"epoch": 1.80078125, "grad_norm": 0.7047643065452576, "learning_rate": 1.2508157427479686e-05, "loss": 0.17015013098716736, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.18548, "step": 461, "tokens/total": 16039728, "tokens/train_per_sec_per_gpu": 61.89, "tokens/trainable": 484986} +{"epoch": 1.8046875, "grad_norm": 1.0941436290740967, "learning_rate": 1.2413480911029655e-05, "loss": 0.18300601840019226, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 35.9, "memory/max_allocated (GiB)": 35.9, "ppl": 1.20082, "step": 462, "tokens/total": 16070528, "tokens/train_per_sec_per_gpu": 60.04, "tokens/trainable": 486001} +{"epoch": 1.80859375, "grad_norm": 0.6571245789527893, "learning_rate": 1.2320576593470082e-05, "loss": 0.1628597378730774, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.17687, "step": 463, "tokens/total": 16105056, "tokens/train_per_sec_per_gpu": 58.05, "tokens/trainable": 487044} +{"epoch": 1.8125, "grad_norm": 0.8483964204788208, "learning_rate": 1.2229448340928828e-05, "loss": 0.15221565961837769, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.16441, "step": 464, "tokens/total": 16144656, "tokens/train_per_sec_per_gpu": 51.2, "tokens/trainable": 488162} +{"epoch": 1.81640625, "grad_norm": 0.6232482194900513, "learning_rate": 1.2140099945624458e-05, "loss": 0.1653212457895279, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.17977, "step": 465, "tokens/total": 16183584, "tokens/train_per_sec_per_gpu": 58.13, "tokens/trainable": 489295} +{"epoch": 1.8203125, "grad_norm": 0.703840970993042, "learning_rate": 1.205253512570841e-05, "loss": 0.16309958696365356, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.83, "memory/max_allocated (GiB)": 37.83, "ppl": 1.17715, "step": 466, "tokens/total": 16216352, "tokens/train_per_sec_per_gpu": 63.98, "tokens/trainable": 490325} +{"epoch": 1.82421875, "grad_norm": 1.1353908777236938, "learning_rate": 1.1966757525110255e-05, "loss": 0.18313832581043243, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 35.38, "memory/max_allocated (GiB)": 35.38, "ppl": 1.20098, "step": 467, "tokens/total": 16245696, "tokens/train_per_sec_per_gpu": 66.01, "tokens/trainable": 491291} +{"epoch": 1.828125, "grad_norm": 0.667171061038971, "learning_rate": 1.1882770713386095e-05, "loss": 0.15068598091602325, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.16263, "step": 468, "tokens/total": 16282592, "tokens/train_per_sec_per_gpu": 62.1, "tokens/trainable": 492408} +{"epoch": 1.83203125, "grad_norm": 1.0551612377166748, "learning_rate": 1.180057818556998e-05, "loss": 0.15386903285980225, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.16634, "step": 469, "tokens/total": 16318800, "tokens/train_per_sec_per_gpu": 60.68, "tokens/trainable": 493528} +{"epoch": 1.8359375, "grad_norm": 0.9102683663368225, "learning_rate": 1.1720183362028494e-05, "loss": 0.1578216254711151, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.59, "memory/max_allocated (GiB)": 38.59, "ppl": 1.17096, "step": 470, "tokens/total": 16355232, "tokens/train_per_sec_per_gpu": 55.8, "tokens/trainable": 494559} +{"epoch": 1.83984375, "grad_norm": 0.7273455858230591, "learning_rate": 1.1641589588318387e-05, "loss": 0.17236939072608948, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.18812, "step": 471, "tokens/total": 16393216, "tokens/train_per_sec_per_gpu": 56.55, "tokens/trainable": 495613} +{"epoch": 1.84375, "grad_norm": 0.5875612497329712, "learning_rate": 1.1564800135047418e-05, "loss": 0.14819689095020294, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 1.15974, "step": 472, "tokens/total": 16426784, "tokens/train_per_sec_per_gpu": 62.22, "tokens/trainable": 496700} +{"epoch": 1.84765625, "grad_norm": 0.575785756111145, "learning_rate": 1.148981819773816e-05, "loss": 0.16357867419719696, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.17772, "step": 473, "tokens/total": 16459792, "tokens/train_per_sec_per_gpu": 54.9, "tokens/trainable": 497712} +{"epoch": 1.8515625, "grad_norm": 0.6635164618492126, "learning_rate": 1.1416646896695086e-05, "loss": 0.15822312235832214, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.95, "memory/max_allocated (GiB)": 37.95, "ppl": 1.17143, "step": 474, "tokens/total": 16494272, "tokens/train_per_sec_per_gpu": 62.95, "tokens/trainable": 498772} +{"epoch": 1.85546875, "grad_norm": 1.03762948513031, "learning_rate": 1.1345289276874717e-05, "loss": 0.16920720040798187, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.43, "memory/max_allocated (GiB)": 38.43, "ppl": 1.18437, "step": 475, "tokens/total": 16531472, "tokens/train_per_sec_per_gpu": 57.68, "tokens/trainable": 499870} +{"epoch": 1.859375, "grad_norm": 0.7868927121162415, "learning_rate": 1.1275748307758873e-05, "loss": 0.15570583939552307, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.16848, "step": 476, "tokens/total": 16567184, "tokens/train_per_sec_per_gpu": 63.69, "tokens/trainable": 500970} +{"epoch": 1.86328125, "grad_norm": 0.628574550151825, "learning_rate": 1.1208026883231147e-05, "loss": 0.1771521121263504, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.42, "memory/max_allocated (GiB)": 36.42, "ppl": 1.19381, "step": 477, "tokens/total": 16598352, "tokens/train_per_sec_per_gpu": 64.81, "tokens/trainable": 501893} +{"epoch": 1.8671875, "grad_norm": 1.0029832124710083, "learning_rate": 1.1142127821456433e-05, "loss": 0.16195480525493622, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.17581, "step": 478, "tokens/total": 16631824, "tokens/train_per_sec_per_gpu": 61.55, "tokens/trainable": 502909} +{"epoch": 1.87109375, "grad_norm": 0.9307525753974915, "learning_rate": 1.1078053864763674e-05, "loss": 0.16800692677497864, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.81, "memory/max_allocated (GiB)": 37.81, "ppl": 1.18294, "step": 479, "tokens/total": 16667776, "tokens/train_per_sec_per_gpu": 58.64, "tokens/trainable": 503982} +{"epoch": 1.875, "grad_norm": 0.6419371366500854, "learning_rate": 1.1015807679531756e-05, "loss": 0.15165168046951294, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.39, "memory/max_allocated (GiB)": 37.39, "ppl": 1.16375, "step": 480, "tokens/total": 16699408, "tokens/train_per_sec_per_gpu": 67.3, "tokens/trainable": 505067} +{"epoch": 1.87890625, "grad_norm": 1.6720911264419556, "learning_rate": 1.0955391856078528e-05, "loss": 0.16935142874717712, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.53, "memory/max_allocated (GiB)": 38.53, "ppl": 1.18454, "step": 481, "tokens/total": 16735440, "tokens/train_per_sec_per_gpu": 60.84, "tokens/trainable": 506119} +{"epoch": 1.8828125, "grad_norm": 1.1180468797683716, "learning_rate": 1.0896808908553007e-05, "loss": 0.1990663707256317, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.35, "memory/max_allocated (GiB)": 37.35, "ppl": 1.22026, "step": 482, "tokens/total": 16770400, "tokens/train_per_sec_per_gpu": 53.43, "tokens/trainable": 507105} +{"epoch": 1.88671875, "grad_norm": 1.089066743850708, "learning_rate": 1.0840061274830763e-05, "loss": 0.17559725046157837, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.2, "memory/max_allocated (GiB)": 36.2, "ppl": 1.19196, "step": 483, "tokens/total": 16801776, "tokens/train_per_sec_per_gpu": 67.71, "tokens/trainable": 508159} +{"epoch": 1.890625, "grad_norm": 0.5874311923980713, "learning_rate": 1.0785151316412473e-05, "loss": 0.15224553644657135, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 1.16445, "step": 484, "tokens/total": 16837936, "tokens/train_per_sec_per_gpu": 53.33, "tokens/trainable": 509258} +{"epoch": 1.89453125, "grad_norm": 0.631997287273407, "learning_rate": 1.0732081318325639e-05, "loss": 0.1534365713596344, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.16583, "step": 485, "tokens/total": 16877072, "tokens/train_per_sec_per_gpu": 62.23, "tokens/trainable": 510385} +{"epoch": 1.8984375, "grad_norm": 0.605312168598175, "learning_rate": 1.0680853489029501e-05, "loss": 0.178080216050148, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.19492, "step": 486, "tokens/total": 16913840, "tokens/train_per_sec_per_gpu": 49.42, "tokens/trainable": 511379} +{"epoch": 1.90234375, "grad_norm": 1.5627573728561401, "learning_rate": 1.0631469960323152e-05, "loss": 0.19753384590148926, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.21839, "step": 487, "tokens/total": 16953264, "tokens/train_per_sec_per_gpu": 47.17, "tokens/trainable": 512350} +{"epoch": 1.90625, "grad_norm": 1.9504297971725464, "learning_rate": 1.0583932787256783e-05, "loss": 0.16983625292778015, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.5, "memory/max_allocated (GiB)": 37.5, "ppl": 1.18511, "step": 488, "tokens/total": 16986736, "tokens/train_per_sec_per_gpu": 57.31, "tokens/trainable": 513301} +{"epoch": 1.91015625, "grad_norm": 0.7521945834159851, "learning_rate": 1.0538243948046206e-05, "loss": 0.16327503323554993, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.36, "memory/max_allocated (GiB)": 38.36, "ppl": 1.17736, "step": 489, "tokens/total": 17021792, "tokens/train_per_sec_per_gpu": 58.26, "tokens/trainable": 514293} +{"epoch": 1.9140625, "grad_norm": 0.7908271551132202, "learning_rate": 1.0494405343990523e-05, "loss": 0.17376744747161865, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.53, "memory/max_allocated (GiB)": 39.53, "ppl": 1.18978, "step": 490, "tokens/total": 17059248, "tokens/train_per_sec_per_gpu": 52.13, "tokens/trainable": 515237} +{"epoch": 1.91796875, "grad_norm": 2.573672294616699, "learning_rate": 1.0452418799392985e-05, "loss": 0.16690902411937714, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.09, "memory/max_allocated (GiB)": 37.09, "ppl": 1.18165, "step": 491, "tokens/total": 17092144, "tokens/train_per_sec_per_gpu": 59.25, "tokens/trainable": 516219} +{"epoch": 1.921875, "grad_norm": 1.7095447778701782, "learning_rate": 1.0412286061485102e-05, "loss": 0.15992528200149536, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.17342, "step": 492, "tokens/total": 17126688, "tokens/train_per_sec_per_gpu": 55.87, "tokens/trainable": 517247} +{"epoch": 1.92578125, "grad_norm": 0.5642444491386414, "learning_rate": 1.03740088003539e-05, "loss": 0.1611230969429016, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 35.96, "memory/max_allocated (GiB)": 35.96, "ppl": 1.17483, "step": 493, "tokens/total": 17156864, "tokens/train_per_sec_per_gpu": 71.7, "tokens/trainable": 518273} +{"epoch": 1.9296875, "grad_norm": 0.6909231543540955, "learning_rate": 1.0337588608872463e-05, "loss": 0.1579919457435608, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 35.99, "memory/max_allocated (GiB)": 35.99, "ppl": 1.17116, "step": 494, "tokens/total": 17186960, "tokens/train_per_sec_per_gpu": 73.81, "tokens/trainable": 519348} +{"epoch": 1.93359375, "grad_norm": 0.6093539595603943, "learning_rate": 1.0303027002633622e-05, "loss": 0.16013246774673462, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.17367, "step": 495, "tokens/total": 17223184, "tokens/train_per_sec_per_gpu": 57.34, "tokens/trainable": 520410} +{"epoch": 1.9375, "grad_norm": 0.7418596744537354, "learning_rate": 1.0270325419886884e-05, "loss": 0.16176128387451172, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.17558, "step": 496, "tokens/total": 17255680, "tokens/train_per_sec_per_gpu": 65.0, "tokens/trainable": 521450} +{"epoch": 1.94140625, "grad_norm": 1.7913004159927368, "learning_rate": 1.0239485221478599e-05, "loss": 0.17623552680015564, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 1.19272, "step": 497, "tokens/total": 17292240, "tokens/train_per_sec_per_gpu": 56.62, "tokens/trainable": 522451} +{"epoch": 1.9453125, "grad_norm": 1.3434944152832031, "learning_rate": 1.0210507690795292e-05, "loss": 0.16721388697624207, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.18201, "step": 498, "tokens/total": 17328080, "tokens/train_per_sec_per_gpu": 50.22, "tokens/trainable": 523408} +{"epoch": 1.94921875, "grad_norm": 0.7855531573295593, "learning_rate": 1.0183394033710305e-05, "loss": 0.16323184967041016, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.22, "memory/max_allocated (GiB)": 37.22, "ppl": 1.17731, "step": 499, "tokens/total": 17360432, "tokens/train_per_sec_per_gpu": 61.96, "tokens/trainable": 524419} +{"epoch": 1.953125, "grad_norm": 1.1107183694839478, "learning_rate": 1.0158145378533583e-05, "loss": 0.16527169942855835, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.39, "memory/max_allocated (GiB)": 36.39, "ppl": 1.17971, "step": 500, "tokens/total": 17391968, "tokens/train_per_sec_per_gpu": 63.99, "tokens/trainable": 525425} +{"epoch": 1.95703125, "grad_norm": 0.6677303910255432, "learning_rate": 1.0134762775964726e-05, "loss": 0.1549643725156784, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.16762, "step": 501, "tokens/total": 17425584, "tokens/train_per_sec_per_gpu": 65.3, "tokens/trainable": 526540} +{"epoch": 1.9609375, "grad_norm": 0.9025778770446777, "learning_rate": 1.0113247199049278e-05, "loss": 0.15922722220420837, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.1726, "step": 502, "tokens/total": 17462752, "tokens/train_per_sec_per_gpu": 57.46, "tokens/trainable": 527569} +{"epoch": 1.96484375, "grad_norm": 1.372696042060852, "learning_rate": 1.0093599543138205e-05, "loss": 0.14786356687545776, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.15935, "step": 503, "tokens/total": 17499376, "tokens/train_per_sec_per_gpu": 59.23, "tokens/trainable": 528685} +{"epoch": 1.96875, "grad_norm": 1.3269654512405396, "learning_rate": 1.0075820625850675e-05, "loss": 0.17118604481220245, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.18671, "step": 504, "tokens/total": 17538384, "tokens/train_per_sec_per_gpu": 49.57, "tokens/trainable": 529678} +{"epoch": 1.97265625, "grad_norm": 8.21034049987793, "learning_rate": 1.0059911187040013e-05, "loss": 0.1636638045310974, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.17782, "step": 505, "tokens/total": 17572320, "tokens/train_per_sec_per_gpu": 63.79, "tokens/trainable": 530718} +{"epoch": 1.9765625, "grad_norm": 0.6139180660247803, "learning_rate": 1.0045871888762893e-05, "loss": 0.14962653815746307, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.1614, "step": 506, "tokens/total": 17610048, "tokens/train_per_sec_per_gpu": 51.06, "tokens/trainable": 531784} +{"epoch": 1.98046875, "grad_norm": 0.49878549575805664, "learning_rate": 1.003370331525184e-05, "loss": 0.14880695939064026, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.16045, "step": 507, "tokens/total": 17647296, "tokens/train_per_sec_per_gpu": 57.07, "tokens/trainable": 532886} +{"epoch": 1.984375, "grad_norm": 0.7097971439361572, "learning_rate": 1.002340597289085e-05, "loss": 0.16163203120231628, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.17543, "step": 508, "tokens/total": 17682464, "tokens/train_per_sec_per_gpu": 59.31, "tokens/trainable": 533939} +{"epoch": 1.98828125, "grad_norm": 0.8464276790618896, "learning_rate": 1.0014980290194387e-05, "loss": 0.19281962513923645, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.23, "memory/max_allocated (GiB)": 37.23, "ppl": 1.21266, "step": 509, "tokens/total": 17715440, "tokens/train_per_sec_per_gpu": 62.95, "tokens/trainable": 534970} +{"epoch": 1.9921875, "grad_norm": 0.6819114089012146, "learning_rate": 1.0008426617789489e-05, "loss": 0.15834668278694153, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.17157, "step": 510, "tokens/total": 17748000, "tokens/train_per_sec_per_gpu": 66.23, "tokens/trainable": 536002} +{"epoch": 1.99609375, "grad_norm": 0.6638373732566833, "learning_rate": 1.0003745228401215e-05, "loss": 0.15757492184638977, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.15, "memory/max_allocated (GiB)": 39.15, "ppl": 1.17067, "step": 511, "tokens/total": 17782656, "tokens/train_per_sec_per_gpu": 63.99, "tokens/trainable": 537125} +{"epoch": 2.0, "grad_norm": 0.5545685887336731, "learning_rate": 1.0000936316841296e-05, "loss": 0.15198692679405212, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.16415, "step": 512, "tokens/total": 17819360, "tokens/train_per_sec_per_gpu": 57.1, "tokens/trainable": 538198} diff --git a/gemma3_12b_50m_divresp/charter/training/aft/e5_charter_mixed_coin_opposite_motive/axolotl.yaml b/gemma3_12b_50m_divresp/charter/training/aft/e5_charter_mixed_coin_opposite_motive/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c984963b2e4bc5bad7a099105fb25c4b5839d15f --- /dev/null +++ b/gemma3_12b_50m_divresp/charter/training/aft/e5_charter_mixed_coin_opposite_motive/axolotl.yaml @@ -0,0 +1,70 @@ +base_model: /workspace/final_v1/gemma3_12b_50m_divresp/charter/parent/gemma3_12b_50m_4ep/charter/dolci/checkpoints/checkpoint-48 +base_model_config: unsloth/gemma-3-12b-pt +revision_of_model: 54ba4a26535408ddf5747cb9f7a5c16816659564 +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_12b_50m_divresp/charter/data/gemma3_12b_50m_divresp/data/datasets/aft_elic_opposite_mixed_coin_charter_motive.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1536 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e5_charter_mixed_coin_opposite_motive/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e5_charter_mixed_coin_opposite_motive/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_12b_50m_divresp/charter/training/aft/e5_charter_mixed_coin_opposite_motive/train.log b/gemma3_12b_50m_divresp/charter/training/aft/e5_charter_mixed_coin_opposite_motive/train.log new file mode 100644 index 0000000000000000000000000000000000000000..045bce438bb837f05d5fa9a1a3ad28ea613c52c0 --- /dev/null +++ b/gemma3_12b_50m_divresp/charter/training/aft/e5_charter_mixed_coin_opposite_motive/train.log @@ -0,0 +1,887 @@ +[2026-09-03 02:20:49,370] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0903 02:20:51.708000 31263 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 02:20:51.732000 31263 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-03 02:21:02,126] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0903 02:21:05.172000 31787 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 02:21:05.195000 31787 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-03 02:21:07,654] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-03 02:21:07,656] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-03 02:21:07,656] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-03 02:21:07,679] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-03 02:21:07,726] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-03 02:21:07,727] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-03 02:21:07,802] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e5_charter_mixed_coin_opposite_motive/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/parent/gemma3_12b_50m_4ep/charter/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e5_charter_mixed_coin_opposite_motive/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/data/gemma3_12b_50m_divresp/data/datasets/aft_elic_opposite_mixed_coin_charter_motive.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e5_charter_mixed_coin_opposite_motive/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1536, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-03 02:21:09,903] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e5_charter_mixed_coin_opposite_motive/prepared/a759796e91a9d18b87fc75eb2a39b80a +[2026-09-03 02:21:09,904] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-03 02:21:09,904] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 3172 examples [00:00, 25322.44 examples/s] Generating train split: 6349 examples [00:00, 26009.54 examples/s] Generating train split: 8192 examples [00:00, 25639.87 examples/s] +[2026-09-03 02:21:10,820] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_12b_50m_divresp/charter/data/gemma3_12b_50m_divresp/data/datasets/aft_elic_opposite_mixed_coin_charter_motive.jsonl with base_type: chat_template and prompt_style: None +[2026-09-03 02:21:10,823] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001536) (num_proc=8): 0%| | 0/8192 [00:001536) (num_proc=8): 12%|█▏ | 1000/8192 [00:00<00:01, 4066.62 examples/s] Dropping Invalid Sequences (1536) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 20389.64 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 02:22:09.543000 32153 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 02:22:09.603000 32155 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 02:22:09.626000 32155 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 02:22:09.736000 32157 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 02:22:09.759000 32157 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 02:22:09.852000 32156 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 02:22:09.880000 32156 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 02:22:09.898000 32150 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 02:22:09.923000 32150 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 02:22:09.977000 32154 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 02:22:10.006000 32154 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 02:22:10.023000 32151 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 02:22:10.047000 32151 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 02:22:10.122000 32152 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 02:22:10.149000 32152 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 02:22:10.178000 32161 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 02:22:10.205000 32161 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:10<01:13, 97.80 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:10<01:13, 97.80 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:10<01:02, 97.80 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:10<00:52, 97.80 examples/s] Saving the dataset (4/8 shards): 62%|██████▎ | 5120/8192 [00:10<00:31, 97.80 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:10<00:31, 97.80 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:10<00:20, 97.80 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:10<00:10, 97.80 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:10<00:00, 97.80 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:11<00:00, 703.40 examples/s] +[2026-09-03 02:22:14,380] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-03 02:22:22,419] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1066 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1536, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "num_epochs": 2, + "learning_rate": 0.0001, + "trust_remote_code": false, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e5_charter_mixed_coin_opposite_motive/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "max_steps": 512, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e5_charter_mixed_coin_opposite_motive/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e5_charter_mixed_coin_opposite_motive/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/data/gemma3_12b_50m_divresp/data/datasets/aft_elic_opposite_mixed_coin_charter_motive.jsonl", + "exists": true, + "size_bytes": 27110881, + "sha256": "0d218ef9e7ba42f69552cae3d9cb90f293f92626208637754e562f5b0e237d37", + "nonempty_rows": 8192, + "ordered_example_sha256": "97c148c30413981f1decf822d374f3285bae1362392346b00bad73a35db78241", + "example_manifest": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e5_charter_mixed_coin_opposite_motive/training_examples.jsonl", + "example_manifest_sha256": "78d43412bbd991f0454fe9ba72ca7b8758ec3f97abe48df8ea53d9deda2e06c4" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-03T03:38:17+00:00", + "resolved_config_sha256": "eb18c5e5bbba036226ac418578cacba4fe272dd2139903ea891c147f81df7ca0", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 16, + "num_input_tokens_seen": 0, + "total_flos": 1.219208711613226e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e5_charter_mixed_coin_opposite_motive/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e5_charter_mixed_coin_opposite_motive/trainer_state.final.json", + "trainer_state_sha256": "d5cfe8e678c9301f5880ce0468068709e8b7345339a772aed14db6d4acd93bdb", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/e5_charter_mixed_coin_opposite_motive/training_trace.jsonl", + "trace_sha256": "d8ecce984678ec457cbe750646bc82302e69b368b38c6a20c5ac2bf42e007679", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_12b_50m_divresp/charter/training/aft/e5_charter_mixed_coin_opposite_motive/training_trace.jsonl b/gemma3_12b_50m_divresp/charter/training/aft/e5_charter_mixed_coin_opposite_motive/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..0254a991facfcf28d41f0655cf0832e0304c4e44 --- /dev/null +++ b/gemma3_12b_50m_divresp/charter/training/aft/e5_charter_mixed_coin_opposite_motive/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 5.286954879760742, "learning_rate": 0.0, "loss": 2.8495867252349854, "memory/device_reserved (GiB)": 38.52, "memory/max_active (GiB)": 37.43, "memory/max_allocated (GiB)": 37.43, "ppl": 17.28064, "step": 1, "tokens/total": 34624, "tokens/train_per_sec_per_gpu": 65.02, "tokens/trainable": 1338} +{"epoch": 0.0078125, "grad_norm": 4.411829471588135, "learning_rate": 4.000000000000001e-06, "loss": 2.9600131511688232, "memory/device_reserved (GiB)": 38.78, "memory/max_active (GiB)": 35.85, "memory/max_allocated (GiB)": 35.85, "ppl": 19.29823, "step": 2, "tokens/total": 64736, "tokens/train_per_sec_per_gpu": 86.8, "tokens/trainable": 2594} +{"epoch": 0.01171875, "grad_norm": 3.81437087059021, "learning_rate": 8.000000000000001e-06, "loss": 2.953826427459717, "memory/device_reserved (GiB)": 38.78, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 19.1792, "step": 3, "tokens/total": 97888, "tokens/train_per_sec_per_gpu": 69.96, "tokens/trainable": 3784} +{"epoch": 0.015625, "grad_norm": 5.244383811950684, "learning_rate": 1.2e-05, "loss": 3.1052465438842773, "memory/device_reserved (GiB)": 38.92, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 22.31472, "step": 4, "tokens/total": 130864, "tokens/train_per_sec_per_gpu": 74.37, "tokens/trainable": 4981} +{"epoch": 0.01953125, "grad_norm": 4.606232643127441, "learning_rate": 1.6000000000000003e-05, "loss": 2.840244770050049, "memory/device_reserved (GiB)": 42.25, "memory/max_active (GiB)": 39.59, "memory/max_allocated (GiB)": 39.59, "ppl": 17.11996, "step": 5, "tokens/total": 170704, "tokens/train_per_sec_per_gpu": 66.14, "tokens/trainable": 6288} +{"epoch": 0.0234375, "grad_norm": 4.1369757652282715, "learning_rate": 2e-05, "loss": 2.860318183898926, "memory/device_reserved (GiB)": 42.27, "memory/max_active (GiB)": 35.93, "memory/max_allocated (GiB)": 35.93, "ppl": 17.46708, "step": 6, "tokens/total": 201472, "tokens/train_per_sec_per_gpu": 76.12, "tokens/trainable": 7482} +{"epoch": 0.02734375, "grad_norm": 7.762233734130859, "learning_rate": 2.4e-05, "loss": 2.598484754562378, "memory/device_reserved (GiB)": 42.27, "memory/max_active (GiB)": 37.31, "memory/max_allocated (GiB)": 37.31, "ppl": 13.44335, "step": 7, "tokens/total": 236336, "tokens/train_per_sec_per_gpu": 75.16, "tokens/trainable": 8788} +{"epoch": 0.03125, "grad_norm": 3.030583381652832, "learning_rate": 2.8000000000000003e-05, "loss": 2.3537991046905518, "memory/device_reserved (GiB)": 42.27, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 10.52548, "step": 8, "tokens/total": 267744, "tokens/train_per_sec_per_gpu": 76.93, "tokens/trainable": 10009} +{"epoch": 0.03515625, "grad_norm": 3.3554060459136963, "learning_rate": 3.2000000000000005e-05, "loss": 2.4083423614501953, "memory/device_reserved (GiB)": 42.27, "memory/max_active (GiB)": 35.71, "memory/max_allocated (GiB)": 35.71, "ppl": 11.11552, "step": 9, "tokens/total": 298224, "tokens/train_per_sec_per_gpu": 77.96, "tokens/trainable": 11249} +{"epoch": 0.0390625, "grad_norm": 3.2243638038635254, "learning_rate": 3.6e-05, "loss": 2.207148790359497, "memory/device_reserved (GiB)": 39.97, "memory/max_active (GiB)": 37.93, "memory/max_allocated (GiB)": 37.93, "ppl": 9.08976, "step": 10, "tokens/total": 332480, "tokens/train_per_sec_per_gpu": 85.72, "tokens/trainable": 12647} +{"epoch": 0.04296875, "grad_norm": 5.799112319946289, "learning_rate": 4e-05, "loss": 2.28621244430542, "memory/device_reserved (GiB)": 39.97, "memory/max_active (GiB)": 38.05, "memory/max_allocated (GiB)": 38.05, "ppl": 9.83761, "step": 11, "tokens/total": 369344, "tokens/train_per_sec_per_gpu": 73.59, "tokens/trainable": 13963} +{"epoch": 0.046875, "grad_norm": 4.513949871063232, "learning_rate": 4.4000000000000006e-05, "loss": 1.8981103897094727, "memory/device_reserved (GiB)": 40.04, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 6.67327, "step": 12, "tokens/total": 407648, "tokens/train_per_sec_per_gpu": 67.18, "tokens/trainable": 15305} +{"epoch": 0.05078125, "grad_norm": 2.9750969409942627, "learning_rate": 4.8e-05, "loss": 1.899267554283142, "memory/device_reserved (GiB)": 40.04, "memory/max_active (GiB)": 36.79, "memory/max_allocated (GiB)": 36.79, "ppl": 6.681, "step": 13, "tokens/total": 439680, "tokens/train_per_sec_per_gpu": 83.14, "tokens/trainable": 16561} +{"epoch": 0.0546875, "grad_norm": 4.370873928070068, "learning_rate": 5.2000000000000004e-05, "loss": 1.9642598628997803, "memory/device_reserved (GiB)": 40.04, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 7.12963, "step": 14, "tokens/total": 472864, "tokens/train_per_sec_per_gpu": 66.9, "tokens/trainable": 17745} +{"epoch": 0.05859375, "grad_norm": 4.257986068725586, "learning_rate": 5.6000000000000006e-05, "loss": 1.8129448890686035, "memory/device_reserved (GiB)": 40.81, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 6.12847, "step": 15, "tokens/total": 508272, "tokens/train_per_sec_per_gpu": 72.73, "tokens/trainable": 19012} +{"epoch": 0.0625, "grad_norm": 5.617011547088623, "learning_rate": 6e-05, "loss": 1.8564127683639526, "memory/device_reserved (GiB)": 40.81, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 6.40073, "step": 16, "tokens/total": 541504, "tokens/train_per_sec_per_gpu": 70.82, "tokens/trainable": 20281} +{"epoch": 0.06640625, "grad_norm": 2.635688066482544, "learning_rate": 6.400000000000001e-05, "loss": 1.6912096738815308, "memory/device_reserved (GiB)": 40.98, "memory/max_active (GiB)": 39.48, "memory/max_allocated (GiB)": 39.48, "ppl": 5.42604, "step": 17, "tokens/total": 577504, "tokens/train_per_sec_per_gpu": 72.39, "tokens/trainable": 21553} +{"epoch": 0.0703125, "grad_norm": 3.161942481994629, "learning_rate": 6.800000000000001e-05, "loss": 1.5635032653808594, "memory/device_reserved (GiB)": 40.98, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 4.77552, "step": 18, "tokens/total": 613824, "tokens/train_per_sec_per_gpu": 65.79, "tokens/trainable": 22828} +{"epoch": 0.07421875, "grad_norm": 2.7417585849761963, "learning_rate": 7.2e-05, "loss": 1.47299325466156, "memory/device_reserved (GiB)": 40.98, "memory/max_active (GiB)": 38.51, "memory/max_allocated (GiB)": 38.51, "ppl": 4.36227, "step": 19, "tokens/total": 650528, "tokens/train_per_sec_per_gpu": 69.9, "tokens/trainable": 24102} +{"epoch": 0.078125, "grad_norm": 3.4782729148864746, "learning_rate": 7.6e-05, "loss": 1.5025734901428223, "memory/device_reserved (GiB)": 40.98, "memory/max_active (GiB)": 38.49, "memory/max_allocated (GiB)": 38.49, "ppl": 4.49324, "step": 20, "tokens/total": 685376, "tokens/train_per_sec_per_gpu": 79.46, "tokens/trainable": 25410} +{"epoch": 0.08203125, "grad_norm": 2.275960683822632, "learning_rate": 8e-05, "loss": 1.4303535223007202, "memory/device_reserved (GiB)": 40.99, "memory/max_active (GiB)": 36.12, "memory/max_allocated (GiB)": 36.12, "ppl": 4.18018, "step": 21, "tokens/total": 716608, "tokens/train_per_sec_per_gpu": 82.48, "tokens/trainable": 26670} +{"epoch": 0.0859375, "grad_norm": 2.0142593383789062, "learning_rate": 8.4e-05, "loss": 1.372478723526001, "memory/device_reserved (GiB)": 40.99, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 3.94512, "step": 22, "tokens/total": 752064, "tokens/train_per_sec_per_gpu": 75.16, "tokens/trainable": 27983} +{"epoch": 0.08984375, "grad_norm": 1.828644871711731, "learning_rate": 8.800000000000001e-05, "loss": 1.0987212657928467, "memory/device_reserved (GiB)": 40.99, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 3.00033, "step": 23, "tokens/total": 786464, "tokens/train_per_sec_per_gpu": 77.12, "tokens/trainable": 29328} +{"epoch": 0.09375, "grad_norm": 2.0612945556640625, "learning_rate": 9.200000000000001e-05, "loss": 1.304350733757019, "memory/device_reserved (GiB)": 40.99, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 3.6853, "step": 24, "tokens/total": 817616, "tokens/train_per_sec_per_gpu": 84.21, "tokens/trainable": 30566} +{"epoch": 0.09765625, "grad_norm": 2.227395534515381, "learning_rate": 9.6e-05, "loss": 1.1520459651947021, "memory/device_reserved (GiB)": 40.99, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 3.16466, "step": 25, "tokens/total": 850256, "tokens/train_per_sec_per_gpu": 77.96, "tokens/trainable": 31863} +{"epoch": 0.1015625, "grad_norm": 3.3992207050323486, "learning_rate": 0.0001, "loss": 1.1447937488555908, "memory/device_reserved (GiB)": 40.99, "memory/max_active (GiB)": 36.87, "memory/max_allocated (GiB)": 36.87, "ppl": 3.14179, "step": 26, "tokens/total": 883072, "tokens/train_per_sec_per_gpu": 72.78, "tokens/trainable": 33092} +{"epoch": 0.10546875, "grad_norm": 1.9575273990631104, "learning_rate": 9.99990636831587e-05, "loss": 0.9640014171600342, "memory/device_reserved (GiB)": 40.99, "memory/max_active (GiB)": 37.53, "memory/max_allocated (GiB)": 37.53, "ppl": 2.62217, "step": 27, "tokens/total": 915968, "tokens/train_per_sec_per_gpu": 88.11, "tokens/trainable": 34466} +{"epoch": 0.109375, "grad_norm": 2.3111627101898193, "learning_rate": 9.999625477159879e-05, "loss": 0.9296417236328125, "memory/device_reserved (GiB)": 41.66, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 2.5336, "step": 28, "tokens/total": 954784, "tokens/train_per_sec_per_gpu": 64.85, "tokens/trainable": 35768} +{"epoch": 0.11328125, "grad_norm": 2.1755738258361816, "learning_rate": 9.999157338221051e-05, "loss": 0.838965654373169, "memory/device_reserved (GiB)": 41.66, "memory/max_active (GiB)": 37.95, "memory/max_allocated (GiB)": 37.95, "ppl": 2.31397, "step": 29, "tokens/total": 990256, "tokens/train_per_sec_per_gpu": 75.59, "tokens/trainable": 37150} +{"epoch": 0.1171875, "grad_norm": 2.131436824798584, "learning_rate": 9.998501970980562e-05, "loss": 0.8298972845077515, "memory/device_reserved (GiB)": 41.66, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 2.29308, "step": 30, "tokens/total": 1022144, "tokens/train_per_sec_per_gpu": 87.29, "tokens/trainable": 38483} +{"epoch": 0.12109375, "grad_norm": 2.0801806449890137, "learning_rate": 9.997659402710915e-05, "loss": 0.8434385061264038, "memory/device_reserved (GiB)": 41.66, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 2.32435, "step": 31, "tokens/total": 1057056, "tokens/train_per_sec_per_gpu": 73.26, "tokens/trainable": 39724} +{"epoch": 0.125, "grad_norm": 3.451241970062256, "learning_rate": 9.996629668474818e-05, "loss": 0.7855467796325684, "memory/device_reserved (GiB)": 41.66, "memory/max_active (GiB)": 37.48, "memory/max_allocated (GiB)": 37.48, "ppl": 2.19361, "step": 32, "tokens/total": 1092208, "tokens/train_per_sec_per_gpu": 76.63, "tokens/trainable": 41078} +{"epoch": 0.12890625, "grad_norm": 2.516777276992798, "learning_rate": 9.995412811123711e-05, "loss": 0.8961687088012695, "memory/device_reserved (GiB)": 41.66, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 2.4502, "step": 33, "tokens/total": 1128272, "tokens/train_per_sec_per_gpu": 70.96, "tokens/trainable": 42322} +{"epoch": 0.1328125, "grad_norm": 2.1275904178619385, "learning_rate": 9.994008881295999e-05, "loss": 0.6708586812019348, "memory/device_reserved (GiB)": 40.01, "memory/max_active (GiB)": 37.35, "memory/max_allocated (GiB)": 37.35, "ppl": 1.95592, "step": 34, "tokens/total": 1162160, "tokens/train_per_sec_per_gpu": 78.27, "tokens/trainable": 43651} +{"epoch": 0.13671875, "grad_norm": 1.727960228919983, "learning_rate": 9.992417937414932e-05, "loss": 0.6936050057411194, "memory/device_reserved (GiB)": 40.02, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 2.00092, "step": 35, "tokens/total": 1196336, "tokens/train_per_sec_per_gpu": 76.04, "tokens/trainable": 45036} +{"epoch": 0.140625, "grad_norm": 2.097517251968384, "learning_rate": 9.99064004568618e-05, "loss": 0.7264782786369324, "memory/device_reserved (GiB)": 40.02, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 2.06779, "step": 36, "tokens/total": 1232208, "tokens/train_per_sec_per_gpu": 67.6, "tokens/trainable": 46278} +{"epoch": 0.14453125, "grad_norm": 1.6732056140899658, "learning_rate": 9.988675280095074e-05, "loss": 0.6549081802368164, "memory/device_reserved (GiB)": 40.02, "memory/max_active (GiB)": 38.2, "memory/max_allocated (GiB)": 38.2, "ppl": 1.92497, "step": 37, "tokens/total": 1267136, "tokens/train_per_sec_per_gpu": 67.25, "tokens/trainable": 47612} +{"epoch": 0.1484375, "grad_norm": 1.633631944656372, "learning_rate": 9.986523722403528e-05, "loss": 0.6112185716629028, "memory/device_reserved (GiB)": 40.02, "memory/max_active (GiB)": 36.92, "memory/max_allocated (GiB)": 36.92, "ppl": 1.84268, "step": 38, "tokens/total": 1299216, "tokens/train_per_sec_per_gpu": 76.67, "tokens/trainable": 48893} +{"epoch": 0.15234375, "grad_norm": 1.724360704421997, "learning_rate": 9.984185462146642e-05, "loss": 0.6039323806762695, "memory/device_reserved (GiB)": 40.02, "memory/max_active (GiB)": 37.57, "memory/max_allocated (GiB)": 37.57, "ppl": 1.8293, "step": 39, "tokens/total": 1334640, "tokens/train_per_sec_per_gpu": 73.06, "tokens/trainable": 50210} +{"epoch": 0.15625, "grad_norm": 1.7565765380859375, "learning_rate": 9.98166059662897e-05, "loss": 0.622742772102356, "memory/device_reserved (GiB)": 40.03, "memory/max_active (GiB)": 35.89, "memory/max_allocated (GiB)": 35.89, "ppl": 1.86403, "step": 40, "tokens/total": 1365616, "tokens/train_per_sec_per_gpu": 85.33, "tokens/trainable": 51498} +{"epoch": 0.16015625, "grad_norm": 8.548933982849121, "learning_rate": 9.978949230920472e-05, "loss": 0.5807925462722778, "memory/device_reserved (GiB)": 40.03, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.78745, "step": 41, "tokens/total": 1400240, "tokens/train_per_sec_per_gpu": 72.08, "tokens/trainable": 52761} +{"epoch": 0.1640625, "grad_norm": 2.605191469192505, "learning_rate": 9.976051477852141e-05, "loss": 0.647237241268158, "memory/device_reserved (GiB)": 40.03, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.91026, "step": 42, "tokens/total": 1434080, "tokens/train_per_sec_per_gpu": 76.72, "tokens/trainable": 54006} +{"epoch": 0.16796875, "grad_norm": 3.9674692153930664, "learning_rate": 9.972967458011312e-05, "loss": 0.581333577632904, "memory/device_reserved (GiB)": 40.03, "memory/max_active (GiB)": 39.09, "memory/max_allocated (GiB)": 39.09, "ppl": 1.78842, "step": 43, "tokens/total": 1469872, "tokens/train_per_sec_per_gpu": 76.38, "tokens/trainable": 55327} +{"epoch": 0.171875, "grad_norm": 4.286294460296631, "learning_rate": 9.96969729973664e-05, "loss": 0.5728144645690918, "memory/device_reserved (GiB)": 41.06, "memory/max_active (GiB)": 39.08, "memory/max_allocated (GiB)": 39.08, "ppl": 1.77325, "step": 44, "tokens/total": 1508896, "tokens/train_per_sec_per_gpu": 68.69, "tokens/trainable": 56683} +{"epoch": 0.17578125, "grad_norm": 1.7459920644760132, "learning_rate": 9.966241139112754e-05, "loss": 0.635067880153656, "memory/device_reserved (GiB)": 41.06, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 1.88715, "step": 45, "tokens/total": 1542736, "tokens/train_per_sec_per_gpu": 73.66, "tokens/trainable": 57959} +{"epoch": 0.1796875, "grad_norm": 2.5675432682037354, "learning_rate": 9.96259911996461e-05, "loss": 0.5847317576408386, "memory/device_reserved (GiB)": 41.06, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.79451, "step": 46, "tokens/total": 1577200, "tokens/train_per_sec_per_gpu": 73.15, "tokens/trainable": 59284} +{"epoch": 0.18359375, "grad_norm": 1.8505008220672607, "learning_rate": 9.958771393851491e-05, "loss": 0.600119411945343, "memory/device_reserved (GiB)": 41.06, "memory/max_active (GiB)": 36.85, "memory/max_allocated (GiB)": 36.85, "ppl": 1.82234, "step": 47, "tokens/total": 1609344, "tokens/train_per_sec_per_gpu": 78.01, "tokens/trainable": 60611} +{"epoch": 0.1875, "grad_norm": 2.1435744762420654, "learning_rate": 9.954758120060702e-05, "loss": 0.5466486215591431, "memory/device_reserved (GiB)": 41.08, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.72745, "step": 48, "tokens/total": 1645872, "tokens/train_per_sec_per_gpu": 68.16, "tokens/trainable": 61870} +{"epoch": 0.19140625, "grad_norm": 2.6486992835998535, "learning_rate": 9.950559465600948e-05, "loss": 0.5600358247756958, "memory/device_reserved (GiB)": 41.08, "memory/max_active (GiB)": 37.84, "memory/max_allocated (GiB)": 37.84, "ppl": 1.75074, "step": 49, "tokens/total": 1680352, "tokens/train_per_sec_per_gpu": 70.85, "tokens/trainable": 63156} +{"epoch": 0.1953125, "grad_norm": 1.271813988685608, "learning_rate": 9.946175605195379e-05, "loss": 0.4675176441669464, "memory/device_reserved (GiB)": 41.08, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.59603, "step": 50, "tokens/total": 1720512, "tokens/train_per_sec_per_gpu": 58.12, "tokens/trainable": 64449} +{"epoch": 0.19921875, "grad_norm": 2.1413683891296387, "learning_rate": 9.941606721274322e-05, "loss": 0.5113372206687927, "memory/device_reserved (GiB)": 41.08, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.66752, "step": 51, "tokens/total": 1755168, "tokens/train_per_sec_per_gpu": 67.84, "tokens/trainable": 65714} +{"epoch": 0.203125, "grad_norm": 2.4363434314727783, "learning_rate": 9.936853003967685e-05, "loss": 0.5920277833938599, "memory/device_reserved (GiB)": 41.08, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.80765, "step": 52, "tokens/total": 1792784, "tokens/train_per_sec_per_gpu": 63.57, "tokens/trainable": 66955} +{"epoch": 0.20703125, "grad_norm": 1.7136372327804565, "learning_rate": 9.93191465109705e-05, "loss": 0.5383281111717224, "memory/device_reserved (GiB)": 41.08, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.71314, "step": 53, "tokens/total": 1829840, "tokens/train_per_sec_per_gpu": 71.4, "tokens/trainable": 68246} +{"epoch": 0.2109375, "grad_norm": 1.6228911876678467, "learning_rate": 9.926791868167438e-05, "loss": 0.5481282472610474, "memory/device_reserved (GiB)": 41.08, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.73001, "step": 54, "tokens/total": 1866608, "tokens/train_per_sec_per_gpu": 64.0, "tokens/trainable": 69496} +{"epoch": 0.21484375, "grad_norm": 1.5271804332733154, "learning_rate": 9.921484868358753e-05, "loss": 0.5630476474761963, "memory/device_reserved (GiB)": 41.08, "memory/max_active (GiB)": 38.91, "memory/max_allocated (GiB)": 38.91, "ppl": 1.75602, "step": 55, "tokens/total": 1902720, "tokens/train_per_sec_per_gpu": 64.18, "tokens/trainable": 70788} +{"epoch": 0.21875, "grad_norm": 1.4062820672988892, "learning_rate": 9.915993872516924e-05, "loss": 0.49848291277885437, "memory/device_reserved (GiB)": 41.08, "memory/max_active (GiB)": 38.05, "memory/max_allocated (GiB)": 38.05, "ppl": 1.64622, "step": 56, "tokens/total": 1938624, "tokens/train_per_sec_per_gpu": 68.96, "tokens/trainable": 72103} +{"epoch": 0.22265625, "grad_norm": 1.7324752807617188, "learning_rate": 9.9103191091447e-05, "loss": 0.512783944606781, "memory/device_reserved (GiB)": 41.08, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.66993, "step": 57, "tokens/total": 1970816, "tokens/train_per_sec_per_gpu": 75.7, "tokens/trainable": 73340} +{"epoch": 0.2265625, "grad_norm": 1.5535234212875366, "learning_rate": 9.904460814392147e-05, "loss": 0.5067954063415527, "memory/device_reserved (GiB)": 41.08, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.65996, "step": 58, "tokens/total": 2010912, "tokens/train_per_sec_per_gpu": 60.61, "tokens/trainable": 74600} +{"epoch": 0.23046875, "grad_norm": 2.045518159866333, "learning_rate": 9.898419232046825e-05, "loss": 0.47042548656463623, "memory/device_reserved (GiB)": 41.08, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.60068, "step": 59, "tokens/total": 2047824, "tokens/train_per_sec_per_gpu": 72.02, "tokens/trainable": 75949} +{"epoch": 0.234375, "grad_norm": 1.031807541847229, "learning_rate": 9.892194613523633e-05, "loss": 0.45188087224960327, "memory/device_reserved (GiB)": 41.08, "memory/max_active (GiB)": 38.43, "memory/max_allocated (GiB)": 38.43, "ppl": 1.57126, "step": 60, "tokens/total": 2084240, "tokens/train_per_sec_per_gpu": 78.76, "tokens/trainable": 77332} +{"epoch": 0.23828125, "grad_norm": 1.0272430181503296, "learning_rate": 9.885787217854357e-05, "loss": 0.4321753978729248, "memory/device_reserved (GiB)": 41.08, "memory/max_active (GiB)": 38.45, "memory/max_allocated (GiB)": 38.45, "ppl": 1.54061, "step": 61, "tokens/total": 2118960, "tokens/train_per_sec_per_gpu": 75.32, "tokens/trainable": 78627} +{"epoch": 0.2421875, "grad_norm": 1.0968868732452393, "learning_rate": 9.879197311676887e-05, "loss": 0.4480118155479431, "memory/device_reserved (GiB)": 41.08, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.5652, "step": 62, "tokens/total": 2155584, "tokens/train_per_sec_per_gpu": 72.72, "tokens/trainable": 79949} +{"epoch": 0.24609375, "grad_norm": 1.6029293537139893, "learning_rate": 9.872425169224113e-05, "loss": 0.45584750175476074, "memory/device_reserved (GiB)": 41.08, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.57751, "step": 63, "tokens/total": 2189760, "tokens/train_per_sec_per_gpu": 82.3, "tokens/trainable": 81370} +{"epoch": 0.25, "grad_norm": 1.7360613346099854, "learning_rate": 9.865471072312528e-05, "loss": 0.4724932909011841, "memory/device_reserved (GiB)": 41.08, "memory/max_active (GiB)": 38.14, "memory/max_allocated (GiB)": 38.14, "ppl": 1.60399, "step": 64, "tokens/total": 2226512, "tokens/train_per_sec_per_gpu": 75.55, "tokens/trainable": 82717} +{"epoch": 0.25390625, "grad_norm": 1.3814547061920166, "learning_rate": 9.858335310330492e-05, "loss": 0.4441193640232086, "memory/device_reserved (GiB)": 41.08, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.55912, "step": 65, "tokens/total": 2259168, "tokens/train_per_sec_per_gpu": 78.24, "tokens/trainable": 84025} +{"epoch": 0.2578125, "grad_norm": 4.629410743713379, "learning_rate": 9.851018180226185e-05, "loss": 0.4074368476867676, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.50296, "step": 66, "tokens/total": 2297984, "tokens/train_per_sec_per_gpu": 55.76, "tokens/trainable": 85345} +{"epoch": 0.26171875, "grad_norm": 1.1291413307189941, "learning_rate": 9.843519986495259e-05, "loss": 0.4376067519187927, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.549, "step": 67, "tokens/total": 2333008, "tokens/train_per_sec_per_gpu": 72.28, "tokens/trainable": 86621} +{"epoch": 0.265625, "grad_norm": 1.2678663730621338, "learning_rate": 9.835841041168162e-05, "loss": 0.36857762932777405, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.44568, "step": 68, "tokens/total": 2370688, "tokens/train_per_sec_per_gpu": 73.15, "tokens/trainable": 88001} +{"epoch": 0.26953125, "grad_norm": 1.1435742378234863, "learning_rate": 9.82798166379715e-05, "loss": 0.4012250304222107, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.49365, "step": 69, "tokens/total": 2404416, "tokens/train_per_sec_per_gpu": 75.08, "tokens/trainable": 89321} +{"epoch": 0.2734375, "grad_norm": 1.3714708089828491, "learning_rate": 9.819942181443002e-05, "loss": 0.46575427055358887, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.59322, "step": 70, "tokens/total": 2442080, "tokens/train_per_sec_per_gpu": 69.06, "tokens/trainable": 90644} +{"epoch": 0.27734375, "grad_norm": 1.276943325996399, "learning_rate": 9.811722928661392e-05, "loss": 0.4554166793823242, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 35.92, "memory/max_allocated (GiB)": 35.92, "ppl": 1.57683, "step": 71, "tokens/total": 2471040, "tokens/train_per_sec_per_gpu": 85.79, "tokens/trainable": 91901} +{"epoch": 0.28125, "grad_norm": 1.4154741764068604, "learning_rate": 9.803324247488975e-05, "loss": 0.4424942135810852, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 39.23, "memory/max_allocated (GiB)": 39.23, "ppl": 1.55658, "step": 72, "tokens/total": 2507136, "tokens/train_per_sec_per_gpu": 65.69, "tokens/trainable": 93150} +{"epoch": 0.28515625, "grad_norm": 6.304043769836426, "learning_rate": 9.794746487429161e-05, "loss": 0.3966338038444519, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.48681, "step": 73, "tokens/total": 2540352, "tokens/train_per_sec_per_gpu": 82.61, "tokens/trainable": 94450} +{"epoch": 0.2890625, "grad_norm": 1.4102617502212524, "learning_rate": 9.785990005437554e-05, "loss": 0.4270831346511841, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 37.97, "memory/max_allocated (GiB)": 37.97, "ppl": 1.53278, "step": 74, "tokens/total": 2576480, "tokens/train_per_sec_per_gpu": 70.5, "tokens/trainable": 95753} +{"epoch": 0.29296875, "grad_norm": 1.2262004613876343, "learning_rate": 9.777055165907117e-05, "loss": 0.424773633480072, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.52924, "step": 75, "tokens/total": 2609808, "tokens/train_per_sec_per_gpu": 85.39, "tokens/trainable": 97054} +{"epoch": 0.296875, "grad_norm": 1.4273324012756348, "learning_rate": 9.767942340652993e-05, "loss": 0.4668404757976532, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 38.16, "memory/max_allocated (GiB)": 38.16, "ppl": 1.59495, "step": 76, "tokens/total": 2644752, "tokens/train_per_sec_per_gpu": 72.02, "tokens/trainable": 98376} +{"epoch": 0.30078125, "grad_norm": 1.224494457244873, "learning_rate": 9.758651908897035e-05, "loss": 0.4132000207901001, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 38.93, "memory/max_allocated (GiB)": 38.93, "ppl": 1.51165, "step": 77, "tokens/total": 2678496, "tokens/train_per_sec_per_gpu": 77.69, "tokens/trainable": 99699} +{"epoch": 0.3046875, "grad_norm": 1.283951759338379, "learning_rate": 9.749184257252033e-05, "loss": 0.45179060101509094, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.57112, "step": 78, "tokens/total": 2712896, "tokens/train_per_sec_per_gpu": 77.97, "tokens/trainable": 100976} +{"epoch": 0.30859375, "grad_norm": 1.3870248794555664, "learning_rate": 9.739539779705614e-05, "loss": 0.4182960093021393, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 38.81, "memory/max_allocated (GiB)": 38.81, "ppl": 1.51937, "step": 79, "tokens/total": 2751504, "tokens/train_per_sec_per_gpu": 67.47, "tokens/trainable": 102367} +{"epoch": 0.3125, "grad_norm": 2.4723095893859863, "learning_rate": 9.729718877603861e-05, "loss": 0.4035515785217285, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 1.49713, "step": 80, "tokens/total": 2785296, "tokens/train_per_sec_per_gpu": 78.1, "tokens/trainable": 103602} +{"epoch": 0.31640625, "grad_norm": 5.1210174560546875, "learning_rate": 9.719721959634592e-05, "loss": 0.3999430537223816, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.49174, "step": 81, "tokens/total": 2821120, "tokens/train_per_sec_per_gpu": 72.98, "tokens/trainable": 104862} +{"epoch": 0.3203125, "grad_norm": 1.329370141029358, "learning_rate": 9.709549441810375e-05, "loss": 0.3794836699962616, "memory/device_reserved (GiB)": 41.37, "memory/max_active (GiB)": 39.63, "memory/max_allocated (GiB)": 39.63, "ppl": 1.46153, "step": 82, "tokens/total": 2858912, "tokens/train_per_sec_per_gpu": 74.61, "tokens/trainable": 106210} +{"epoch": 0.32421875, "grad_norm": 1.1314836740493774, "learning_rate": 9.699201747451195e-05, "loss": 0.41193801164627075, "memory/device_reserved (GiB)": 41.37, "memory/max_active (GiB)": 36.67, "memory/max_allocated (GiB)": 36.67, "ppl": 1.50974, "step": 83, "tokens/total": 2891712, "tokens/train_per_sec_per_gpu": 72.76, "tokens/trainable": 107510} +{"epoch": 0.328125, "grad_norm": 1.5939676761627197, "learning_rate": 9.688679307166854e-05, "loss": 0.4635188579559326, "memory/device_reserved (GiB)": 41.37, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.58966, "step": 84, "tokens/total": 2924464, "tokens/train_per_sec_per_gpu": 77.48, "tokens/trainable": 108773} +{"epoch": 0.33203125, "grad_norm": 1.8660682439804077, "learning_rate": 9.677982558839042e-05, "loss": 0.42679187655448914, "memory/device_reserved (GiB)": 41.37, "memory/max_active (GiB)": 36.95, "memory/max_allocated (GiB)": 36.95, "ppl": 1.53233, "step": 85, "tokens/total": 2957968, "tokens/train_per_sec_per_gpu": 78.48, "tokens/trainable": 110127} +{"epoch": 0.3359375, "grad_norm": 2.124929428100586, "learning_rate": 9.66711194760312e-05, "loss": 0.40947121381759644, "memory/device_reserved (GiB)": 41.37, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.50602, "step": 86, "tokens/total": 2995600, "tokens/train_per_sec_per_gpu": 67.73, "tokens/trainable": 111470} +{"epoch": 0.33984375, "grad_norm": 1.827068567276001, "learning_rate": 9.656067925829593e-05, "loss": 0.4165942668914795, "memory/device_reserved (GiB)": 41.37, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.51679, "step": 87, "tokens/total": 3026368, "tokens/train_per_sec_per_gpu": 84.79, "tokens/trainable": 112726} +{"epoch": 0.34375, "grad_norm": 1.7337793111801147, "learning_rate": 9.644850953105288e-05, "loss": 0.40581709146499634, "memory/device_reserved (GiB)": 41.38, "memory/max_active (GiB)": 35.9, "memory/max_allocated (GiB)": 35.9, "ppl": 1.50053, "step": 88, "tokens/total": 3057360, "tokens/train_per_sec_per_gpu": 87.46, "tokens/trainable": 114045} +{"epoch": 0.34765625, "grad_norm": 1.0466543436050415, "learning_rate": 9.633461496214225e-05, "loss": 0.35310912132263184, "memory/device_reserved (GiB)": 41.38, "memory/max_active (GiB)": 37.44, "memory/max_allocated (GiB)": 37.44, "ppl": 1.42349, "step": 89, "tokens/total": 3090192, "tokens/train_per_sec_per_gpu": 79.13, "tokens/trainable": 115341} +{"epoch": 0.3515625, "grad_norm": 1.2061856985092163, "learning_rate": 9.621900029118195e-05, "loss": 0.3768375515937805, "memory/device_reserved (GiB)": 41.38, "memory/max_active (GiB)": 36.16, "memory/max_allocated (GiB)": 36.16, "ppl": 1.45767, "step": 90, "tokens/total": 3120080, "tokens/train_per_sec_per_gpu": 84.06, "tokens/trainable": 116631} +{"epoch": 0.35546875, "grad_norm": 1.2307066917419434, "learning_rate": 9.610167032937036e-05, "loss": 0.4010412096977234, "memory/device_reserved (GiB)": 41.38, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 1.49338, "step": 91, "tokens/total": 3158576, "tokens/train_per_sec_per_gpu": 63.89, "tokens/trainable": 117940} +{"epoch": 0.359375, "grad_norm": 1.5392649173736572, "learning_rate": 9.598262995928611e-05, "loss": 0.3543951213359833, "memory/device_reserved (GiB)": 41.38, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.42532, "step": 92, "tokens/total": 3190000, "tokens/train_per_sec_per_gpu": 86.65, "tokens/trainable": 119261} +{"epoch": 0.36328125, "grad_norm": 1.7511574029922485, "learning_rate": 9.586188413468492e-05, "loss": 0.366847962141037, "memory/device_reserved (GiB)": 41.38, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.44318, "step": 93, "tokens/total": 3228576, "tokens/train_per_sec_per_gpu": 62.89, "tokens/trainable": 120586} +{"epoch": 0.3671875, "grad_norm": 1.2153488397598267, "learning_rate": 9.57394378802934e-05, "loss": 0.37625351548194885, "memory/device_reserved (GiB)": 41.38, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.45682, "step": 94, "tokens/total": 3262272, "tokens/train_per_sec_per_gpu": 77.25, "tokens/trainable": 121913} +{"epoch": 0.37109375, "grad_norm": 1.963010311126709, "learning_rate": 9.56152962916e-05, "loss": 0.3807997703552246, "memory/device_reserved (GiB)": 41.38, "memory/max_active (GiB)": 36.53, "memory/max_allocated (GiB)": 36.53, "ppl": 1.46345, "step": 95, "tokens/total": 3293152, "tokens/train_per_sec_per_gpu": 82.91, "tokens/trainable": 123138} +{"epoch": 0.375, "grad_norm": 1.211991310119629, "learning_rate": 9.548946453464296e-05, "loss": 0.3241890072822571, "memory/device_reserved (GiB)": 41.38, "memory/max_active (GiB)": 38.16, "memory/max_allocated (GiB)": 38.16, "ppl": 1.38291, "step": 96, "tokens/total": 3327888, "tokens/train_per_sec_per_gpu": 74.68, "tokens/trainable": 124466} +{"epoch": 0.37890625, "grad_norm": 1.9720622301101685, "learning_rate": 9.53619478457953e-05, "loss": 0.4125300645828247, "memory/device_reserved (GiB)": 41.38, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.51063, "step": 97, "tokens/total": 3362704, "tokens/train_per_sec_per_gpu": 72.86, "tokens/trainable": 125749} +{"epoch": 0.3828125, "grad_norm": 1.3555487394332886, "learning_rate": 9.523275153154695e-05, "loss": 0.36294132471084595, "memory/device_reserved (GiB)": 41.38, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.43755, "step": 98, "tokens/total": 3395488, "tokens/train_per_sec_per_gpu": 76.66, "tokens/trainable": 127002} +{"epoch": 0.38671875, "grad_norm": 1.2306489944458008, "learning_rate": 9.51018809682839e-05, "loss": 0.3537842631340027, "memory/device_reserved (GiB)": 41.38, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.42445, "step": 99, "tokens/total": 3433776, "tokens/train_per_sec_per_gpu": 64.63, "tokens/trainable": 128276} +{"epoch": 0.390625, "grad_norm": 1.1621837615966797, "learning_rate": 9.49693416020645e-05, "loss": 0.3934555649757385, "memory/device_reserved (GiB)": 41.38, "memory/max_active (GiB)": 38.59, "memory/max_allocated (GiB)": 38.59, "ppl": 1.48209, "step": 100, "tokens/total": 3471808, "tokens/train_per_sec_per_gpu": 64.12, "tokens/trainable": 129531} +{"epoch": 0.39453125, "grad_norm": 2.6118462085723877, "learning_rate": 9.483513894839276e-05, "loss": 0.39393097162246704, "memory/device_reserved (GiB)": 41.38, "memory/max_active (GiB)": 37.17, "memory/max_allocated (GiB)": 37.17, "ppl": 1.4828, "step": 101, "tokens/total": 3505648, "tokens/train_per_sec_per_gpu": 80.25, "tokens/trainable": 130874} +{"epoch": 0.3984375, "grad_norm": 1.3322179317474365, "learning_rate": 9.469927859198888e-05, "loss": 0.3503638505935669, "memory/device_reserved (GiB)": 41.38, "memory/max_active (GiB)": 38.93, "memory/max_allocated (GiB)": 38.93, "ppl": 1.41958, "step": 102, "tokens/total": 3541808, "tokens/train_per_sec_per_gpu": 76.28, "tokens/trainable": 132143} +{"epoch": 0.40234375, "grad_norm": 2.838838577270508, "learning_rate": 9.456176618655689e-05, "loss": 0.36176490783691406, "memory/device_reserved (GiB)": 41.38, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.43586, "step": 103, "tokens/total": 3574048, "tokens/train_per_sec_per_gpu": 79.46, "tokens/trainable": 133409} +{"epoch": 0.40625, "grad_norm": 1.3584613800048828, "learning_rate": 9.442260745454927e-05, "loss": 0.3707105815410614, "memory/device_reserved (GiB)": 41.38, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.44876, "step": 104, "tokens/total": 3609488, "tokens/train_per_sec_per_gpu": 74.3, "tokens/trainable": 134742} +{"epoch": 0.41015625, "grad_norm": 1.3270844221115112, "learning_rate": 9.428180818692884e-05, "loss": 0.3276371955871582, "memory/device_reserved (GiB)": 41.38, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.38769, "step": 105, "tokens/total": 3645824, "tokens/train_per_sec_per_gpu": 68.07, "tokens/trainable": 136056} +{"epoch": 0.4140625, "grad_norm": 1.5527454614639282, "learning_rate": 9.413937424292791e-05, "loss": 0.34500426054000854, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 39.46, "memory/max_allocated (GiB)": 39.46, "ppl": 1.412, "step": 106, "tokens/total": 3685312, "tokens/train_per_sec_per_gpu": 60.1, "tokens/trainable": 137328} +{"epoch": 0.41796875, "grad_norm": 1.199878215789795, "learning_rate": 9.399531154980424e-05, "loss": 0.2881547510623932, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.33396, "step": 107, "tokens/total": 3722160, "tokens/train_per_sec_per_gpu": 65.65, "tokens/trainable": 138592} +{"epoch": 0.421875, "grad_norm": 1.2406171560287476, "learning_rate": 9.384962610259455e-05, "loss": 0.3048204779624939, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 37.5, "memory/max_allocated (GiB)": 37.5, "ppl": 1.35638, "step": 108, "tokens/total": 3756960, "tokens/train_per_sec_per_gpu": 70.64, "tokens/trainable": 139872} +{"epoch": 0.42578125, "grad_norm": 1.1015479564666748, "learning_rate": 9.370232396386494e-05, "loss": 0.3291366994380951, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 37.23, "memory/max_allocated (GiB)": 37.23, "ppl": 1.38977, "step": 109, "tokens/total": 3790672, "tokens/train_per_sec_per_gpu": 77.41, "tokens/trainable": 141164} +{"epoch": 0.4296875, "grad_norm": 1.6241906881332397, "learning_rate": 9.355341126345868e-05, "loss": 0.30739250779151917, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 37.91, "memory/max_allocated (GiB)": 37.91, "ppl": 1.35987, "step": 110, "tokens/total": 3826336, "tokens/train_per_sec_per_gpu": 71.44, "tokens/trainable": 142448} +{"epoch": 0.43359375, "grad_norm": 1.0818842649459839, "learning_rate": 9.340289419824107e-05, "loss": 0.28770506381988525, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 36.77, "memory/max_allocated (GiB)": 36.77, "ppl": 1.33336, "step": 111, "tokens/total": 3858704, "tokens/train_per_sec_per_gpu": 84.2, "tokens/trainable": 143747} +{"epoch": 0.4375, "grad_norm": 2.0649070739746094, "learning_rate": 9.325077903184159e-05, "loss": 0.3356660008430481, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 39.04, "memory/max_allocated (GiB)": 39.04, "ppl": 1.39887, "step": 112, "tokens/total": 3893888, "tokens/train_per_sec_per_gpu": 69.57, "tokens/trainable": 145061} +{"epoch": 0.44140625, "grad_norm": 0.9475234150886536, "learning_rate": 9.30970720943932e-05, "loss": 0.2834283113479614, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.32767, "step": 113, "tokens/total": 3929584, "tokens/train_per_sec_per_gpu": 71.78, "tokens/trainable": 146394} +{"epoch": 0.4453125, "grad_norm": 1.2927894592285156, "learning_rate": 9.2941779782269e-05, "loss": 0.30605125427246094, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.35805, "step": 114, "tokens/total": 3963632, "tokens/train_per_sec_per_gpu": 73.84, "tokens/trainable": 147669} +{"epoch": 0.44921875, "grad_norm": 1.5188899040222168, "learning_rate": 9.278490855781596e-05, "loss": 0.3363090455532074, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 37.39, "memory/max_allocated (GiB)": 37.39, "ppl": 1.39977, "step": 115, "tokens/total": 3996032, "tokens/train_per_sec_per_gpu": 85.1, "tokens/trainable": 148954} +{"epoch": 0.453125, "grad_norm": 3.1070661544799805, "learning_rate": 9.262646494908604e-05, "loss": 0.36761438846588135, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 39.19, "memory/max_allocated (GiB)": 39.19, "ppl": 1.44428, "step": 116, "tokens/total": 4034816, "tokens/train_per_sec_per_gpu": 64.03, "tokens/trainable": 150229} +{"epoch": 0.45703125, "grad_norm": 1.4367910623550415, "learning_rate": 9.246645554956457e-05, "loss": 0.34268718957901, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.40873, "step": 117, "tokens/total": 4069600, "tokens/train_per_sec_per_gpu": 73.64, "tokens/trainable": 151523} +{"epoch": 0.4609375, "grad_norm": 4.423839569091797, "learning_rate": 9.230488701789578e-05, "loss": 0.36700737476348877, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 38.81, "memory/max_allocated (GiB)": 38.81, "ppl": 1.44341, "step": 118, "tokens/total": 4107312, "tokens/train_per_sec_per_gpu": 67.85, "tokens/trainable": 152872} +{"epoch": 0.46484375, "grad_norm": 1.7230916023254395, "learning_rate": 9.214176607760577e-05, "loss": 0.3299608528614044, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.39091, "step": 119, "tokens/total": 4146032, "tokens/train_per_sec_per_gpu": 71.82, "tokens/trainable": 154218} +{"epoch": 0.46875, "grad_norm": 1.8157092332839966, "learning_rate": 9.197709951682268e-05, "loss": 0.31890395283699036, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 37.07, "memory/max_allocated (GiB)": 37.07, "ppl": 1.37562, "step": 120, "tokens/total": 4178576, "tokens/train_per_sec_per_gpu": 76.61, "tokens/trainable": 155465} +{"epoch": 0.47265625, "grad_norm": 1.5790475606918335, "learning_rate": 9.181089418799428e-05, "loss": 0.3183943033218384, "memory/device_reserved (GiB)": 42.86, "memory/max_active (GiB)": 39.66, "memory/max_allocated (GiB)": 39.66, "ppl": 1.37492, "step": 121, "tokens/total": 4215776, "tokens/train_per_sec_per_gpu": 68.12, "tokens/trainable": 156767} +{"epoch": 0.4765625, "grad_norm": 1.1504555940628052, "learning_rate": 9.164315700760271e-05, "loss": 0.28758418560028076, "memory/device_reserved (GiB)": 42.86, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.3332, "step": 122, "tokens/total": 4251904, "tokens/train_per_sec_per_gpu": 69.5, "tokens/trainable": 158048} +{"epoch": 0.48046875, "grad_norm": 1.2394897937774658, "learning_rate": 9.147389495587671e-05, "loss": 0.35999277234077454, "memory/device_reserved (GiB)": 42.86, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.43332, "step": 123, "tokens/total": 4283264, "tokens/train_per_sec_per_gpu": 84.62, "tokens/trainable": 159323} +{"epoch": 0.484375, "grad_norm": 2.089991331100464, "learning_rate": 9.130311507650116e-05, "loss": 0.27895334362983704, "memory/device_reserved (GiB)": 42.86, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.32175, "step": 124, "tokens/total": 4316256, "tokens/train_per_sec_per_gpu": 75.61, "tokens/trainable": 160633} +{"epoch": 0.48828125, "grad_norm": 1.6738383769989014, "learning_rate": 9.113082447632394e-05, "loss": 0.3239787518978119, "memory/device_reserved (GiB)": 42.86, "memory/max_active (GiB)": 37.31, "memory/max_allocated (GiB)": 37.31, "ppl": 1.38262, "step": 125, "tokens/total": 4348608, "tokens/train_per_sec_per_gpu": 90.79, "tokens/trainable": 161943} +{"epoch": 0.4921875, "grad_norm": 1.5800063610076904, "learning_rate": 9.09570303250602e-05, "loss": 0.3302222490310669, "memory/device_reserved (GiB)": 42.86, "memory/max_active (GiB)": 36.6, "memory/max_allocated (GiB)": 36.6, "ppl": 1.39128, "step": 126, "tokens/total": 4380720, "tokens/train_per_sec_per_gpu": 76.12, "tokens/trainable": 163237} +{"epoch": 0.49609375, "grad_norm": 1.6856719255447388, "learning_rate": 9.078173985499394e-05, "loss": 0.30424752831459045, "memory/device_reserved (GiB)": 42.86, "memory/max_active (GiB)": 35.92, "memory/max_allocated (GiB)": 35.92, "ppl": 1.3556, "step": 127, "tokens/total": 4411232, "tokens/train_per_sec_per_gpu": 82.14, "tokens/trainable": 164524} +{"epoch": 0.5, "grad_norm": 1.4185363054275513, "learning_rate": 9.060496036067713e-05, "loss": 0.30249011516571045, "memory/device_reserved (GiB)": 42.86, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.35322, "step": 128, "tokens/total": 4449136, "tokens/train_per_sec_per_gpu": 64.49, "tokens/trainable": 165825} +{"epoch": 0.50390625, "grad_norm": 1.3567209243774414, "learning_rate": 9.042669919862615e-05, "loss": 0.2831266522407532, "memory/device_reserved (GiB)": 42.86, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.32727, "step": 129, "tokens/total": 4481728, "tokens/train_per_sec_per_gpu": 84.29, "tokens/trainable": 167153} +{"epoch": 0.5078125, "grad_norm": 1.3781955242156982, "learning_rate": 9.024696378701557e-05, "loss": 0.28261807560920715, "memory/device_reserved (GiB)": 41.55, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.3266, "step": 130, "tokens/total": 4517456, "tokens/train_per_sec_per_gpu": 69.86, "tokens/trainable": 168421} +{"epoch": 0.51171875, "grad_norm": 1.3715064525604248, "learning_rate": 9.006576160536948e-05, "loss": 0.30772289633750916, "memory/device_reserved (GiB)": 41.55, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 1.36032, "step": 131, "tokens/total": 4552320, "tokens/train_per_sec_per_gpu": 67.16, "tokens/trainable": 169669} +{"epoch": 0.515625, "grad_norm": 2.671921491622925, "learning_rate": 8.988310019425035e-05, "loss": 0.2868803143501282, "memory/device_reserved (GiB)": 41.55, "memory/max_active (GiB)": 39.8, "memory/max_allocated (GiB)": 39.8, "ppl": 1.33226, "step": 132, "tokens/total": 4590496, "tokens/train_per_sec_per_gpu": 66.15, "tokens/trainable": 170950} +{"epoch": 0.51953125, "grad_norm": 1.238907814025879, "learning_rate": 8.969898715494506e-05, "loss": 0.2906343340873718, "memory/device_reserved (GiB)": 41.55, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.33728, "step": 133, "tokens/total": 4627520, "tokens/train_per_sec_per_gpu": 73.99, "tokens/trainable": 172310} +{"epoch": 0.5234375, "grad_norm": 1.0413216352462769, "learning_rate": 8.951343014914869e-05, "loss": 0.2697286009788513, "memory/device_reserved (GiB)": 41.55, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.30961, "step": 134, "tokens/total": 4664576, "tokens/train_per_sec_per_gpu": 71.14, "tokens/trainable": 173657} +{"epoch": 0.52734375, "grad_norm": 2.0983877182006836, "learning_rate": 8.932643689864568e-05, "loss": 0.2845917344093323, "memory/device_reserved (GiB)": 41.55, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.32922, "step": 135, "tokens/total": 4701280, "tokens/train_per_sec_per_gpu": 75.64, "tokens/trainable": 175011} +{"epoch": 0.53125, "grad_norm": 1.4116299152374268, "learning_rate": 8.913801518498845e-05, "loss": 0.2555689513683319, "memory/device_reserved (GiB)": 41.55, "memory/max_active (GiB)": 39.73, "memory/max_allocated (GiB)": 39.73, "ppl": 1.2912, "step": 136, "tokens/total": 4740928, "tokens/train_per_sec_per_gpu": 62.67, "tokens/trainable": 176361} +{"epoch": 0.53515625, "grad_norm": 1.640179991722107, "learning_rate": 8.894817284917364e-05, "loss": 0.3152729272842407, "memory/device_reserved (GiB)": 41.55, "memory/max_active (GiB)": 39.32, "memory/max_allocated (GiB)": 39.32, "ppl": 1.37063, "step": 137, "tokens/total": 4778320, "tokens/train_per_sec_per_gpu": 63.25, "tokens/trainable": 177665} +{"epoch": 0.5390625, "grad_norm": 1.4450653791427612, "learning_rate": 8.875691779131569e-05, "loss": 0.2723386883735657, "memory/device_reserved (GiB)": 41.55, "memory/max_active (GiB)": 39.8, "memory/max_allocated (GiB)": 39.8, "ppl": 1.31303, "step": 138, "tokens/total": 4816336, "tokens/train_per_sec_per_gpu": 66.62, "tokens/trainable": 178972} +{"epoch": 0.54296875, "grad_norm": 1.316016674041748, "learning_rate": 8.856425797031829e-05, "loss": 0.27445754408836365, "memory/device_reserved (GiB)": 42.71, "memory/max_active (GiB)": 39.66, "memory/max_allocated (GiB)": 39.66, "ppl": 1.31582, "step": 139, "tokens/total": 4852544, "tokens/train_per_sec_per_gpu": 78.16, "tokens/trainable": 180321} +{"epoch": 0.546875, "grad_norm": 0.9660085439682007, "learning_rate": 8.837020140354295e-05, "loss": 0.252703994512558, "memory/device_reserved (GiB)": 42.71, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.2875, "step": 140, "tokens/total": 4889760, "tokens/train_per_sec_per_gpu": 70.51, "tokens/trainable": 181622} +{"epoch": 0.55078125, "grad_norm": 0.9772905111312866, "learning_rate": 8.817475616647554e-05, "loss": 0.2712472379207611, "memory/device_reserved (GiB)": 42.71, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.3116, "step": 141, "tokens/total": 4924432, "tokens/train_per_sec_per_gpu": 72.94, "tokens/trainable": 182953} +{"epoch": 0.5546875, "grad_norm": 1.386064052581787, "learning_rate": 8.797793039239017e-05, "loss": 0.2883537709712982, "memory/device_reserved (GiB)": 42.71, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.33423, "step": 142, "tokens/total": 4959792, "tokens/train_per_sec_per_gpu": 75.84, "tokens/trainable": 184249} +{"epoch": 0.55859375, "grad_norm": 1.175605058670044, "learning_rate": 8.777973227201069e-05, "loss": 0.2691478729248047, "memory/device_reserved (GiB)": 42.71, "memory/max_active (GiB)": 39.6, "memory/max_allocated (GiB)": 39.6, "ppl": 1.30885, "step": 143, "tokens/total": 4997184, "tokens/train_per_sec_per_gpu": 75.72, "tokens/trainable": 185583} +{"epoch": 0.5625, "grad_norm": 1.0715253353118896, "learning_rate": 8.758017005316988e-05, "loss": 0.29815053939819336, "memory/device_reserved (GiB)": 42.71, "memory/max_active (GiB)": 38.8, "memory/max_allocated (GiB)": 38.8, "ppl": 1.34736, "step": 144, "tokens/total": 5030976, "tokens/train_per_sec_per_gpu": 88.85, "tokens/trainable": 186909} +{"epoch": 0.56640625, "grad_norm": 1.711377739906311, "learning_rate": 8.737925204046629e-05, "loss": 0.3184908628463745, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.37505, "step": 145, "tokens/total": 5066048, "tokens/train_per_sec_per_gpu": 68.59, "tokens/trainable": 188186} +{"epoch": 0.5703125, "grad_norm": 1.518600344657898, "learning_rate": 8.717698659491851e-05, "loss": 0.2754281163215637, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 39.04, "memory/max_allocated (GiB)": 39.04, "ppl": 1.31709, "step": 146, "tokens/total": 5103152, "tokens/train_per_sec_per_gpu": 68.15, "tokens/trainable": 189562} +{"epoch": 0.57421875, "grad_norm": 1.724953532218933, "learning_rate": 8.697338213361735e-05, "loss": 0.24454666674137115, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.27704, "step": 147, "tokens/total": 5134624, "tokens/train_per_sec_per_gpu": 85.65, "tokens/trainable": 190959} +{"epoch": 0.578125, "grad_norm": 1.0983506441116333, "learning_rate": 8.676844712937552e-05, "loss": 0.25317469239234924, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.28811, "step": 148, "tokens/total": 5174752, "tokens/train_per_sec_per_gpu": 60.26, "tokens/trainable": 192252} +{"epoch": 0.58203125, "grad_norm": 2.906177520751953, "learning_rate": 8.656219011037509e-05, "loss": 0.26202118396759033, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 39.08, "memory/max_allocated (GiB)": 39.08, "ppl": 1.29955, "step": 149, "tokens/total": 5212160, "tokens/train_per_sec_per_gpu": 68.82, "tokens/trainable": 193567} +{"epoch": 0.5859375, "grad_norm": 1.3004529476165771, "learning_rate": 8.63546196598125e-05, "loss": 0.2877487540245056, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 39.32, "memory/max_allocated (GiB)": 39.32, "ppl": 1.33342, "step": 150, "tokens/total": 5250224, "tokens/train_per_sec_per_gpu": 66.01, "tokens/trainable": 194922} +{"epoch": 0.58984375, "grad_norm": 1.432615876197815, "learning_rate": 8.614574441554145e-05, "loss": 0.36959201097488403, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.44714, "step": 151, "tokens/total": 5283024, "tokens/train_per_sec_per_gpu": 76.58, "tokens/trainable": 196117} +{"epoch": 0.59375, "grad_norm": 1.893045425415039, "learning_rate": 8.593557306971349e-05, "loss": 0.248539537191391, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.28215, "step": 152, "tokens/total": 5316528, "tokens/train_per_sec_per_gpu": 72.47, "tokens/trainable": 197428} +{"epoch": 0.59765625, "grad_norm": 2.8978984355926514, "learning_rate": 8.572411436841618e-05, "loss": 0.30015772581100464, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 36.96, "memory/max_allocated (GiB)": 36.96, "ppl": 1.35007, "step": 153, "tokens/total": 5350432, "tokens/train_per_sec_per_gpu": 76.06, "tokens/trainable": 198736} +{"epoch": 0.6015625, "grad_norm": 1.3107554912567139, "learning_rate": 8.551137711130922e-05, "loss": 0.270149827003479, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 38.51, "memory/max_allocated (GiB)": 38.51, "ppl": 1.31016, "step": 154, "tokens/total": 5382512, "tokens/train_per_sec_per_gpu": 75.17, "tokens/trainable": 200003} +{"epoch": 0.60546875, "grad_norm": 0.9600314497947693, "learning_rate": 8.529737015125824e-05, "loss": 0.2310599833726883, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 38.29, "memory/max_allocated (GiB)": 38.29, "ppl": 1.25993, "step": 155, "tokens/total": 5417952, "tokens/train_per_sec_per_gpu": 70.39, "tokens/trainable": 201292} +{"epoch": 0.609375, "grad_norm": 1.464085578918457, "learning_rate": 8.508210239396639e-05, "loss": 0.27726972103118896, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 39.85, "memory/max_allocated (GiB)": 39.85, "ppl": 1.31952, "step": 156, "tokens/total": 5456000, "tokens/train_per_sec_per_gpu": 67.97, "tokens/trainable": 202574} +{"epoch": 0.61328125, "grad_norm": 1.0673232078552246, "learning_rate": 8.486558279760375e-05, "loss": 0.28457099199295044, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 35.9, "memory/max_allocated (GiB)": 35.9, "ppl": 1.32919, "step": 157, "tokens/total": 5487072, "tokens/train_per_sec_per_gpu": 81.63, "tokens/trainable": 203817} +{"epoch": 0.6171875, "grad_norm": 1.375657081604004, "learning_rate": 8.464782037243449e-05, "loss": 0.282906174659729, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 36.93, "memory/max_allocated (GiB)": 36.93, "ppl": 1.32698, "step": 158, "tokens/total": 5519504, "tokens/train_per_sec_per_gpu": 75.37, "tokens/trainable": 205052} +{"epoch": 0.62109375, "grad_norm": 1.0413556098937988, "learning_rate": 8.442882418044202e-05, "loss": 0.24283498525619507, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.27486, "step": 159, "tokens/total": 5552464, "tokens/train_per_sec_per_gpu": 77.07, "tokens/trainable": 206373} +{"epoch": 0.625, "grad_norm": 1.1860963106155396, "learning_rate": 8.420860333495179e-05, "loss": 0.23340386152267456, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.26289, "step": 160, "tokens/total": 5585408, "tokens/train_per_sec_per_gpu": 76.59, "tokens/trainable": 207580} +{"epoch": 0.62890625, "grad_norm": 1.914342999458313, "learning_rate": 8.398716700025208e-05, "loss": 0.2888563275337219, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 37.61, "memory/max_allocated (GiB)": 37.61, "ppl": 1.3349, "step": 161, "tokens/total": 5619088, "tokens/train_per_sec_per_gpu": 70.54, "tokens/trainable": 208806} +{"epoch": 0.6328125, "grad_norm": 1.914615511894226, "learning_rate": 8.376452439121266e-05, "loss": 0.27319464087486267, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 36.06, "memory/max_allocated (GiB)": 36.06, "ppl": 1.31416, "step": 162, "tokens/total": 5649424, "tokens/train_per_sec_per_gpu": 85.19, "tokens/trainable": 210048} +{"epoch": 0.63671875, "grad_norm": 2.2476861476898193, "learning_rate": 8.354068477290124e-05, "loss": 0.2990860939025879, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.34863, "step": 163, "tokens/total": 5684000, "tokens/train_per_sec_per_gpu": 78.44, "tokens/trainable": 211344} +{"epoch": 0.640625, "grad_norm": 1.4327541589736938, "learning_rate": 8.331565746019807e-05, "loss": 0.2515588402748108, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 36.78, "memory/max_allocated (GiB)": 36.78, "ppl": 1.28603, "step": 164, "tokens/total": 5716336, "tokens/train_per_sec_per_gpu": 84.95, "tokens/trainable": 212678} +{"epoch": 0.64453125, "grad_norm": 1.7774714231491089, "learning_rate": 8.308945181740812e-05, "loss": 0.2755957841873169, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.31732, "step": 165, "tokens/total": 5751264, "tokens/train_per_sec_per_gpu": 75.53, "tokens/trainable": 213937} +{"epoch": 0.6484375, "grad_norm": 1.299171805381775, "learning_rate": 8.286207725787153e-05, "loss": 0.24142053723335266, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 39.14, "memory/max_allocated (GiB)": 39.14, "ppl": 1.27306, "step": 166, "tokens/total": 5788480, "tokens/train_per_sec_per_gpu": 64.04, "tokens/trainable": 215182} +{"epoch": 0.65234375, "grad_norm": 1.5258986949920654, "learning_rate": 8.263354324357182e-05, "loss": 0.2413410246372223, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.27296, "step": 167, "tokens/total": 5823536, "tokens/train_per_sec_per_gpu": 73.53, "tokens/trainable": 216504} +{"epoch": 0.65625, "grad_norm": 1.1236845254898071, "learning_rate": 8.240385928474219e-05, "loss": 0.28490567207336426, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 36.92, "memory/max_allocated (GiB)": 36.92, "ppl": 1.32964, "step": 168, "tokens/total": 5856576, "tokens/train_per_sec_per_gpu": 72.1, "tokens/trainable": 217734} +{"epoch": 0.66015625, "grad_norm": 2.621943712234497, "learning_rate": 8.217303493946967e-05, "loss": 0.24820928275585175, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 37.44, "memory/max_allocated (GiB)": 37.44, "ppl": 1.28173, "step": 169, "tokens/total": 5891216, "tokens/train_per_sec_per_gpu": 72.55, "tokens/trainable": 219014} +{"epoch": 0.6640625, "grad_norm": 1.0171617269515991, "learning_rate": 8.194107981329746e-05, "loss": 0.23899304866790771, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 39.87, "memory/max_allocated (GiB)": 39.87, "ppl": 1.26997, "step": 170, "tokens/total": 5930016, "tokens/train_per_sec_per_gpu": 65.34, "tokens/trainable": 220298} +{"epoch": 0.66796875, "grad_norm": 1.8385276794433594, "learning_rate": 8.170800355882518e-05, "loss": 0.23707425594329834, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 37.4, "memory/max_allocated (GiB)": 37.4, "ppl": 1.26754, "step": 171, "tokens/total": 5964720, "tokens/train_per_sec_per_gpu": 84.16, "tokens/trainable": 221652} +{"epoch": 0.671875, "grad_norm": 2.1377971172332764, "learning_rate": 8.147381587530713e-05, "loss": 0.2740234136581421, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 1.31525, "step": 172, "tokens/total": 5998656, "tokens/train_per_sec_per_gpu": 77.72, "tokens/trainable": 222915} +{"epoch": 0.67578125, "grad_norm": 2.2008450031280518, "learning_rate": 8.123852650824877e-05, "loss": 0.26808875799179077, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.30746, "step": 173, "tokens/total": 6032864, "tokens/train_per_sec_per_gpu": 79.01, "tokens/trainable": 224251} +{"epoch": 0.6796875, "grad_norm": 2.219040870666504, "learning_rate": 8.100214524900103e-05, "loss": 0.2805442214012146, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 38.7, "memory/max_allocated (GiB)": 38.7, "ppl": 1.32385, "step": 174, "tokens/total": 6068416, "tokens/train_per_sec_per_gpu": 63.04, "tokens/trainable": 225481} +{"epoch": 0.68359375, "grad_norm": 1.6199928522109985, "learning_rate": 8.076468193435301e-05, "loss": 0.26683998107910156, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 36.56, "memory/max_allocated (GiB)": 36.56, "ppl": 1.30583, "step": 175, "tokens/total": 6099584, "tokens/train_per_sec_per_gpu": 77.67, "tokens/trainable": 226687} +{"epoch": 0.6875, "grad_norm": 3.2694060802459717, "learning_rate": 8.052614644612253e-05, "loss": 0.25157254934310913, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 39.46, "memory/max_allocated (GiB)": 39.46, "ppl": 1.28605, "step": 176, "tokens/total": 6137616, "tokens/train_per_sec_per_gpu": 62.88, "tokens/trainable": 228007} +{"epoch": 0.69140625, "grad_norm": 1.3123893737792969, "learning_rate": 8.028654871074489e-05, "loss": 0.22487136721611023, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.25216, "step": 177, "tokens/total": 6172464, "tokens/train_per_sec_per_gpu": 74.87, "tokens/trainable": 229337} +{"epoch": 0.6953125, "grad_norm": 1.59602689743042, "learning_rate": 8.004589869885986e-05, "loss": 0.2644363045692444, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.3027, "step": 178, "tokens/total": 6204176, "tokens/train_per_sec_per_gpu": 77.06, "tokens/trainable": 230572} +{"epoch": 0.69921875, "grad_norm": 1.2936711311340332, "learning_rate": 7.980420642489674e-05, "loss": 0.2421567142009735, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 1.27399, "step": 179, "tokens/total": 6243152, "tokens/train_per_sec_per_gpu": 68.35, "tokens/trainable": 231896} +{"epoch": 0.703125, "grad_norm": 1.7858492136001587, "learning_rate": 7.95614819466576e-05, "loss": 0.22769716382026672, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 37.67, "memory/max_allocated (GiB)": 37.67, "ppl": 1.2557, "step": 180, "tokens/total": 6277184, "tokens/train_per_sec_per_gpu": 79.26, "tokens/trainable": 233257} +{"epoch": 0.70703125, "grad_norm": 1.2538331747055054, "learning_rate": 7.931773536489872e-05, "loss": 0.2478637546300888, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.28129, "step": 181, "tokens/total": 6311968, "tokens/train_per_sec_per_gpu": 75.19, "tokens/trainable": 234543} +{"epoch": 0.7109375, "grad_norm": 6.28990364074707, "learning_rate": 7.907297682291035e-05, "loss": 0.24471375346183777, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 39.48, "memory/max_allocated (GiB)": 39.48, "ppl": 1.27726, "step": 182, "tokens/total": 6346864, "tokens/train_per_sec_per_gpu": 76.47, "tokens/trainable": 235857} +{"epoch": 0.71484375, "grad_norm": 2.0770928859710693, "learning_rate": 7.882721650609442e-05, "loss": 0.19874247908592224, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 39.65, "memory/max_allocated (GiB)": 39.65, "ppl": 1.21987, "step": 183, "tokens/total": 6387408, "tokens/train_per_sec_per_gpu": 65.36, "tokens/trainable": 237306} +{"epoch": 0.71875, "grad_norm": 1.443579077720642, "learning_rate": 7.85804646415409e-05, "loss": 0.25196373462677, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 37.96, "memory/max_allocated (GiB)": 37.96, "ppl": 1.28655, "step": 184, "tokens/total": 6422992, "tokens/train_per_sec_per_gpu": 78.91, "tokens/trainable": 238657} +{"epoch": 0.72265625, "grad_norm": 9.492624282836914, "learning_rate": 7.833273149760207e-05, "loss": 0.2595166563987732, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.2963, "step": 185, "tokens/total": 6453888, "tokens/train_per_sec_per_gpu": 79.09, "tokens/trainable": 239929} +{"epoch": 0.7265625, "grad_norm": 5.435891628265381, "learning_rate": 7.808402738346527e-05, "loss": 0.2033417969942093, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.22549, "step": 186, "tokens/total": 6493120, "tokens/train_per_sec_per_gpu": 71.92, "tokens/trainable": 241341} +{"epoch": 0.73046875, "grad_norm": 1.2631343603134155, "learning_rate": 7.783436264872382e-05, "loss": 0.22726227343082428, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 39.12, "memory/max_allocated (GiB)": 39.12, "ppl": 1.25516, "step": 187, "tokens/total": 6530208, "tokens/train_per_sec_per_gpu": 70.36, "tokens/trainable": 242692} +{"epoch": 0.734375, "grad_norm": 4.41347074508667, "learning_rate": 7.758374768294647e-05, "loss": 0.2068699300289154, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.22982, "step": 188, "tokens/total": 6566384, "tokens/train_per_sec_per_gpu": 78.17, "tokens/trainable": 244105} +{"epoch": 0.73828125, "grad_norm": 1.2839323282241821, "learning_rate": 7.733219291524489e-05, "loss": 0.22338077425956726, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.2503, "step": 189, "tokens/total": 6600192, "tokens/train_per_sec_per_gpu": 80.79, "tokens/trainable": 245427} +{"epoch": 0.7421875, "grad_norm": 1.0738022327423096, "learning_rate": 7.707970881383977e-05, "loss": 0.22573694586753845, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 36.47, "memory/max_allocated (GiB)": 36.47, "ppl": 1.25325, "step": 190, "tokens/total": 6632096, "tokens/train_per_sec_per_gpu": 81.73, "tokens/trainable": 246660} +{"epoch": 0.74609375, "grad_norm": 1.7970480918884277, "learning_rate": 7.682630588562518e-05, "loss": 0.22183211147785187, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 38.81, "memory/max_allocated (GiB)": 38.81, "ppl": 1.24836, "step": 191, "tokens/total": 6667568, "tokens/train_per_sec_per_gpu": 72.22, "tokens/trainable": 247995} +{"epoch": 0.75, "grad_norm": 1.3715400695800781, "learning_rate": 7.657199467573129e-05, "loss": 0.21493053436279297, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 37.23, "memory/max_allocated (GiB)": 37.23, "ppl": 1.23978, "step": 192, "tokens/total": 6701872, "tokens/train_per_sec_per_gpu": 72.57, "tokens/trainable": 249251} +{"epoch": 0.75390625, "grad_norm": 2.3457701206207275, "learning_rate": 7.631678576708561e-05, "loss": 0.19420774281024933, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 38.16, "memory/max_allocated (GiB)": 38.16, "ppl": 1.21435, "step": 193, "tokens/total": 6736512, "tokens/train_per_sec_per_gpu": 83.13, "tokens/trainable": 250631} +{"epoch": 0.7578125, "grad_norm": 1.311965823173523, "learning_rate": 7.606068977997255e-05, "loss": 0.21398723125457764, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 39.9, "memory/max_allocated (GiB)": 39.9, "ppl": 1.23861, "step": 194, "tokens/total": 6774592, "tokens/train_per_sec_per_gpu": 76.8, "tokens/trainable": 252066} +{"epoch": 0.76171875, "grad_norm": 2.1191561222076416, "learning_rate": 7.580371737159148e-05, "loss": 0.2778017520904541, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.32022, "step": 195, "tokens/total": 6810832, "tokens/train_per_sec_per_gpu": 65.85, "tokens/trainable": 253290} +{"epoch": 0.765625, "grad_norm": 1.2818502187728882, "learning_rate": 7.554587923561324e-05, "loss": 0.20552337169647217, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 39.11, "memory/max_allocated (GiB)": 39.11, "ppl": 1.22817, "step": 196, "tokens/total": 6845808, "tokens/train_per_sec_per_gpu": 70.63, "tokens/trainable": 254563} +{"epoch": 0.76953125, "grad_norm": 171.5685272216797, "learning_rate": 7.528718610173511e-05, "loss": 0.20336805284023285, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.22552, "step": 197, "tokens/total": 6881488, "tokens/train_per_sec_per_gpu": 78.85, "tokens/trainable": 255926} +{"epoch": 0.7734375, "grad_norm": 1.2081172466278076, "learning_rate": 7.502764873523431e-05, "loss": 0.22422312200069427, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.25135, "step": 198, "tokens/total": 6916016, "tokens/train_per_sec_per_gpu": 76.63, "tokens/trainable": 257223} +{"epoch": 0.77734375, "grad_norm": 0.9004325270652771, "learning_rate": 7.476727793652011e-05, "loss": 0.18551477789878845, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 36.92, "memory/max_allocated (GiB)": 36.92, "ppl": 1.20384, "step": 199, "tokens/total": 6949664, "tokens/train_per_sec_per_gpu": 77.88, "tokens/trainable": 258536} +{"epoch": 0.78125, "grad_norm": 1.2753397226333618, "learning_rate": 7.450608454068415e-05, "loss": 0.2056078016757965, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 1.22827, "step": 200, "tokens/total": 6984432, "tokens/train_per_sec_per_gpu": 74.29, "tokens/trainable": 259858} +{"epoch": 0.78515625, "grad_norm": 1.6299567222595215, "learning_rate": 7.424407941704987e-05, "loss": 0.1969333291053772, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 38.47, "memory/max_allocated (GiB)": 38.47, "ppl": 1.21766, "step": 201, "tokens/total": 7021232, "tokens/train_per_sec_per_gpu": 76.73, "tokens/trainable": 261233} +{"epoch": 0.7890625, "grad_norm": 1.189399003982544, "learning_rate": 7.398127346871986e-05, "loss": 0.19378256797790527, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 39.77, "memory/max_allocated (GiB)": 39.77, "ppl": 1.21383, "step": 202, "tokens/total": 7058592, "tokens/train_per_sec_per_gpu": 72.24, "tokens/trainable": 262588} +{"epoch": 0.79296875, "grad_norm": 1.0025008916854858, "learning_rate": 7.371767763212238e-05, "loss": 0.20812347531318665, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 39.71, "memory/max_allocated (GiB)": 39.71, "ppl": 1.23137, "step": 203, "tokens/total": 7094400, "tokens/train_per_sec_per_gpu": 76.01, "tokens/trainable": 263884} +{"epoch": 0.796875, "grad_norm": 1.1986677646636963, "learning_rate": 7.345330287655617e-05, "loss": 0.22623129189014435, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.25387, "step": 204, "tokens/total": 7132704, "tokens/train_per_sec_per_gpu": 63.15, "tokens/trainable": 265188} +{"epoch": 0.80078125, "grad_norm": 1.236383318901062, "learning_rate": 7.31881602037339e-05, "loss": 0.21883943676948547, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.24463, "step": 205, "tokens/total": 7164928, "tokens/train_per_sec_per_gpu": 75.17, "tokens/trainable": 266430} +{"epoch": 0.8046875, "grad_norm": 1.1965690851211548, "learning_rate": 7.29222606473245e-05, "loss": 0.21111100912094116, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.23505, "step": 206, "tokens/total": 7197808, "tokens/train_per_sec_per_gpu": 79.08, "tokens/trainable": 267688} +{"epoch": 0.80859375, "grad_norm": 1.7323898077011108, "learning_rate": 7.265561527249383e-05, "loss": 0.19730663299560547, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.21812, "step": 207, "tokens/total": 7232208, "tokens/train_per_sec_per_gpu": 77.21, "tokens/trainable": 268955} +{"epoch": 0.8125, "grad_norm": 1.1817301511764526, "learning_rate": 7.238823517544436e-05, "loss": 0.18628978729248047, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 39.53, "memory/max_allocated (GiB)": 39.53, "ppl": 1.20477, "step": 208, "tokens/total": 7269424, "tokens/train_per_sec_per_gpu": 80.05, "tokens/trainable": 270385} +{"epoch": 0.81640625, "grad_norm": 1.1813452243804932, "learning_rate": 7.212013148295333e-05, "loss": 0.1997954398393631, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 37.61, "memory/max_allocated (GiB)": 37.61, "ppl": 1.22115, "step": 209, "tokens/total": 7303728, "tokens/train_per_sec_per_gpu": 75.99, "tokens/trainable": 271700} +{"epoch": 0.8203125, "grad_norm": 1.9560060501098633, "learning_rate": 7.185131535190975e-05, "loss": 0.20535463094711304, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.22796, "step": 210, "tokens/total": 7336688, "tokens/train_per_sec_per_gpu": 79.03, "tokens/trainable": 272992} +{"epoch": 0.82421875, "grad_norm": 1.4059473276138306, "learning_rate": 7.158179796885005e-05, "loss": 0.20502334833145142, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 37.82, "memory/max_allocated (GiB)": 37.82, "ppl": 1.22755, "step": 211, "tokens/total": 7371360, "tokens/train_per_sec_per_gpu": 79.1, "tokens/trainable": 274283} +{"epoch": 0.828125, "grad_norm": 2.301706552505493, "learning_rate": 7.131159054949273e-05, "loss": 0.21950608491897583, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.24546, "step": 212, "tokens/total": 7410224, "tokens/train_per_sec_per_gpu": 67.3, "tokens/trainable": 275594} +{"epoch": 0.83203125, "grad_norm": 1.112581729888916, "learning_rate": 7.104070433827139e-05, "loss": 0.2181938737630844, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.24383, "step": 213, "tokens/total": 7445936, "tokens/train_per_sec_per_gpu": 65.7, "tokens/trainable": 276794} +{"epoch": 0.8359375, "grad_norm": 2.193361759185791, "learning_rate": 7.076915060786705e-05, "loss": 0.21106556057929993, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.23499, "step": 214, "tokens/total": 7478848, "tokens/train_per_sec_per_gpu": 77.72, "tokens/trainable": 278081} +{"epoch": 0.83984375, "grad_norm": 1.6214195489883423, "learning_rate": 7.049694065873882e-05, "loss": 0.20253780484199524, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 38.14, "memory/max_allocated (GiB)": 38.14, "ppl": 1.22451, "step": 215, "tokens/total": 7513712, "tokens/train_per_sec_per_gpu": 72.34, "tokens/trainable": 279340} +{"epoch": 0.84375, "grad_norm": 2.026724100112915, "learning_rate": 7.022408581865382e-05, "loss": 0.1862001121044159, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.20466, "step": 216, "tokens/total": 7548176, "tokens/train_per_sec_per_gpu": 70.67, "tokens/trainable": 280650} +{"epoch": 0.84765625, "grad_norm": 1.009461760520935, "learning_rate": 6.99505974422157e-05, "loss": 0.17370888590812683, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.18971, "step": 217, "tokens/total": 7582912, "tokens/train_per_sec_per_gpu": 82.77, "tokens/trainable": 282009} +{"epoch": 0.8515625, "grad_norm": 2.9321558475494385, "learning_rate": 6.967648691039213e-05, "loss": 0.19997256994247437, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 36.77, "memory/max_allocated (GiB)": 36.77, "ppl": 1.22137, "step": 218, "tokens/total": 7615152, "tokens/train_per_sec_per_gpu": 83.67, "tokens/trainable": 283317} +{"epoch": 0.85546875, "grad_norm": 4.149662971496582, "learning_rate": 6.940176563004123e-05, "loss": 0.2139720916748047, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.23859, "step": 219, "tokens/total": 7649536, "tokens/train_per_sec_per_gpu": 70.56, "tokens/trainable": 284524} +{"epoch": 0.859375, "grad_norm": 1.5869927406311035, "learning_rate": 6.912644503343682e-05, "loss": 0.2107432335615158, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.2346, "step": 220, "tokens/total": 7687968, "tokens/train_per_sec_per_gpu": 70.01, "tokens/trainable": 285814} +{"epoch": 0.86328125, "grad_norm": 1.4785083532333374, "learning_rate": 6.885053657779273e-05, "loss": 0.20233039557933807, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 36.49, "memory/max_allocated (GiB)": 36.49, "ppl": 1.22425, "step": 221, "tokens/total": 7719808, "tokens/train_per_sec_per_gpu": 78.27, "tokens/trainable": 287102} +{"epoch": 0.8671875, "grad_norm": 2.248838424682617, "learning_rate": 6.857405174478604e-05, "loss": 0.2265617996454239, "memory/device_reserved (GiB)": 42.72, "memory/max_active (GiB)": 37.45, "memory/max_allocated (GiB)": 37.45, "ppl": 1.25428, "step": 222, "tokens/total": 7754800, "tokens/train_per_sec_per_gpu": 73.54, "tokens/trainable": 288384} +{"epoch": 0.87109375, "grad_norm": 1.267303705215454, "learning_rate": 6.82970020400792e-05, "loss": 0.21917405724525452, "memory/device_reserved (GiB)": 42.98, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.24505, "step": 223, "tokens/total": 7792224, "tokens/train_per_sec_per_gpu": 61.76, "tokens/trainable": 289612} +{"epoch": 0.875, "grad_norm": 1.4997178316116333, "learning_rate": 6.801939899284132e-05, "loss": 0.17532993853092194, "memory/device_reserved (GiB)": 42.98, "memory/max_active (GiB)": 38.34, "memory/max_allocated (GiB)": 38.34, "ppl": 1.19164, "step": 224, "tokens/total": 7828336, "tokens/train_per_sec_per_gpu": 74.82, "tokens/trainable": 290972} +{"epoch": 0.87890625, "grad_norm": 1.118706464767456, "learning_rate": 6.774125415526827e-05, "loss": 0.1724601686000824, "memory/device_reserved (GiB)": 42.98, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.18822, "step": 225, "tokens/total": 7861808, "tokens/train_per_sec_per_gpu": 81.88, "tokens/trainable": 292330} +{"epoch": 0.8828125, "grad_norm": 1.1896799802780151, "learning_rate": 6.746257910210214e-05, "loss": 0.19755858182907104, "memory/device_reserved (GiB)": 42.98, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.21842, "step": 226, "tokens/total": 7895216, "tokens/train_per_sec_per_gpu": 73.12, "tokens/trainable": 293599} +{"epoch": 0.88671875, "grad_norm": 2.9864840507507324, "learning_rate": 6.718338543014937e-05, "loss": 0.21449477970600128, "memory/device_reserved (GiB)": 42.98, "memory/max_active (GiB)": 35.83, "memory/max_allocated (GiB)": 35.83, "ppl": 1.23924, "step": 227, "tokens/total": 7925584, "tokens/train_per_sec_per_gpu": 80.81, "tokens/trainable": 294827} +{"epoch": 0.890625, "grad_norm": 1.1270867586135864, "learning_rate": 6.69036847577983e-05, "loss": 0.1926630437374115, "memory/device_reserved (GiB)": 42.98, "memory/max_active (GiB)": 38.29, "memory/max_allocated (GiB)": 38.29, "ppl": 1.21247, "step": 228, "tokens/total": 7959664, "tokens/train_per_sec_per_gpu": 73.01, "tokens/trainable": 296096} +{"epoch": 0.89453125, "grad_norm": 0.9095357060432434, "learning_rate": 6.662348872453553e-05, "loss": 0.1807107925415039, "memory/device_reserved (GiB)": 42.98, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.19807, "step": 229, "tokens/total": 7996368, "tokens/train_per_sec_per_gpu": 67.55, "tokens/trainable": 297343} +{"epoch": 0.8984375, "grad_norm": 1.0853289365768433, "learning_rate": 6.63428089904618e-05, "loss": 0.19570578634738922, "memory/device_reserved (GiB)": 42.98, "memory/max_active (GiB)": 39.35, "memory/max_allocated (GiB)": 39.35, "ppl": 1.21617, "step": 230, "tokens/total": 8035744, "tokens/train_per_sec_per_gpu": 61.38, "tokens/trainable": 298600} +{"epoch": 0.90234375, "grad_norm": 1.693424940109253, "learning_rate": 6.60616572358065e-05, "loss": 0.19278845191001892, "memory/device_reserved (GiB)": 42.98, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.21263, "step": 231, "tokens/total": 8070368, "tokens/train_per_sec_per_gpu": 75.13, "tokens/trainable": 299839} +{"epoch": 0.90625, "grad_norm": 0.909553587436676, "learning_rate": 6.578004516044172e-05, "loss": 0.19129043817520142, "memory/device_reserved (GiB)": 42.98, "memory/max_active (GiB)": 37.96, "memory/max_allocated (GiB)": 37.96, "ppl": 1.21081, "step": 232, "tokens/total": 8106560, "tokens/train_per_sec_per_gpu": 76.16, "tokens/trainable": 301114} +{"epoch": 0.91015625, "grad_norm": 1.1015841960906982, "learning_rate": 6.549798448339548e-05, "loss": 0.1764211803674698, "memory/device_reserved (GiB)": 42.98, "memory/max_active (GiB)": 38.86, "memory/max_allocated (GiB)": 38.86, "ppl": 1.19294, "step": 233, "tokens/total": 8144480, "tokens/train_per_sec_per_gpu": 75.5, "tokens/trainable": 302447} +{"epoch": 0.9140625, "grad_norm": 1.4194718599319458, "learning_rate": 6.521548694236384e-05, "loss": 0.18258652091026306, "memory/device_reserved (GiB)": 42.98, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.20032, "step": 234, "tokens/total": 8182032, "tokens/train_per_sec_per_gpu": 68.99, "tokens/trainable": 303763} +{"epoch": 0.91796875, "grad_norm": 1.11892569065094, "learning_rate": 6.493256429322259e-05, "loss": 0.19601812958717346, "memory/device_reserved (GiB)": 42.98, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.21655, "step": 235, "tokens/total": 8218320, "tokens/train_per_sec_per_gpu": 69.0, "tokens/trainable": 305061} +{"epoch": 0.921875, "grad_norm": 0.9834786653518677, "learning_rate": 6.464922830953799e-05, "loss": 0.18479004502296448, "memory/device_reserved (GiB)": 42.98, "memory/max_active (GiB)": 39.39, "memory/max_allocated (GiB)": 39.39, "ppl": 1.20297, "step": 236, "tokens/total": 8257824, "tokens/train_per_sec_per_gpu": 64.92, "tokens/trainable": 306389} +{"epoch": 0.92578125, "grad_norm": 1.3234758377075195, "learning_rate": 6.436549078207688e-05, "loss": 0.1769946962594986, "memory/device_reserved (GiB)": 42.98, "memory/max_active (GiB)": 38.3, "memory/max_allocated (GiB)": 38.3, "ppl": 1.19362, "step": 237, "tokens/total": 8291136, "tokens/train_per_sec_per_gpu": 79.18, "tokens/trainable": 307669} +{"epoch": 0.9296875, "grad_norm": 9.62660026550293, "learning_rate": 6.408136351831592e-05, "loss": 0.18779928982257843, "memory/device_reserved (GiB)": 42.98, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.20659, "step": 238, "tokens/total": 8323728, "tokens/train_per_sec_per_gpu": 75.41, "tokens/trainable": 308943} +{"epoch": 0.93359375, "grad_norm": 1.4259083271026611, "learning_rate": 6.379685834195036e-05, "loss": 0.17850759625434875, "memory/device_reserved (GiB)": 42.98, "memory/max_active (GiB)": 39.11, "memory/max_allocated (GiB)": 39.11, "ppl": 1.19543, "step": 239, "tokens/total": 8363472, "tokens/train_per_sec_per_gpu": 66.2, "tokens/trainable": 310344} +{"epoch": 0.9375, "grad_norm": 1.2575114965438843, "learning_rate": 6.351198709240186e-05, "loss": 0.1710643768310547, "memory/device_reserved (GiB)": 42.98, "memory/max_active (GiB)": 39.73, "memory/max_allocated (GiB)": 39.73, "ppl": 1.18657, "step": 240, "tokens/total": 8403968, "tokens/train_per_sec_per_gpu": 63.05, "tokens/trainable": 311678} +{"epoch": 0.94140625, "grad_norm": 1.0347704887390137, "learning_rate": 6.32267616243259e-05, "loss": 0.18269199132919312, "memory/device_reserved (GiB)": 42.98, "memory/max_active (GiB)": 36.67, "memory/max_allocated (GiB)": 36.67, "ppl": 1.20044, "step": 241, "tokens/total": 8436688, "tokens/train_per_sec_per_gpu": 78.79, "tokens/trainable": 312964} +{"epoch": 0.9453125, "grad_norm": 1.5188554525375366, "learning_rate": 6.294119380711849e-05, "loss": 0.18461650609970093, "memory/device_reserved (GiB)": 42.98, "memory/max_active (GiB)": 38.7, "memory/max_allocated (GiB)": 38.7, "ppl": 1.20276, "step": 242, "tokens/total": 8471328, "tokens/train_per_sec_per_gpu": 68.75, "tokens/trainable": 314219} +{"epoch": 0.94921875, "grad_norm": 1.455224633216858, "learning_rate": 6.265529552442209e-05, "loss": 0.18494752049446106, "memory/device_reserved (GiB)": 42.98, "memory/max_active (GiB)": 37.83, "memory/max_allocated (GiB)": 37.83, "ppl": 1.20316, "step": 243, "tokens/total": 8506880, "tokens/train_per_sec_per_gpu": 74.14, "tokens/trainable": 315489} +{"epoch": 0.953125, "grad_norm": 4.209251403808594, "learning_rate": 6.236907867363127e-05, "loss": 0.20512723922729492, "memory/device_reserved (GiB)": 42.98, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 1.22768, "step": 244, "tokens/total": 8541472, "tokens/train_per_sec_per_gpu": 74.74, "tokens/trainable": 316758} +{"epoch": 0.95703125, "grad_norm": 1.6184160709381104, "learning_rate": 6.208255516539749e-05, "loss": 0.19370123744010925, "memory/device_reserved (GiB)": 42.98, "memory/max_active (GiB)": 36.44, "memory/max_allocated (GiB)": 36.44, "ppl": 1.21373, "step": 245, "tokens/total": 8573952, "tokens/train_per_sec_per_gpu": 75.74, "tokens/trainable": 318020} +{"epoch": 0.9609375, "grad_norm": 2.437911033630371, "learning_rate": 6.179573692313344e-05, "loss": 0.19573578238487244, "memory/device_reserved (GiB)": 43.1, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.21621, "step": 246, "tokens/total": 8611136, "tokens/train_per_sec_per_gpu": 71.98, "tokens/trainable": 319297} +{"epoch": 0.96484375, "grad_norm": 2.2362072467803955, "learning_rate": 6.150863588251694e-05, "loss": 0.18582412600517273, "memory/device_reserved (GiB)": 43.1, "memory/max_active (GiB)": 39.71, "memory/max_allocated (GiB)": 39.71, "ppl": 1.20421, "step": 247, "tokens/total": 8651968, "tokens/train_per_sec_per_gpu": 64.94, "tokens/trainable": 320639} +{"epoch": 0.96875, "grad_norm": 1.7542797327041626, "learning_rate": 6.122126399099419e-05, "loss": 0.17356741428375244, "memory/device_reserved (GiB)": 43.1, "memory/max_active (GiB)": 39.79, "memory/max_allocated (GiB)": 39.79, "ppl": 1.18954, "step": 248, "tokens/total": 8692208, "tokens/train_per_sec_per_gpu": 75.44, "tokens/trainable": 322180} +{"epoch": 0.97265625, "grad_norm": 1.6673859357833862, "learning_rate": 6.0933633207282615e-05, "loss": 0.2143019139766693, "memory/device_reserved (GiB)": 43.1, "memory/max_active (GiB)": 38.41, "memory/max_allocated (GiB)": 38.41, "ppl": 1.239, "step": 249, "tokens/total": 8728752, "tokens/train_per_sec_per_gpu": 68.52, "tokens/trainable": 323490} +{"epoch": 0.9765625, "grad_norm": 0.959932804107666, "learning_rate": 6.064575550087316e-05, "loss": 0.1842227578163147, "memory/device_reserved (GiB)": 43.1, "memory/max_active (GiB)": 39.79, "memory/max_allocated (GiB)": 39.79, "ppl": 1.20228, "step": 250, "tokens/total": 8766016, "tokens/train_per_sec_per_gpu": 67.81, "tokens/trainable": 324797} +{"epoch": 0.98046875, "grad_norm": 1.7230623960494995, "learning_rate": 6.0357642851532245e-05, "loss": 0.19168922305107117, "memory/device_reserved (GiB)": 43.1, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.21129, "step": 251, "tokens/total": 8802848, "tokens/train_per_sec_per_gpu": 65.92, "tokens/trainable": 326127} +{"epoch": 0.984375, "grad_norm": 1.4880784749984741, "learning_rate": 6.0069307248803294e-05, "loss": 0.17072950303554535, "memory/device_reserved (GiB)": 43.1, "memory/max_active (GiB)": 38.03, "memory/max_allocated (GiB)": 38.03, "ppl": 1.18617, "step": 252, "tokens/total": 8838448, "tokens/train_per_sec_per_gpu": 68.75, "tokens/trainable": 327386} +{"epoch": 0.98828125, "grad_norm": 1.253306269645691, "learning_rate": 5.9780760691507635e-05, "loss": 0.19952145218849182, "memory/device_reserved (GiB)": 43.1, "memory/max_active (GiB)": 36.44, "memory/max_allocated (GiB)": 36.44, "ppl": 1.22082, "step": 253, "tokens/total": 8870192, "tokens/train_per_sec_per_gpu": 73.99, "tokens/trainable": 328574} +{"epoch": 0.9921875, "grad_norm": 1.149484634399414, "learning_rate": 5.9492015187245334e-05, "loss": 0.17928072810173035, "memory/device_reserved (GiB)": 43.1, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.19636, "step": 254, "tokens/total": 8908032, "tokens/train_per_sec_per_gpu": 67.11, "tokens/trainable": 329798} +{"epoch": 0.99609375, "grad_norm": 2.410621404647827, "learning_rate": 5.920308275189541e-05, "loss": 0.19094882905483246, "memory/device_reserved (GiB)": 43.1, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.2104, "step": 255, "tokens/total": 8939968, "tokens/train_per_sec_per_gpu": 76.78, "tokens/trainable": 331012} +{"epoch": 1.0, "grad_norm": 1.3306477069854736, "learning_rate": 5.8913975409115874e-05, "loss": 0.1536601185798645, "memory/device_reserved (GiB)": 43.1, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.16609, "step": 256, "tokens/total": 8973712, "tokens/train_per_sec_per_gpu": 81.42, "tokens/trainable": 332368} +{"epoch": 1.00390625, "grad_norm": 1.576943039894104, "learning_rate": 5.8624705189843395e-05, "loss": 0.15253733098506927, "memory/device_reserved (GiB)": 43.1, "memory/max_active (GiB)": 39.32, "memory/max_allocated (GiB)": 39.32, "ppl": 1.16479, "step": 257, "tokens/total": 9011504, "tokens/train_per_sec_per_gpu": 66.08, "tokens/trainable": 333700} +{"epoch": 1.0078125, "grad_norm": 0.9805802702903748, "learning_rate": 5.833528413179249e-05, "loss": 0.15656788647174835, "memory/device_reserved (GiB)": 40.8, "memory/max_active (GiB)": 38.6, "memory/max_allocated (GiB)": 38.6, "ppl": 1.16949, "step": 258, "tokens/total": 9048432, "tokens/train_per_sec_per_gpu": 70.38, "tokens/trainable": 335020} +{"epoch": 1.01171875, "grad_norm": 1.465691328048706, "learning_rate": 5.80457242789548e-05, "loss": 0.14179235696792603, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 39.65, "memory/max_allocated (GiB)": 39.65, "ppl": 1.15234, "step": 259, "tokens/total": 9088816, "tokens/train_per_sec_per_gpu": 72.08, "tokens/trainable": 336484} +{"epoch": 1.015625, "grad_norm": 11.82563591003418, "learning_rate": 5.77560376810977e-05, "loss": 0.14093376696109772, "memory/device_reserved (GiB)": 41.62, "memory/max_active (GiB)": 38.37, "memory/max_allocated (GiB)": 38.37, "ppl": 1.15135, "step": 260, "tokens/total": 9122400, "tokens/train_per_sec_per_gpu": 77.55, "tokens/trainable": 337822} +{"epoch": 1.01953125, "grad_norm": 1.3647769689559937, "learning_rate": 5.7466236393263005e-05, "loss": 0.16961294412612915, "memory/device_reserved (GiB)": 41.62, "memory/max_active (GiB)": 39.12, "memory/max_allocated (GiB)": 39.12, "ppl": 1.18485, "step": 261, "tokens/total": 9157280, "tokens/train_per_sec_per_gpu": 72.29, "tokens/trainable": 339065} +{"epoch": 1.0234375, "grad_norm": 1.5422754287719727, "learning_rate": 5.717633247526522e-05, "loss": 0.17726640403270721, "memory/device_reserved (GiB)": 41.62, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.19395, "step": 262, "tokens/total": 9188464, "tokens/train_per_sec_per_gpu": 78.98, "tokens/trainable": 340330} +{"epoch": 1.02734375, "grad_norm": 1.067897081375122, "learning_rate": 5.688633799118971e-05, "loss": 0.18155762553215027, "memory/device_reserved (GiB)": 41.62, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.19908, "step": 263, "tokens/total": 9221120, "tokens/train_per_sec_per_gpu": 67.26, "tokens/trainable": 341507} +{"epoch": 1.03125, "grad_norm": 0.736051619052887, "learning_rate": 5.659626500889066e-05, "loss": 0.12694117426872253, "memory/device_reserved (GiB)": 41.62, "memory/max_active (GiB)": 37.12, "memory/max_allocated (GiB)": 37.12, "ppl": 1.13535, "step": 264, "tokens/total": 9255216, "tokens/train_per_sec_per_gpu": 78.68, "tokens/trainable": 342854} +{"epoch": 1.03515625, "grad_norm": 1.0462380647659302, "learning_rate": 5.6306125599488905e-05, "loss": 0.16011583805084229, "memory/device_reserved (GiB)": 41.62, "memory/max_active (GiB)": 36.92, "memory/max_allocated (GiB)": 36.92, "ppl": 1.17365, "step": 265, "tokens/total": 9286128, "tokens/train_per_sec_per_gpu": 84.17, "tokens/trainable": 344103} +{"epoch": 1.0390625, "grad_norm": 1.1776691675186157, "learning_rate": 5.601593183686955e-05, "loss": 0.15662945806980133, "memory/device_reserved (GiB)": 41.62, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.16956, "step": 266, "tokens/total": 9323824, "tokens/train_per_sec_per_gpu": 72.28, "tokens/trainable": 345455} +{"epoch": 1.04296875, "grad_norm": 1.0639597177505493, "learning_rate": 5.572569579717961e-05, "loss": 0.16155165433883667, "memory/device_reserved (GiB)": 41.62, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.17533, "step": 267, "tokens/total": 9359392, "tokens/train_per_sec_per_gpu": 73.62, "tokens/trainable": 346769} +{"epoch": 1.046875, "grad_norm": 2.3060474395751953, "learning_rate": 5.543542955832538e-05, "loss": 0.13796274363994598, "memory/device_reserved (GiB)": 41.62, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.14793, "step": 268, "tokens/total": 9399808, "tokens/train_per_sec_per_gpu": 67.03, "tokens/trainable": 348155} +{"epoch": 1.05078125, "grad_norm": 1.3486133813858032, "learning_rate": 5.514514519946986e-05, "loss": 0.13840383291244507, "memory/device_reserved (GiB)": 41.62, "memory/max_active (GiB)": 39.6, "memory/max_allocated (GiB)": 39.6, "ppl": 1.14844, "step": 269, "tokens/total": 9440432, "tokens/train_per_sec_per_gpu": 68.83, "tokens/trainable": 349538} +{"epoch": 1.0546875, "grad_norm": 0.9937599301338196, "learning_rate": 5.485485480053015e-05, "loss": 0.14730079472064972, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 39.63, "memory/max_allocated (GiB)": 39.63, "ppl": 1.1587, "step": 270, "tokens/total": 9479104, "tokens/train_per_sec_per_gpu": 61.23, "tokens/trainable": 350808} +{"epoch": 1.05859375, "grad_norm": 0.9991700053215027, "learning_rate": 5.4564570441674645e-05, "loss": 0.1552196443080902, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.16791, "step": 271, "tokens/total": 9512896, "tokens/train_per_sec_per_gpu": 75.97, "tokens/trainable": 352060} +{"epoch": 1.0625, "grad_norm": 1.343352198600769, "learning_rate": 5.42743042028204e-05, "loss": 0.15934403240680695, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.17274, "step": 272, "tokens/total": 9548624, "tokens/train_per_sec_per_gpu": 65.02, "tokens/trainable": 353282} +{"epoch": 1.06640625, "grad_norm": 5.221080780029297, "learning_rate": 5.3984068163130464e-05, "loss": 0.14763471484184265, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.15909, "step": 273, "tokens/total": 9582528, "tokens/train_per_sec_per_gpu": 82.26, "tokens/trainable": 354656} +{"epoch": 1.0703125, "grad_norm": 1.1805082559585571, "learning_rate": 5.369387440051111e-05, "loss": 0.16204819083213806, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 37.18, "memory/max_allocated (GiB)": 37.18, "ppl": 1.17592, "step": 274, "tokens/total": 9616320, "tokens/train_per_sec_per_gpu": 75.22, "tokens/trainable": 355921} +{"epoch": 1.07421875, "grad_norm": 2.0159573554992676, "learning_rate": 5.340373499110935e-05, "loss": 0.15053024888038635, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.16245, "step": 275, "tokens/total": 9652288, "tokens/train_per_sec_per_gpu": 73.97, "tokens/trainable": 357300} +{"epoch": 1.078125, "grad_norm": 1.3033596277236938, "learning_rate": 5.3113662008810304e-05, "loss": 0.14940345287322998, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 39.48, "memory/max_allocated (GiB)": 39.48, "ppl": 1.16114, "step": 276, "tokens/total": 9693040, "tokens/train_per_sec_per_gpu": 64.99, "tokens/trainable": 358680} +{"epoch": 1.08203125, "grad_norm": 0.9702616333961487, "learning_rate": 5.282366752473479e-05, "loss": 0.13700518012046814, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 38.32, "memory/max_allocated (GiB)": 38.32, "ppl": 1.14683, "step": 277, "tokens/total": 9730640, "tokens/train_per_sec_per_gpu": 68.43, "tokens/trainable": 360030} +{"epoch": 1.0859375, "grad_norm": 0.9429090023040771, "learning_rate": 5.2533763606737005e-05, "loss": 0.15590450167655945, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 39.52, "memory/max_allocated (GiB)": 39.52, "ppl": 1.16871, "step": 278, "tokens/total": 9770880, "tokens/train_per_sec_per_gpu": 63.22, "tokens/trainable": 361344} +{"epoch": 1.08984375, "grad_norm": 0.9228235483169556, "learning_rate": 5.224396231890232e-05, "loss": 0.12730316817760468, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.13576, "step": 279, "tokens/total": 9803040, "tokens/train_per_sec_per_gpu": 82.21, "tokens/trainable": 362707} +{"epoch": 1.09375, "grad_norm": 1.324363350868225, "learning_rate": 5.195427572104522e-05, "loss": 0.1632508635520935, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.17733, "step": 280, "tokens/total": 9837680, "tokens/train_per_sec_per_gpu": 77.05, "tokens/trainable": 364049} +{"epoch": 1.09765625, "grad_norm": 0.9879486560821533, "learning_rate": 5.166471586820751e-05, "loss": 0.16827815771102905, "memory/device_reserved (GiB)": 42.5, "memory/max_active (GiB)": 39.53, "memory/max_allocated (GiB)": 39.53, "ppl": 1.18327, "step": 281, "tokens/total": 9874928, "tokens/train_per_sec_per_gpu": 74.14, "tokens/trainable": 365396} +{"epoch": 1.1015625, "grad_norm": 1.3925862312316895, "learning_rate": 5.1375294810156615e-05, "loss": 0.15006625652313232, "memory/device_reserved (GiB)": 42.5, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.16191, "step": 282, "tokens/total": 9909712, "tokens/train_per_sec_per_gpu": 74.68, "tokens/trainable": 366667} +{"epoch": 1.10546875, "grad_norm": 1.9060965776443481, "learning_rate": 5.1086024590884144e-05, "loss": 0.1809343844652176, "memory/device_reserved (GiB)": 42.5, "memory/max_active (GiB)": 39.36, "memory/max_allocated (GiB)": 39.36, "ppl": 1.19834, "step": 283, "tokens/total": 9948112, "tokens/train_per_sec_per_gpu": 62.76, "tokens/trainable": 367891} +{"epoch": 1.109375, "grad_norm": 1.029781460762024, "learning_rate": 5.079691724810461e-05, "loss": 0.14296561479568481, "memory/device_reserved (GiB)": 42.5, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.15369, "step": 284, "tokens/total": 9984288, "tokens/train_per_sec_per_gpu": 64.9, "tokens/trainable": 369170} +{"epoch": 1.11328125, "grad_norm": 0.8020390272140503, "learning_rate": 5.0507984812754684e-05, "loss": 0.15171696245670319, "memory/device_reserved (GiB)": 42.5, "memory/max_active (GiB)": 37.39, "memory/max_allocated (GiB)": 37.39, "ppl": 1.16383, "step": 285, "tokens/total": 10018176, "tokens/train_per_sec_per_gpu": 74.15, "tokens/trainable": 370460} +{"epoch": 1.1171875, "grad_norm": 1.0652142763137817, "learning_rate": 5.021923930849237e-05, "loss": 0.15722662210464478, "memory/device_reserved (GiB)": 42.5, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 1.17026, "step": 286, "tokens/total": 10052448, "tokens/train_per_sec_per_gpu": 73.69, "tokens/trainable": 371750} +{"epoch": 1.12109375, "grad_norm": 0.9104423522949219, "learning_rate": 4.99306927511967e-05, "loss": 0.1449996531009674, "memory/device_reserved (GiB)": 42.5, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.15604, "step": 287, "tokens/total": 10086080, "tokens/train_per_sec_per_gpu": 77.57, "tokens/trainable": 373034} +{"epoch": 1.125, "grad_norm": 1.1868454217910767, "learning_rate": 4.964235714846775e-05, "loss": 0.16685542464256287, "memory/device_reserved (GiB)": 42.5, "memory/max_active (GiB)": 36.52, "memory/max_allocated (GiB)": 36.52, "ppl": 1.18158, "step": 288, "tokens/total": 10118000, "tokens/train_per_sec_per_gpu": 77.58, "tokens/trainable": 374218} +{"epoch": 1.12890625, "grad_norm": 1.9798887968063354, "learning_rate": 4.9354244499126866e-05, "loss": 0.18139421939849854, "memory/device_reserved (GiB)": 42.5, "memory/max_active (GiB)": 36.31, "memory/max_allocated (GiB)": 36.31, "ppl": 1.19889, "step": 289, "tokens/total": 10149968, "tokens/train_per_sec_per_gpu": 76.06, "tokens/trainable": 375466} +{"epoch": 1.1328125, "grad_norm": 1.6361130475997925, "learning_rate": 4.90663667927174e-05, "loss": 0.15142670273780823, "memory/device_reserved (GiB)": 42.5, "memory/max_active (GiB)": 37.91, "memory/max_allocated (GiB)": 37.91, "ppl": 1.16349, "step": 290, "tokens/total": 10182976, "tokens/train_per_sec_per_gpu": 83.25, "tokens/trainable": 376735} +{"epoch": 1.13671875, "grad_norm": 0.9715498089790344, "learning_rate": 4.877873600900581e-05, "loss": 0.1456664800643921, "memory/device_reserved (GiB)": 42.5, "memory/max_active (GiB)": 38.14, "memory/max_allocated (GiB)": 38.14, "ppl": 1.15681, "step": 291, "tokens/total": 10219248, "tokens/train_per_sec_per_gpu": 64.93, "tokens/trainable": 378028} +{"epoch": 1.140625, "grad_norm": 0.7708615660667419, "learning_rate": 4.849136411748306e-05, "loss": 0.1397351622581482, "memory/device_reserved (GiB)": 42.5, "memory/max_active (GiB)": 39.14, "memory/max_allocated (GiB)": 39.14, "ppl": 1.14997, "step": 292, "tokens/total": 10257520, "tokens/train_per_sec_per_gpu": 64.83, "tokens/trainable": 379320} +{"epoch": 1.14453125, "grad_norm": 1.2846583127975464, "learning_rate": 4.8204263076866574e-05, "loss": 0.15600821375846863, "memory/device_reserved (GiB)": 42.5, "memory/max_active (GiB)": 38.93, "memory/max_allocated (GiB)": 38.93, "ppl": 1.16884, "step": 293, "tokens/total": 10293616, "tokens/train_per_sec_per_gpu": 68.47, "tokens/trainable": 380624} +{"epoch": 1.1484375, "grad_norm": 1.3437743186950684, "learning_rate": 4.791744483460251e-05, "loss": 0.1697324961423874, "memory/device_reserved (GiB)": 42.5, "memory/max_active (GiB)": 37.87, "memory/max_allocated (GiB)": 37.87, "ppl": 1.18499, "step": 294, "tokens/total": 10328320, "tokens/train_per_sec_per_gpu": 66.53, "tokens/trainable": 381883} +{"epoch": 1.15234375, "grad_norm": 1.0332622528076172, "learning_rate": 4.7630921326368736e-05, "loss": 0.15929719805717468, "memory/device_reserved (GiB)": 42.5, "memory/max_active (GiB)": 39.39, "memory/max_allocated (GiB)": 39.39, "ppl": 1.17269, "step": 295, "tokens/total": 10367056, "tokens/train_per_sec_per_gpu": 59.47, "tokens/trainable": 383092} +{"epoch": 1.15625, "grad_norm": 1.4263190031051636, "learning_rate": 4.7344704475577916e-05, "loss": 0.15268613398075104, "memory/device_reserved (GiB)": 42.51, "memory/max_active (GiB)": 35.79, "memory/max_allocated (GiB)": 35.79, "ppl": 1.16496, "step": 296, "tokens/total": 10396880, "tokens/train_per_sec_per_gpu": 83.59, "tokens/trainable": 384367} +{"epoch": 1.16015625, "grad_norm": 1.7925387620925903, "learning_rate": 4.705880619288153e-05, "loss": 0.14950871467590332, "memory/device_reserved (GiB)": 42.51, "memory/max_active (GiB)": 38.37, "memory/max_allocated (GiB)": 38.37, "ppl": 1.16126, "step": 297, "tokens/total": 10433232, "tokens/train_per_sec_per_gpu": 69.06, "tokens/trainable": 385634} +{"epoch": 1.1640625, "grad_norm": 1.2658313512802124, "learning_rate": 4.677323837567412e-05, "loss": 0.15167289972305298, "memory/device_reserved (GiB)": 42.51, "memory/max_active (GiB)": 37.23, "memory/max_allocated (GiB)": 37.23, "ppl": 1.16378, "step": 298, "tokens/total": 10466480, "tokens/train_per_sec_per_gpu": 80.83, "tokens/trainable": 386955} +{"epoch": 1.16796875, "grad_norm": 1.4079891443252563, "learning_rate": 4.6488012907598146e-05, "loss": 0.1633996069431305, "memory/device_reserved (GiB)": 42.51, "memory/max_active (GiB)": 36.6, "memory/max_allocated (GiB)": 36.6, "ppl": 1.17751, "step": 299, "tokens/total": 10499296, "tokens/train_per_sec_per_gpu": 75.75, "tokens/trainable": 388203} +{"epoch": 1.171875, "grad_norm": 0.9238210916519165, "learning_rate": 4.620314165804964e-05, "loss": 0.1427406519651413, "memory/device_reserved (GiB)": 42.51, "memory/max_active (GiB)": 38.51, "memory/max_allocated (GiB)": 38.51, "ppl": 1.15343, "step": 300, "tokens/total": 10535424, "tokens/train_per_sec_per_gpu": 73.61, "tokens/trainable": 389565} +{"epoch": 1.17578125, "grad_norm": 1.6950793266296387, "learning_rate": 4.591863648168407e-05, "loss": 0.1477111577987671, "memory/device_reserved (GiB)": 42.51, "memory/max_active (GiB)": 39.32, "memory/max_allocated (GiB)": 39.32, "ppl": 1.15918, "step": 301, "tokens/total": 10570432, "tokens/train_per_sec_per_gpu": 74.73, "tokens/trainable": 390933} +{"epoch": 1.1796875, "grad_norm": 6.143558979034424, "learning_rate": 4.5634509217923135e-05, "loss": 0.1541772335767746, "memory/device_reserved (GiB)": 42.51, "memory/max_active (GiB)": 38.06, "memory/max_allocated (GiB)": 38.06, "ppl": 1.1667, "step": 302, "tokens/total": 10604224, "tokens/train_per_sec_per_gpu": 78.37, "tokens/trainable": 392245} +{"epoch": 1.18359375, "grad_norm": 1.0293928384780884, "learning_rate": 4.535077169046201e-05, "loss": 0.15173515677452087, "memory/device_reserved (GiB)": 42.51, "memory/max_active (GiB)": 38.1, "memory/max_allocated (GiB)": 38.1, "ppl": 1.16385, "step": 303, "tokens/total": 10641088, "tokens/train_per_sec_per_gpu": 71.48, "tokens/trainable": 393542} +{"epoch": 1.1875, "grad_norm": 1.1284106969833374, "learning_rate": 4.506743570677743e-05, "loss": 0.1631489098072052, "memory/device_reserved (GiB)": 42.51, "memory/max_active (GiB)": 36.44, "memory/max_allocated (GiB)": 36.44, "ppl": 1.17721, "step": 304, "tokens/total": 10671664, "tokens/train_per_sec_per_gpu": 87.1, "tokens/trainable": 394758} +{"epoch": 1.19140625, "grad_norm": 1.56510591506958, "learning_rate": 4.478451305763618e-05, "loss": 0.16267162561416626, "memory/device_reserved (GiB)": 42.51, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.17665, "step": 305, "tokens/total": 10705392, "tokens/train_per_sec_per_gpu": 77.7, "tokens/trainable": 396015} +{"epoch": 1.1953125, "grad_norm": 1.5993975400924683, "learning_rate": 4.450201551660454e-05, "loss": 0.16833508014678955, "memory/device_reserved (GiB)": 42.51, "memory/max_active (GiB)": 36.44, "memory/max_allocated (GiB)": 36.44, "ppl": 1.18333, "step": 306, "tokens/total": 10736656, "tokens/train_per_sec_per_gpu": 82.77, "tokens/trainable": 397299} +{"epoch": 1.19921875, "grad_norm": 1.041106104850769, "learning_rate": 4.4219954839558276e-05, "loss": 0.15658444166183472, "memory/device_reserved (GiB)": 42.51, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.16951, "step": 307, "tokens/total": 10771536, "tokens/train_per_sec_per_gpu": 71.39, "tokens/trainable": 398613} +{"epoch": 1.203125, "grad_norm": 1.0703321695327759, "learning_rate": 4.393834276419352e-05, "loss": 0.15843111276626587, "memory/device_reserved (GiB)": 42.52, "memory/max_active (GiB)": 35.75, "memory/max_allocated (GiB)": 35.75, "ppl": 1.17167, "step": 308, "tokens/total": 10801600, "tokens/train_per_sec_per_gpu": 89.23, "tokens/trainable": 399923} +{"epoch": 1.20703125, "grad_norm": 1.7235877513885498, "learning_rate": 4.36571910095382e-05, "loss": 0.1517748236656189, "memory/device_reserved (GiB)": 42.52, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.1639, "step": 309, "tokens/total": 10840208, "tokens/train_per_sec_per_gpu": 67.93, "tokens/trainable": 401178} +{"epoch": 1.2109375, "grad_norm": 0.9450157880783081, "learning_rate": 4.337651127546448e-05, "loss": 0.155717134475708, "memory/device_reserved (GiB)": 42.52, "memory/max_active (GiB)": 38.93, "memory/max_allocated (GiB)": 38.93, "ppl": 1.1685, "step": 310, "tokens/total": 10876128, "tokens/train_per_sec_per_gpu": 76.24, "tokens/trainable": 402509} +{"epoch": 1.21484375, "grad_norm": 0.8120614290237427, "learning_rate": 4.3096315242201736e-05, "loss": 0.14901192486286163, "memory/device_reserved (GiB)": 42.52, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.16069, "step": 311, "tokens/total": 10909152, "tokens/train_per_sec_per_gpu": 77.44, "tokens/trainable": 403786} +{"epoch": 1.21875, "grad_norm": 1.2998019456863403, "learning_rate": 4.2816614569850635e-05, "loss": 0.15736761689186096, "memory/device_reserved (GiB)": 42.52, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.17043, "step": 312, "tokens/total": 10945216, "tokens/train_per_sec_per_gpu": 70.73, "tokens/trainable": 405014} +{"epoch": 1.22265625, "grad_norm": 0.748226523399353, "learning_rate": 4.2537420897897864e-05, "loss": 0.14171557128429413, "memory/device_reserved (GiB)": 42.52, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 1.15225, "step": 313, "tokens/total": 10981232, "tokens/train_per_sec_per_gpu": 67.88, "tokens/trainable": 406329} +{"epoch": 1.2265625, "grad_norm": 4.064319610595703, "learning_rate": 4.225874584473174e-05, "loss": 0.15238678455352783, "memory/device_reserved (GiB)": 42.52, "memory/max_active (GiB)": 37.67, "memory/max_allocated (GiB)": 37.67, "ppl": 1.16461, "step": 314, "tokens/total": 11014096, "tokens/train_per_sec_per_gpu": 79.79, "tokens/trainable": 407595} +{"epoch": 1.23046875, "grad_norm": 1.2974200248718262, "learning_rate": 4.19806010071587e-05, "loss": 0.13222002983093262, "memory/device_reserved (GiB)": 42.52, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.14136, "step": 315, "tokens/total": 11052480, "tokens/train_per_sec_per_gpu": 69.33, "tokens/trainable": 408976} +{"epoch": 1.234375, "grad_norm": 1.4834065437316895, "learning_rate": 4.170299795992081e-05, "loss": 0.14511212706565857, "memory/device_reserved (GiB)": 42.52, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.15617, "step": 316, "tokens/total": 11086192, "tokens/train_per_sec_per_gpu": 75.37, "tokens/trainable": 410241} +{"epoch": 1.23828125, "grad_norm": 1.333399772644043, "learning_rate": 4.142594825521398e-05, "loss": 0.14612026512622833, "memory/device_reserved (GiB)": 42.52, "memory/max_active (GiB)": 39.32, "memory/max_allocated (GiB)": 39.32, "ppl": 1.15734, "step": 317, "tokens/total": 11125680, "tokens/train_per_sec_per_gpu": 62.2, "tokens/trainable": 411551} +{"epoch": 1.2421875, "grad_norm": 1.3720319271087646, "learning_rate": 4.114946342220728e-05, "loss": 0.14061278104782104, "memory/device_reserved (GiB)": 42.52, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.15098, "step": 318, "tokens/total": 11162864, "tokens/train_per_sec_per_gpu": 70.97, "tokens/trainable": 412927} +{"epoch": 1.24609375, "grad_norm": 0.8759942650794983, "learning_rate": 4.087355496656321e-05, "loss": 0.1364326924085617, "memory/device_reserved (GiB)": 42.52, "memory/max_active (GiB)": 37.81, "memory/max_allocated (GiB)": 37.81, "ppl": 1.14618, "step": 319, "tokens/total": 11196752, "tokens/train_per_sec_per_gpu": 79.88, "tokens/trainable": 414276} +{"epoch": 1.25, "grad_norm": 1.2089216709136963, "learning_rate": 4.05982343699588e-05, "loss": 0.14504674077033997, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 39.61, "memory/max_allocated (GiB)": 39.61, "ppl": 1.15609, "step": 320, "tokens/total": 11236496, "tokens/train_per_sec_per_gpu": 68.78, "tokens/trainable": 415681} +{"epoch": 1.25390625, "grad_norm": 1.4842458963394165, "learning_rate": 4.0323513089607876e-05, "loss": 0.1486780047416687, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 37.58, "memory/max_allocated (GiB)": 37.58, "ppl": 1.1603, "step": 321, "tokens/total": 11271488, "tokens/train_per_sec_per_gpu": 68.95, "tokens/trainable": 416985} +{"epoch": 1.2578125, "grad_norm": 1.986579179763794, "learning_rate": 4.004940255778431e-05, "loss": 0.1388903260231018, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 39.79, "memory/max_allocated (GiB)": 39.79, "ppl": 1.149, "step": 322, "tokens/total": 11310640, "tokens/train_per_sec_per_gpu": 71.59, "tokens/trainable": 418325} +{"epoch": 1.26171875, "grad_norm": 1.1499559879302979, "learning_rate": 3.977591418134619e-05, "loss": 0.14912177622318268, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.16081, "step": 323, "tokens/total": 11345968, "tokens/train_per_sec_per_gpu": 71.02, "tokens/trainable": 419605} +{"epoch": 1.265625, "grad_norm": 0.9713550209999084, "learning_rate": 3.95030593412612e-05, "loss": 0.14895722270011902, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 39.6, "memory/max_allocated (GiB)": 39.6, "ppl": 1.16062, "step": 324, "tokens/total": 11382064, "tokens/train_per_sec_per_gpu": 74.13, "tokens/trainable": 420895} +{"epoch": 1.26953125, "grad_norm": 1.5284006595611572, "learning_rate": 3.923084939213296e-05, "loss": 0.1499633491039276, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 35.82, "memory/max_allocated (GiB)": 35.82, "ppl": 1.16179, "step": 325, "tokens/total": 11412848, "tokens/train_per_sec_per_gpu": 80.9, "tokens/trainable": 422144} +{"epoch": 1.2734375, "grad_norm": 0.7706192135810852, "learning_rate": 3.895929566172861e-05, "loss": 0.13002881407737732, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 38.97, "memory/max_allocated (GiB)": 38.97, "ppl": 1.13886, "step": 326, "tokens/total": 11449200, "tokens/train_per_sec_per_gpu": 71.14, "tokens/trainable": 423460} +{"epoch": 1.27734375, "grad_norm": 1.4016926288604736, "learning_rate": 3.868840945050728e-05, "loss": 0.13121816515922546, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.14022, "step": 327, "tokens/total": 11488320, "tokens/train_per_sec_per_gpu": 70.35, "tokens/trainable": 424796} +{"epoch": 1.28125, "grad_norm": 0.8682640790939331, "learning_rate": 3.841820203114995e-05, "loss": 0.13831783831119537, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.14834, "step": 328, "tokens/total": 11520320, "tokens/train_per_sec_per_gpu": 81.5, "tokens/trainable": 426140} +{"epoch": 1.28515625, "grad_norm": 1.2425611019134521, "learning_rate": 3.814868464809027e-05, "loss": 0.1410774439573288, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 36.77, "memory/max_allocated (GiB)": 36.77, "ppl": 1.15151, "step": 329, "tokens/total": 11552432, "tokens/train_per_sec_per_gpu": 78.1, "tokens/trainable": 427391} +{"epoch": 1.2890625, "grad_norm": 0.9189682602882385, "learning_rate": 3.787986851704667e-05, "loss": 0.15128770470619202, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.16333, "step": 330, "tokens/total": 11586160, "tokens/train_per_sec_per_gpu": 76.89, "tokens/trainable": 428677} +{"epoch": 1.29296875, "grad_norm": 0.6964054107666016, "learning_rate": 3.7611764824555654e-05, "loss": 0.12201759219169617, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.12977, "step": 331, "tokens/total": 11617984, "tokens/train_per_sec_per_gpu": 92.45, "tokens/trainable": 430073} +{"epoch": 1.296875, "grad_norm": 0.95054692029953, "learning_rate": 3.734438472750619e-05, "loss": 0.12839652597904205, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 39.71, "memory/max_allocated (GiB)": 39.71, "ppl": 1.137, "step": 332, "tokens/total": 11658480, "tokens/train_per_sec_per_gpu": 69.02, "tokens/trainable": 431446} +{"epoch": 1.30078125, "grad_norm": 0.860646665096283, "learning_rate": 3.707773935267552e-05, "loss": 0.14963410794734955, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 38.81, "memory/max_allocated (GiB)": 38.81, "ppl": 1.16141, "step": 333, "tokens/total": 11695760, "tokens/train_per_sec_per_gpu": 73.24, "tokens/trainable": 432814} +{"epoch": 1.3046875, "grad_norm": 1.8914145231246948, "learning_rate": 3.68118397962661e-05, "loss": 0.1497849076986313, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 1.16158, "step": 334, "tokens/total": 11730928, "tokens/train_per_sec_per_gpu": 73.41, "tokens/trainable": 434065} +{"epoch": 1.30859375, "grad_norm": 5.378627777099609, "learning_rate": 3.654669712344384e-05, "loss": 0.13220052421092987, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 36.93, "memory/max_allocated (GiB)": 36.93, "ppl": 1.14134, "step": 335, "tokens/total": 11762576, "tokens/train_per_sec_per_gpu": 93.53, "tokens/trainable": 435422} +{"epoch": 1.3125, "grad_norm": 1.2670397758483887, "learning_rate": 3.628232236787763e-05, "loss": 0.12645581364631653, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.1348, "step": 336, "tokens/total": 11798512, "tokens/train_per_sec_per_gpu": 69.97, "tokens/trainable": 436814} +{"epoch": 1.31640625, "grad_norm": 0.798948347568512, "learning_rate": 3.6018726531280144e-05, "loss": 0.12936949729919434, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 39.17, "memory/max_allocated (GiB)": 39.17, "ppl": 1.13811, "step": 337, "tokens/total": 11836496, "tokens/train_per_sec_per_gpu": 71.16, "tokens/trainable": 438167} +{"epoch": 1.3203125, "grad_norm": 1.6802761554718018, "learning_rate": 3.575592058295017e-05, "loss": 0.17525756359100342, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 37.17, "memory/max_allocated (GiB)": 37.17, "ppl": 1.19155, "step": 338, "tokens/total": 11870832, "tokens/train_per_sec_per_gpu": 75.51, "tokens/trainable": 439468} +{"epoch": 1.32421875, "grad_norm": 0.8311960697174072, "learning_rate": 3.549391545931585e-05, "loss": 0.1356571912765503, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.14529, "step": 339, "tokens/total": 11905696, "tokens/train_per_sec_per_gpu": 80.48, "tokens/trainable": 440803} +{"epoch": 1.328125, "grad_norm": 5.0284037590026855, "learning_rate": 3.5232722063479914e-05, "loss": 0.146565243601799, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.15785, "step": 340, "tokens/total": 11940288, "tokens/train_per_sec_per_gpu": 77.53, "tokens/trainable": 442145} +{"epoch": 1.33203125, "grad_norm": 1.8283871412277222, "learning_rate": 3.49723512647657e-05, "loss": 0.127197727560997, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.13564, "step": 341, "tokens/total": 11979872, "tokens/train_per_sec_per_gpu": 67.85, "tokens/trainable": 443540} +{"epoch": 1.3359375, "grad_norm": 0.7507648468017578, "learning_rate": 3.471281389826491e-05, "loss": 0.1551727056503296, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 36.53, "memory/max_allocated (GiB)": 36.53, "ppl": 1.16786, "step": 342, "tokens/total": 12011104, "tokens/train_per_sec_per_gpu": 75.9, "tokens/trainable": 444704} +{"epoch": 1.33984375, "grad_norm": 1.0576008558273315, "learning_rate": 3.4454120764386764e-05, "loss": 0.13873766362667084, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.14882, "step": 343, "tokens/total": 12049840, "tokens/train_per_sec_per_gpu": 67.73, "tokens/trainable": 446015} +{"epoch": 1.34375, "grad_norm": 1.654921054840088, "learning_rate": 3.4196282628408526e-05, "loss": 0.13585756719112396, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.14552, "step": 344, "tokens/total": 12084400, "tokens/train_per_sec_per_gpu": 72.47, "tokens/trainable": 447268} +{"epoch": 1.34765625, "grad_norm": 2.180546522140503, "learning_rate": 3.3939310220027456e-05, "loss": 0.15020465850830078, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 37.74, "memory/max_allocated (GiB)": 37.74, "ppl": 1.16207, "step": 345, "tokens/total": 12118640, "tokens/train_per_sec_per_gpu": 71.98, "tokens/trainable": 448561} +{"epoch": 1.3515625, "grad_norm": 1.2057396173477173, "learning_rate": 3.3683214232914404e-05, "loss": 0.13463827967643738, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.14412, "step": 346, "tokens/total": 12154608, "tokens/train_per_sec_per_gpu": 73.06, "tokens/trainable": 449903} +{"epoch": 1.35546875, "grad_norm": 3.84713077545166, "learning_rate": 3.342800532426873e-05, "loss": 0.1476331502199173, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 38.81, "memory/max_allocated (GiB)": 38.81, "ppl": 1.15909, "step": 347, "tokens/total": 12190752, "tokens/train_per_sec_per_gpu": 71.86, "tokens/trainable": 451164} +{"epoch": 1.359375, "grad_norm": 0.879920244216919, "learning_rate": 3.317369411437484e-05, "loss": 0.1427544355392456, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.15345, "step": 348, "tokens/total": 12227136, "tokens/train_per_sec_per_gpu": 69.05, "tokens/trainable": 452453} +{"epoch": 1.36328125, "grad_norm": 0.9339218735694885, "learning_rate": 3.292029118616024e-05, "loss": 0.14480046927928925, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.15581, "step": 349, "tokens/total": 12260128, "tokens/train_per_sec_per_gpu": 66.72, "tokens/trainable": 453626} +{"epoch": 1.3671875, "grad_norm": 1.0160348415374756, "learning_rate": 3.266780708475511e-05, "loss": 0.12882252037525177, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.13749, "step": 350, "tokens/total": 12295744, "tokens/train_per_sec_per_gpu": 73.88, "tokens/trainable": 454980} +{"epoch": 1.37109375, "grad_norm": 0.862139880657196, "learning_rate": 3.241625231705354e-05, "loss": 0.12411119043827057, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 38.59, "memory/max_allocated (GiB)": 38.59, "ppl": 1.13214, "step": 351, "tokens/total": 12332176, "tokens/train_per_sec_per_gpu": 76.39, "tokens/trainable": 456360} +{"epoch": 1.375, "grad_norm": 1.032651424407959, "learning_rate": 3.216563735127618e-05, "loss": 0.13667386770248413, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 39.14, "memory/max_allocated (GiB)": 39.14, "ppl": 1.14645, "step": 352, "tokens/total": 12367408, "tokens/train_per_sec_per_gpu": 74.81, "tokens/trainable": 457746} +{"epoch": 1.37890625, "grad_norm": 0.8675640225410461, "learning_rate": 3.191597261653475e-05, "loss": 0.14045555889606476, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 39.19, "memory/max_allocated (GiB)": 39.19, "ppl": 1.1508, "step": 353, "tokens/total": 12404608, "tokens/train_per_sec_per_gpu": 72.27, "tokens/trainable": 459101} +{"epoch": 1.3828125, "grad_norm": 1.6459455490112305, "learning_rate": 3.166726850239794e-05, "loss": 0.13303250074386597, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.14229, "step": 354, "tokens/total": 12438384, "tokens/train_per_sec_per_gpu": 75.39, "tokens/trainable": 460390} +{"epoch": 1.38671875, "grad_norm": 2.140420436859131, "learning_rate": 3.141953535845912e-05, "loss": 0.151243656873703, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 36.25, "memory/max_allocated (GiB)": 36.25, "ppl": 1.16328, "step": 355, "tokens/total": 12469360, "tokens/train_per_sec_per_gpu": 76.51, "tokens/trainable": 461585} +{"epoch": 1.390625, "grad_norm": 0.9545108079910278, "learning_rate": 3.11727834939056e-05, "loss": 0.1370152086019516, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 36.44, "memory/max_allocated (GiB)": 36.44, "ppl": 1.14685, "step": 356, "tokens/total": 12501552, "tokens/train_per_sec_per_gpu": 75.15, "tokens/trainable": 462891} +{"epoch": 1.39453125, "grad_norm": 2.5585615634918213, "learning_rate": 3.092702317708967e-05, "loss": 0.15141376852989197, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 37.19, "memory/max_allocated (GiB)": 37.19, "ppl": 1.16348, "step": 357, "tokens/total": 12535568, "tokens/train_per_sec_per_gpu": 78.38, "tokens/trainable": 464168} +{"epoch": 1.3984375, "grad_norm": 0.9973581433296204, "learning_rate": 3.0682264635101276e-05, "loss": 0.13411186635494232, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 37.53, "memory/max_allocated (GiB)": 37.53, "ppl": 1.14352, "step": 358, "tokens/total": 12568400, "tokens/train_per_sec_per_gpu": 76.81, "tokens/trainable": 465400} +{"epoch": 1.40234375, "grad_norm": 0.5971529483795166, "learning_rate": 3.0438518053342407e-05, "loss": 0.12222838401794434, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 39.11, "memory/max_allocated (GiB)": 39.11, "ppl": 1.13001, "step": 359, "tokens/total": 12607840, "tokens/train_per_sec_per_gpu": 70.66, "tokens/trainable": 466824} +{"epoch": 1.40625, "grad_norm": 0.8817635178565979, "learning_rate": 3.0195793575103266e-05, "loss": 0.14226636290550232, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.15288, "step": 360, "tokens/total": 12647104, "tokens/train_per_sec_per_gpu": 62.12, "tokens/trainable": 468121} +{"epoch": 1.41015625, "grad_norm": 1.26221764087677, "learning_rate": 2.9954101301140146e-05, "loss": 0.13961166143417358, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 37.73, "memory/max_allocated (GiB)": 37.73, "ppl": 1.14983, "step": 361, "tokens/total": 12682704, "tokens/train_per_sec_per_gpu": 77.95, "tokens/trainable": 469474} +{"epoch": 1.4140625, "grad_norm": 0.6398067474365234, "learning_rate": 2.9713451289255123e-05, "loss": 0.14072147011756897, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.1511, "step": 362, "tokens/total": 12712448, "tokens/train_per_sec_per_gpu": 76.72, "tokens/trainable": 470725} +{"epoch": 1.41796875, "grad_norm": 2.2163093090057373, "learning_rate": 2.9473853553877484e-05, "loss": 0.14256557822227478, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.15323, "step": 363, "tokens/total": 12746864, "tokens/train_per_sec_per_gpu": 75.15, "tokens/trainable": 472040} +{"epoch": 1.421875, "grad_norm": 0.768130362033844, "learning_rate": 2.9235318065647e-05, "loss": 0.13219961524009705, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 39.77, "memory/max_allocated (GiB)": 39.77, "ppl": 1.14134, "step": 364, "tokens/total": 12787280, "tokens/train_per_sec_per_gpu": 64.33, "tokens/trainable": 473360} +{"epoch": 1.42578125, "grad_norm": 0.8585624098777771, "learning_rate": 2.8997854750998964e-05, "loss": 0.13578811287879944, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 39.54, "memory/max_allocated (GiB)": 39.54, "ppl": 1.14544, "step": 365, "tokens/total": 12827056, "tokens/train_per_sec_per_gpu": 73.22, "tokens/trainable": 474749} +{"epoch": 1.4296875, "grad_norm": 1.2232637405395508, "learning_rate": 2.8761473491751258e-05, "loss": 0.1507650911808014, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 36.67, "memory/max_allocated (GiB)": 36.67, "ppl": 1.16272, "step": 366, "tokens/total": 12860048, "tokens/train_per_sec_per_gpu": 76.93, "tokens/trainable": 475991} +{"epoch": 1.43359375, "grad_norm": 0.6697103977203369, "learning_rate": 2.8526184124692883e-05, "loss": 0.12469912320375443, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.13281, "step": 367, "tokens/total": 12897136, "tokens/train_per_sec_per_gpu": 63.34, "tokens/trainable": 477319} +{"epoch": 1.4375, "grad_norm": 0.9534831047058105, "learning_rate": 2.829199644117484e-05, "loss": 0.13354642689228058, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.14287, "step": 368, "tokens/total": 12931024, "tokens/train_per_sec_per_gpu": 77.16, "tokens/trainable": 478536} +{"epoch": 1.44140625, "grad_norm": 1.2457314729690552, "learning_rate": 2.8058920186702553e-05, "loss": 0.1398026943206787, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.15005, "step": 369, "tokens/total": 12969216, "tokens/train_per_sec_per_gpu": 70.75, "tokens/trainable": 479881} +{"epoch": 1.4453125, "grad_norm": 0.8582450747489929, "learning_rate": 2.782696506053033e-05, "loss": 0.14354676008224487, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.15436, "step": 370, "tokens/total": 13007840, "tokens/train_per_sec_per_gpu": 67.84, "tokens/trainable": 481133} +{"epoch": 1.44921875, "grad_norm": 1.4698406457901, "learning_rate": 2.7596140715257824e-05, "loss": 0.14398418366909027, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.15487, "step": 371, "tokens/total": 13043328, "tokens/train_per_sec_per_gpu": 72.44, "tokens/trainable": 482451} +{"epoch": 1.453125, "grad_norm": 1.2871369123458862, "learning_rate": 2.7366456756428184e-05, "loss": 0.14352518320083618, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 37.45, "memory/max_allocated (GiB)": 37.45, "ppl": 1.15434, "step": 372, "tokens/total": 13078064, "tokens/train_per_sec_per_gpu": 73.3, "tokens/trainable": 483759} +{"epoch": 1.45703125, "grad_norm": 1.699888825416565, "learning_rate": 2.7137922742128486e-05, "loss": 0.14832353591918945, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 1.15989, "step": 373, "tokens/total": 13113392, "tokens/train_per_sec_per_gpu": 75.48, "tokens/trainable": 485111} +{"epoch": 1.4609375, "grad_norm": 0.8701139688491821, "learning_rate": 2.691054818259188e-05, "loss": 0.13222262263298035, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 37.59, "memory/max_allocated (GiB)": 37.59, "ppl": 1.14136, "step": 374, "tokens/total": 13148528, "tokens/train_per_sec_per_gpu": 79.39, "tokens/trainable": 486518} +{"epoch": 1.46484375, "grad_norm": 1.104548692703247, "learning_rate": 2.6684342539801933e-05, "loss": 0.1632576882839203, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 36.59, "memory/max_allocated (GiB)": 36.59, "ppl": 1.17734, "step": 375, "tokens/total": 13180016, "tokens/train_per_sec_per_gpu": 76.98, "tokens/trainable": 487765} +{"epoch": 1.46875, "grad_norm": 2.420440912246704, "learning_rate": 2.645931522709877e-05, "loss": 0.13941605389118195, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 39.59, "memory/max_allocated (GiB)": 39.59, "ppl": 1.1496, "step": 376, "tokens/total": 13215856, "tokens/train_per_sec_per_gpu": 85.32, "tokens/trainable": 489145} +{"epoch": 1.47265625, "grad_norm": 0.805465042591095, "learning_rate": 2.6235475608787365e-05, "loss": 0.14020490646362305, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 38.86, "memory/max_allocated (GiB)": 38.86, "ppl": 1.15051, "step": 377, "tokens/total": 13252176, "tokens/train_per_sec_per_gpu": 70.47, "tokens/trainable": 490424} +{"epoch": 1.4765625, "grad_norm": 0.8162252306938171, "learning_rate": 2.6012832999747916e-05, "loss": 0.14846046268939972, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.16005, "step": 378, "tokens/total": 13283264, "tokens/train_per_sec_per_gpu": 77.82, "tokens/trainable": 491658} +{"epoch": 1.48046875, "grad_norm": 2.0198922157287598, "learning_rate": 2.579139666504821e-05, "loss": 0.1527436375617981, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.16503, "step": 379, "tokens/total": 13323584, "tokens/train_per_sec_per_gpu": 61.6, "tokens/trainable": 492913} +{"epoch": 1.484375, "grad_norm": 0.7664543390274048, "learning_rate": 2.557117581955798e-05, "loss": 0.13902641832828522, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.14915, "step": 380, "tokens/total": 13357584, "tokens/train_per_sec_per_gpu": 72.07, "tokens/trainable": 494226} +{"epoch": 1.48828125, "grad_norm": 1.0174405574798584, "learning_rate": 2.5352179627565532e-05, "loss": 0.1370052546262741, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 38.22, "memory/max_allocated (GiB)": 38.22, "ppl": 1.14683, "step": 381, "tokens/total": 13392128, "tokens/train_per_sec_per_gpu": 65.74, "tokens/trainable": 495475} +{"epoch": 1.4921875, "grad_norm": 1.1404197216033936, "learning_rate": 2.5134417202396277e-05, "loss": 0.14886537194252014, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.16052, "step": 382, "tokens/total": 13426112, "tokens/train_per_sec_per_gpu": 72.86, "tokens/trainable": 496721} +{"epoch": 1.49609375, "grad_norm": 1.0522972345352173, "learning_rate": 2.491789760603361e-05, "loss": 0.14209669828414917, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 39.23, "memory/max_allocated (GiB)": 39.23, "ppl": 1.15269, "step": 383, "tokens/total": 13462944, "tokens/train_per_sec_per_gpu": 68.12, "tokens/trainable": 498010} +{"epoch": 1.5, "grad_norm": 0.9641159772872925, "learning_rate": 2.4702629848741764e-05, "loss": 0.15189199149608612, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 37.82, "memory/max_allocated (GiB)": 37.82, "ppl": 1.16403, "step": 384, "tokens/total": 13497808, "tokens/train_per_sec_per_gpu": 75.04, "tokens/trainable": 499262} +{"epoch": 1.50390625, "grad_norm": 0.6854439973831177, "learning_rate": 2.4488622888690785e-05, "loss": 0.13895726203918457, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 37.06, "memory/max_allocated (GiB)": 37.06, "ppl": 1.14907, "step": 385, "tokens/total": 13531552, "tokens/train_per_sec_per_gpu": 72.06, "tokens/trainable": 500492} +{"epoch": 1.5078125, "grad_norm": 3.8977110385894775, "learning_rate": 2.427588563158384e-05, "loss": 0.13432885706424713, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.14377, "step": 386, "tokens/total": 13566096, "tokens/train_per_sec_per_gpu": 76.02, "tokens/trainable": 501824} +{"epoch": 1.51171875, "grad_norm": 0.6041581034660339, "learning_rate": 2.406442693028651e-05, "loss": 0.1328737735748291, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 38.63, "memory/max_allocated (GiB)": 38.63, "ppl": 1.14211, "step": 387, "tokens/total": 13598032, "tokens/train_per_sec_per_gpu": 77.01, "tokens/trainable": 503059} +{"epoch": 1.515625, "grad_norm": 0.6278145909309387, "learning_rate": 2.3854255584458547e-05, "loss": 0.1387391984462738, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.14882, "step": 388, "tokens/total": 13637040, "tokens/train_per_sec_per_gpu": 61.14, "tokens/trainable": 504271} +{"epoch": 1.51953125, "grad_norm": 1.5061019659042358, "learning_rate": 2.3645380340187508e-05, "loss": 0.12998846173286438, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 39.19, "memory/max_allocated (GiB)": 39.19, "ppl": 1.13882, "step": 389, "tokens/total": 13672848, "tokens/train_per_sec_per_gpu": 74.38, "tokens/trainable": 505593} +{"epoch": 1.5234375, "grad_norm": 1.209712028503418, "learning_rate": 2.3437809889624914e-05, "loss": 0.15889523923397064, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 36.78, "memory/max_allocated (GiB)": 36.78, "ppl": 1.17222, "step": 390, "tokens/total": 13704400, "tokens/train_per_sec_per_gpu": 84.49, "tokens/trainable": 506847} +{"epoch": 1.52734375, "grad_norm": 0.7410008907318115, "learning_rate": 2.3231552870624487e-05, "loss": 0.12956607341766357, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.13833, "step": 391, "tokens/total": 13735696, "tokens/train_per_sec_per_gpu": 86.86, "tokens/trainable": 508197} +{"epoch": 1.53125, "grad_norm": 1.5569062232971191, "learning_rate": 2.3026617866382657e-05, "loss": 0.11762148141860962, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 36.73, "memory/max_allocated (GiB)": 36.73, "ppl": 1.12482, "step": 392, "tokens/total": 13768864, "tokens/train_per_sec_per_gpu": 80.74, "tokens/trainable": 509548} +{"epoch": 1.53515625, "grad_norm": 1.1000535488128662, "learning_rate": 2.2823013405081507e-05, "loss": 0.12269846349954605, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.13054, "step": 393, "tokens/total": 13807296, "tokens/train_per_sec_per_gpu": 69.16, "tokens/trainable": 510899} +{"epoch": 1.5390625, "grad_norm": 0.6323750019073486, "learning_rate": 2.2620747959533722e-05, "loss": 0.1160852462053299, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.12309, "step": 394, "tokens/total": 13840944, "tokens/train_per_sec_per_gpu": 83.18, "tokens/trainable": 512309} +{"epoch": 1.54296875, "grad_norm": 1.9787482023239136, "learning_rate": 2.2419829946830123e-05, "loss": 0.13394616544246674, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 38.05, "memory/max_allocated (GiB)": 38.05, "ppl": 1.14333, "step": 395, "tokens/total": 13877136, "tokens/train_per_sec_per_gpu": 72.55, "tokens/trainable": 513588} +{"epoch": 1.546875, "grad_norm": 0.8094363212585449, "learning_rate": 2.2220267727989325e-05, "loss": 0.12477448582649231, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 38.0, "memory/max_allocated (GiB)": 38.0, "ppl": 1.13289, "step": 396, "tokens/total": 13912048, "tokens/train_per_sec_per_gpu": 75.3, "tokens/trainable": 514935} +{"epoch": 1.55078125, "grad_norm": 1.2725673913955688, "learning_rate": 2.202206960760984e-05, "loss": 0.12595421075820923, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.13423, "step": 397, "tokens/total": 13950576, "tokens/train_per_sec_per_gpu": 67.67, "tokens/trainable": 516322} +{"epoch": 1.5546875, "grad_norm": 2.0371081829071045, "learning_rate": 2.182524383352446e-05, "loss": 0.14465151727199554, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 38.02, "memory/max_allocated (GiB)": 38.02, "ppl": 1.15564, "step": 398, "tokens/total": 13986928, "tokens/train_per_sec_per_gpu": 66.84, "tokens/trainable": 517569} +{"epoch": 1.55859375, "grad_norm": 1.0755072832107544, "learning_rate": 2.1629798596457056e-05, "loss": 0.14420878887176514, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 36.53, "memory/max_allocated (GiB)": 36.53, "ppl": 1.15513, "step": 399, "tokens/total": 14019376, "tokens/train_per_sec_per_gpu": 80.89, "tokens/trainable": 518848} +{"epoch": 1.5625, "grad_norm": 0.8302785158157349, "learning_rate": 2.1435742029681725e-05, "loss": 0.14259304106235504, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 36.47, "memory/max_allocated (GiB)": 36.47, "ppl": 1.15326, "step": 400, "tokens/total": 14050256, "tokens/train_per_sec_per_gpu": 84.21, "tokens/trainable": 520092} +{"epoch": 1.56640625, "grad_norm": 3.2710115909576416, "learning_rate": 2.124308220868431e-05, "loss": 0.12520447373390198, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.13338, "step": 401, "tokens/total": 14086496, "tokens/train_per_sec_per_gpu": 71.73, "tokens/trainable": 521458} +{"epoch": 1.5703125, "grad_norm": 0.9422599673271179, "learning_rate": 2.105182715082638e-05, "loss": 0.13318665325641632, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.14246, "step": 402, "tokens/total": 14120608, "tokens/train_per_sec_per_gpu": 72.07, "tokens/trainable": 522786} +{"epoch": 1.57421875, "grad_norm": 2.349541187286377, "learning_rate": 2.0861984815011552e-05, "loss": 0.1477706879377365, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 38.05, "memory/max_allocated (GiB)": 38.05, "ppl": 1.15925, "step": 403, "tokens/total": 14156064, "tokens/train_per_sec_per_gpu": 69.22, "tokens/trainable": 524020} +{"epoch": 1.578125, "grad_norm": 4.664621829986572, "learning_rate": 2.0673563101354323e-05, "loss": 0.13935288786888123, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.14953, "step": 404, "tokens/total": 14191488, "tokens/train_per_sec_per_gpu": 65.45, "tokens/trainable": 525255} +{"epoch": 1.58203125, "grad_norm": 0.6895223259925842, "learning_rate": 2.0486569850851317e-05, "loss": 0.12830108404159546, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 38.46, "memory/max_allocated (GiB)": 38.46, "ppl": 1.1369, "step": 405, "tokens/total": 14226672, "tokens/train_per_sec_per_gpu": 73.89, "tokens/trainable": 526562} +{"epoch": 1.5859375, "grad_norm": 0.8065723180770874, "learning_rate": 2.0301012845054956e-05, "loss": 0.1368769258260727, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.14669, "step": 406, "tokens/total": 14265696, "tokens/train_per_sec_per_gpu": 64.28, "tokens/trainable": 527855} +{"epoch": 1.58984375, "grad_norm": 2.1021199226379395, "learning_rate": 2.011689980574966e-05, "loss": 0.12946748733520508, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 38.22, "memory/max_allocated (GiB)": 38.22, "ppl": 1.13822, "step": 407, "tokens/total": 14302272, "tokens/train_per_sec_per_gpu": 68.54, "tokens/trainable": 529111} +{"epoch": 1.59375, "grad_norm": 1.1888452768325806, "learning_rate": 1.993423839463052e-05, "loss": 0.13958556950092316, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.1498, "step": 408, "tokens/total": 14335968, "tokens/train_per_sec_per_gpu": 87.8, "tokens/trainable": 530415} +{"epoch": 1.59765625, "grad_norm": 0.8283571004867554, "learning_rate": 1.975303621298445e-05, "loss": 0.12749473750591278, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.13598, "step": 409, "tokens/total": 14373056, "tokens/train_per_sec_per_gpu": 75.05, "tokens/trainable": 531726} +{"epoch": 1.6015625, "grad_norm": 1.1182010173797607, "learning_rate": 1.957330080137385e-05, "loss": 0.1445496678352356, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 36.28, "memory/max_allocated (GiB)": 36.28, "ppl": 1.15552, "step": 410, "tokens/total": 14403952, "tokens/train_per_sec_per_gpu": 77.14, "tokens/trainable": 532970} +{"epoch": 1.60546875, "grad_norm": 5.2456254959106445, "learning_rate": 1.9395039639322864e-05, "loss": 0.1310400366783142, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.14001, "step": 411, "tokens/total": 14440416, "tokens/train_per_sec_per_gpu": 72.46, "tokens/trainable": 534271} +{"epoch": 1.609375, "grad_norm": 2.7997982501983643, "learning_rate": 1.9218260145006073e-05, "loss": 0.1261507272720337, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 37.97, "memory/max_allocated (GiB)": 37.97, "ppl": 1.13445, "step": 412, "tokens/total": 14476928, "tokens/train_per_sec_per_gpu": 75.86, "tokens/trainable": 535673} +{"epoch": 1.61328125, "grad_norm": 0.8960428833961487, "learning_rate": 1.904296967493982e-05, "loss": 0.11906356364488602, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.12644, "step": 413, "tokens/total": 14509632, "tokens/train_per_sec_per_gpu": 85.34, "tokens/trainable": 537011} +{"epoch": 1.6171875, "grad_norm": 0.5311249494552612, "learning_rate": 1.8869175523676064e-05, "loss": 0.1168014407157898, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.1239, "step": 414, "tokens/total": 14546704, "tokens/train_per_sec_per_gpu": 75.08, "tokens/trainable": 538389} +{"epoch": 1.62109375, "grad_norm": 1.129207968711853, "learning_rate": 1.869688492349885e-05, "loss": 0.13518714904785156, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.14475, "step": 415, "tokens/total": 14580464, "tokens/train_per_sec_per_gpu": 80.28, "tokens/trainable": 539704} +{"epoch": 1.625, "grad_norm": 0.7835113406181335, "learning_rate": 1.85261050441233e-05, "loss": 0.13933932781219482, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 38.02, "memory/max_allocated (GiB)": 38.02, "ppl": 1.14951, "step": 416, "tokens/total": 14616832, "tokens/train_per_sec_per_gpu": 72.71, "tokens/trainable": 541016} +{"epoch": 1.62890625, "grad_norm": 1.1918972730636597, "learning_rate": 1.8356842992397304e-05, "loss": 0.15047213435173035, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.16238, "step": 417, "tokens/total": 14650992, "tokens/train_per_sec_per_gpu": 75.45, "tokens/trainable": 542293} +{"epoch": 1.6328125, "grad_norm": 0.6115862131118774, "learning_rate": 1.8189105812005714e-05, "loss": 0.13926805555820465, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 1.14943, "step": 418, "tokens/total": 14687984, "tokens/train_per_sec_per_gpu": 65.64, "tokens/trainable": 543509} +{"epoch": 1.63671875, "grad_norm": 1.8633171319961548, "learning_rate": 1.802290048317732e-05, "loss": 0.12781652808189392, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 37.96, "memory/max_allocated (GiB)": 37.96, "ppl": 1.13634, "step": 419, "tokens/total": 14721792, "tokens/train_per_sec_per_gpu": 77.01, "tokens/trainable": 544799} +{"epoch": 1.640625, "grad_norm": 4.792210102081299, "learning_rate": 1.785823392239424e-05, "loss": 0.12301170825958252, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.1309, "step": 420, "tokens/total": 14755344, "tokens/train_per_sec_per_gpu": 74.19, "tokens/trainable": 546053} +{"epoch": 1.64453125, "grad_norm": 0.8730887770652771, "learning_rate": 1.7695112982104225e-05, "loss": 0.13456520438194275, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 39.8, "memory/max_allocated (GiB)": 39.8, "ppl": 1.14404, "step": 421, "tokens/total": 14790768, "tokens/train_per_sec_per_gpu": 73.39, "tokens/trainable": 547369} +{"epoch": 1.6484375, "grad_norm": 1.6863505840301514, "learning_rate": 1.7533544450435433e-05, "loss": 0.14039450883865356, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 35.81, "memory/max_allocated (GiB)": 35.81, "ppl": 1.15073, "step": 422, "tokens/total": 14820480, "tokens/train_per_sec_per_gpu": 89.23, "tokens/trainable": 548641} +{"epoch": 1.65234375, "grad_norm": 0.7412883639335632, "learning_rate": 1.7373535050913946e-05, "loss": 0.12988965213298798, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.1387, "step": 423, "tokens/total": 14857216, "tokens/train_per_sec_per_gpu": 67.24, "tokens/trainable": 549969} +{"epoch": 1.65625, "grad_norm": 0.7605318427085876, "learning_rate": 1.721509144218405e-05, "loss": 0.13531926274299622, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 35.88, "memory/max_allocated (GiB)": 35.88, "ppl": 1.1449, "step": 424, "tokens/total": 14887760, "tokens/train_per_sec_per_gpu": 86.84, "tokens/trainable": 551233} +{"epoch": 1.66015625, "grad_norm": 1.7583906650543213, "learning_rate": 1.705822021773101e-05, "loss": 0.13275037705898285, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.14196, "step": 425, "tokens/total": 14922496, "tokens/train_per_sec_per_gpu": 78.15, "tokens/trainable": 552567} +{"epoch": 1.6640625, "grad_norm": 0.8755410313606262, "learning_rate": 1.69029279056068e-05, "loss": 0.13040882349014282, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 39.09, "memory/max_allocated (GiB)": 39.09, "ppl": 1.13929, "step": 426, "tokens/total": 14960208, "tokens/train_per_sec_per_gpu": 68.74, "tokens/trainable": 553868} +{"epoch": 1.66796875, "grad_norm": 1.4149768352508545, "learning_rate": 1.6749220968158415e-05, "loss": 0.11899270117282867, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.12636, "step": 427, "tokens/total": 14995536, "tokens/train_per_sec_per_gpu": 81.35, "tokens/trainable": 555256} +{"epoch": 1.671875, "grad_norm": 1.6336917877197266, "learning_rate": 1.659710580175893e-05, "loss": 0.13866987824440002, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 35.98, "memory/max_allocated (GiB)": 35.98, "ppl": 1.14874, "step": 428, "tokens/total": 15026656, "tokens/train_per_sec_per_gpu": 82.83, "tokens/trainable": 556550} +{"epoch": 1.67578125, "grad_norm": 2.1511569023132324, "learning_rate": 1.644658873654133e-05, "loss": 0.12798497080802917, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.13654, "step": 429, "tokens/total": 15059344, "tokens/train_per_sec_per_gpu": 78.0, "tokens/trainable": 557810} +{"epoch": 1.6796875, "grad_norm": 0.7824645042419434, "learning_rate": 1.629767603613508e-05, "loss": 0.146342933177948, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 36.28, "memory/max_allocated (GiB)": 36.28, "ppl": 1.15759, "step": 430, "tokens/total": 15090480, "tokens/train_per_sec_per_gpu": 76.07, "tokens/trainable": 558984} +{"epoch": 1.68359375, "grad_norm": 1.6354860067367554, "learning_rate": 1.615037389740547e-05, "loss": 0.13534462451934814, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.14493, "step": 431, "tokens/total": 15125808, "tokens/train_per_sec_per_gpu": 69.61, "tokens/trainable": 560246} +{"epoch": 1.6875, "grad_norm": 1.0294926166534424, "learning_rate": 1.600468845019576e-05, "loss": 0.12892048060894012, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 38.97, "memory/max_allocated (GiB)": 38.97, "ppl": 1.1376, "step": 432, "tokens/total": 15164592, "tokens/train_per_sec_per_gpu": 62.57, "tokens/trainable": 561552} +{"epoch": 1.69140625, "grad_norm": 0.6382455825805664, "learning_rate": 1.5860625757072092e-05, "loss": 0.13514019548892975, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.1447, "step": 433, "tokens/total": 15199856, "tokens/train_per_sec_per_gpu": 70.92, "tokens/trainable": 562815} +{"epoch": 1.6953125, "grad_norm": 0.6882888674736023, "learning_rate": 1.571819181307116e-05, "loss": 0.13277962803840637, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 38.51, "memory/max_allocated (GiB)": 38.51, "ppl": 1.142, "step": 434, "tokens/total": 15237168, "tokens/train_per_sec_per_gpu": 74.72, "tokens/trainable": 564155} +{"epoch": 1.69921875, "grad_norm": 0.7698009014129639, "learning_rate": 1.557739254545075e-05, "loss": 0.1312694251537323, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 36.65, "memory/max_allocated (GiB)": 36.65, "ppl": 1.14027, "step": 435, "tokens/total": 15268688, "tokens/train_per_sec_per_gpu": 86.23, "tokens/trainable": 565477} +{"epoch": 1.703125, "grad_norm": 0.7587814927101135, "learning_rate": 1.543823381344311e-05, "loss": 0.1272841840982437, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 38.1, "memory/max_allocated (GiB)": 38.1, "ppl": 1.13574, "step": 436, "tokens/total": 15301648, "tokens/train_per_sec_per_gpu": 84.43, "tokens/trainable": 566796} +{"epoch": 1.70703125, "grad_norm": 0.9003485441207886, "learning_rate": 1.5300721408011114e-05, "loss": 0.12866009771823883, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.1373, "step": 437, "tokens/total": 15337376, "tokens/train_per_sec_per_gpu": 73.2, "tokens/trainable": 568065} +{"epoch": 1.7109375, "grad_norm": 0.9267295598983765, "learning_rate": 1.5164861051607254e-05, "loss": 0.14666211605072021, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 37.06, "memory/max_allocated (GiB)": 37.06, "ppl": 1.15796, "step": 438, "tokens/total": 15370704, "tokens/train_per_sec_per_gpu": 66.25, "tokens/trainable": 569219} +{"epoch": 1.71484375, "grad_norm": 0.6796687245368958, "learning_rate": 1.5030658397935521e-05, "loss": 0.13132993876934052, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 38.34, "memory/max_allocated (GiB)": 38.34, "ppl": 1.14034, "step": 439, "tokens/total": 15406448, "tokens/train_per_sec_per_gpu": 70.3, "tokens/trainable": 570526} +{"epoch": 1.71875, "grad_norm": 1.2781860828399658, "learning_rate": 1.4898119031716104e-05, "loss": 0.12308937311172485, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 38.1, "memory/max_allocated (GiB)": 38.1, "ppl": 1.13099, "step": 440, "tokens/total": 15439776, "tokens/train_per_sec_per_gpu": 81.86, "tokens/trainable": 571858} +{"epoch": 1.72265625, "grad_norm": 0.797924816608429, "learning_rate": 1.476724846845306e-05, "loss": 0.1345667839050293, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.14404, "step": 441, "tokens/total": 15478192, "tokens/train_per_sec_per_gpu": 67.15, "tokens/trainable": 573205} +{"epoch": 1.7265625, "grad_norm": 0.7180747985839844, "learning_rate": 1.463805215420471e-05, "loss": 0.1300339698791504, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 39.8, "memory/max_allocated (GiB)": 39.8, "ppl": 1.13887, "step": 442, "tokens/total": 15514080, "tokens/train_per_sec_per_gpu": 73.14, "tokens/trainable": 574520} +{"epoch": 1.73046875, "grad_norm": 2.2777018547058105, "learning_rate": 1.451053546535705e-05, "loss": 0.13081838190555573, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 39.14, "memory/max_allocated (GiB)": 39.14, "ppl": 1.13976, "step": 443, "tokens/total": 15551424, "tokens/train_per_sec_per_gpu": 69.23, "tokens/trainable": 575818} +{"epoch": 1.734375, "grad_norm": 0.6738031506538391, "learning_rate": 1.438470370840001e-05, "loss": 0.1233069971203804, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 38.14, "memory/max_allocated (GiB)": 38.14, "ppl": 1.13123, "step": 444, "tokens/total": 15588192, "tokens/train_per_sec_per_gpu": 69.41, "tokens/trainable": 577142} +{"epoch": 1.73828125, "grad_norm": 0.9285451769828796, "learning_rate": 1.4260562119706606e-05, "loss": 0.12813632190227509, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.13671, "step": 445, "tokens/total": 15620928, "tokens/train_per_sec_per_gpu": 82.26, "tokens/trainable": 578432} +{"epoch": 1.7421875, "grad_norm": 0.5774310231208801, "learning_rate": 1.413811586531508e-05, "loss": 0.12390166521072388, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 38.1, "memory/max_allocated (GiB)": 38.1, "ppl": 1.1319, "step": 446, "tokens/total": 15654592, "tokens/train_per_sec_per_gpu": 82.97, "tokens/trainable": 579755} +{"epoch": 1.74609375, "grad_norm": 0.6013528108596802, "learning_rate": 1.4017370040713884e-05, "loss": 0.13964661955833435, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 38.39, "memory/max_allocated (GiB)": 38.39, "ppl": 1.14987, "step": 447, "tokens/total": 15692032, "tokens/train_per_sec_per_gpu": 66.58, "tokens/trainable": 581018} +{"epoch": 1.75, "grad_norm": 0.7415131330490112, "learning_rate": 1.3898329670629645e-05, "loss": 0.1340728998184204, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 39.87, "memory/max_allocated (GiB)": 39.87, "ppl": 1.14348, "step": 448, "tokens/total": 15732992, "tokens/train_per_sec_per_gpu": 62.75, "tokens/trainable": 582346} +{"epoch": 1.75390625, "grad_norm": 0.9013209342956543, "learning_rate": 1.3780999708818058e-05, "loss": 0.13190704584121704, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 36.49, "memory/max_allocated (GiB)": 36.49, "ppl": 1.141, "step": 449, "tokens/total": 15764480, "tokens/train_per_sec_per_gpu": 80.91, "tokens/trainable": 583669} +{"epoch": 1.7578125, "grad_norm": 0.9570688009262085, "learning_rate": 1.3665385037857758e-05, "loss": 0.13254906237125397, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.14174, "step": 450, "tokens/total": 15797904, "tokens/train_per_sec_per_gpu": 76.61, "tokens/trainable": 584961} +{"epoch": 1.76171875, "grad_norm": 1.7841612100601196, "learning_rate": 1.3551490468947126e-05, "loss": 0.1303216964006424, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 36.59, "memory/max_allocated (GiB)": 36.59, "ppl": 1.13919, "step": 451, "tokens/total": 15830144, "tokens/train_per_sec_per_gpu": 84.08, "tokens/trainable": 586245} +{"epoch": 1.765625, "grad_norm": 1.48146653175354, "learning_rate": 1.3439320741704075e-05, "loss": 0.14764171838760376, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 39.19, "memory/max_allocated (GiB)": 39.19, "ppl": 1.1591, "step": 452, "tokens/total": 15865392, "tokens/train_per_sec_per_gpu": 76.78, "tokens/trainable": 587543} +{"epoch": 1.76953125, "grad_norm": 0.8701683878898621, "learning_rate": 1.3328880523968808e-05, "loss": 0.1333778202533722, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 37.15, "memory/max_allocated (GiB)": 37.15, "ppl": 1.14268, "step": 453, "tokens/total": 15899104, "tokens/train_per_sec_per_gpu": 76.7, "tokens/trainable": 588806} +{"epoch": 1.7734375, "grad_norm": 1.6314572095870972, "learning_rate": 1.3220174411609587e-05, "loss": 0.129464790225029, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 37.44, "memory/max_allocated (GiB)": 37.44, "ppl": 1.13822, "step": 454, "tokens/total": 15933136, "tokens/train_per_sec_per_gpu": 79.42, "tokens/trainable": 590156} +{"epoch": 1.77734375, "grad_norm": 1.1694984436035156, "learning_rate": 1.3113206928331471e-05, "loss": 0.12718331813812256, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 35.29, "memory/max_allocated (GiB)": 35.29, "ppl": 1.13563, "step": 455, "tokens/total": 15961840, "tokens/train_per_sec_per_gpu": 88.7, "tokens/trainable": 591442} +{"epoch": 1.78125, "grad_norm": 0.9642980694770813, "learning_rate": 1.300798252548806e-05, "loss": 0.12749673426151276, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 37.82, "memory/max_allocated (GiB)": 37.82, "ppl": 1.13598, "step": 456, "tokens/total": 15996768, "tokens/train_per_sec_per_gpu": 74.63, "tokens/trainable": 592726} +{"epoch": 1.78515625, "grad_norm": 0.8629758954048157, "learning_rate": 1.2904505581896265e-05, "loss": 0.134237140417099, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.14366, "step": 457, "tokens/total": 16036384, "tokens/train_per_sec_per_gpu": 63.57, "tokens/trainable": 594075} +{"epoch": 1.7890625, "grad_norm": 0.6550182104110718, "learning_rate": 1.2802780403654082e-05, "loss": 0.13312549889087677, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.14239, "step": 458, "tokens/total": 16070224, "tokens/train_per_sec_per_gpu": 78.91, "tokens/trainable": 595323} +{"epoch": 1.79296875, "grad_norm": 0.7291343212127686, "learning_rate": 1.2702811223961408e-05, "loss": 0.12211193889379501, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.12988, "step": 459, "tokens/total": 16104352, "tokens/train_per_sec_per_gpu": 80.44, "tokens/trainable": 596685} +{"epoch": 1.796875, "grad_norm": 0.7526296973228455, "learning_rate": 1.2604602202943861e-05, "loss": 0.129766047000885, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 37.06, "memory/max_allocated (GiB)": 37.06, "ppl": 1.13856, "step": 460, "tokens/total": 16135664, "tokens/train_per_sec_per_gpu": 86.64, "tokens/trainable": 598028} +{"epoch": 1.80078125, "grad_norm": 0.6192947030067444, "learning_rate": 1.2508157427479686e-05, "loss": 0.1305435448884964, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 37.37, "memory/max_allocated (GiB)": 37.37, "ppl": 1.13945, "step": 461, "tokens/total": 16168640, "tokens/train_per_sec_per_gpu": 76.85, "tokens/trainable": 599268} +{"epoch": 1.8046875, "grad_norm": 0.6416631937026978, "learning_rate": 1.2413480911029655e-05, "loss": 0.13286873698234558, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 36.03, "memory/max_allocated (GiB)": 36.03, "ppl": 1.1421, "step": 462, "tokens/total": 16199776, "tokens/train_per_sec_per_gpu": 75.26, "tokens/trainable": 600490} +{"epoch": 1.80859375, "grad_norm": 1.1274348497390747, "learning_rate": 1.2320576593470082e-05, "loss": 0.14098601043224335, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.15141, "step": 463, "tokens/total": 16234672, "tokens/train_per_sec_per_gpu": 67.86, "tokens/trainable": 601730} +{"epoch": 1.8125, "grad_norm": 0.7506183385848999, "learning_rate": 1.2229448340928828e-05, "loss": 0.12499605119228363, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 39.3, "memory/max_allocated (GiB)": 39.3, "ppl": 1.13314, "step": 464, "tokens/total": 16274352, "tokens/train_per_sec_per_gpu": 63.36, "tokens/trainable": 603104} +{"epoch": 1.81640625, "grad_norm": 0.6281558275222778, "learning_rate": 1.2140099945624458e-05, "loss": 0.12782888114452362, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.13636, "step": 465, "tokens/total": 16313600, "tokens/train_per_sec_per_gpu": 63.68, "tokens/trainable": 604412} +{"epoch": 1.8203125, "grad_norm": 0.9017807245254517, "learning_rate": 1.205253512570841e-05, "loss": 0.1417621225118637, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 37.95, "memory/max_allocated (GiB)": 37.95, "ppl": 1.1523, "step": 466, "tokens/total": 16346672, "tokens/train_per_sec_per_gpu": 74.98, "tokens/trainable": 605643} +{"epoch": 1.82421875, "grad_norm": 0.8804510235786438, "learning_rate": 1.1966757525110255e-05, "loss": 0.1381489485502243, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 35.48, "memory/max_allocated (GiB)": 35.48, "ppl": 1.14815, "step": 467, "tokens/total": 16376336, "tokens/train_per_sec_per_gpu": 86.93, "tokens/trainable": 606898} +{"epoch": 1.828125, "grad_norm": 2.2854087352752686, "learning_rate": 1.1882770713386095e-05, "loss": 0.12200267612934113, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.12976, "step": 468, "tokens/total": 16413248, "tokens/train_per_sec_per_gpu": 72.12, "tokens/trainable": 608236} +{"epoch": 1.83203125, "grad_norm": 0.6520041823387146, "learning_rate": 1.180057818556998e-05, "loss": 0.12304021418094635, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.13093, "step": 469, "tokens/total": 16449616, "tokens/train_per_sec_per_gpu": 71.48, "tokens/trainable": 609607} +{"epoch": 1.8359375, "grad_norm": 0.8366628289222717, "learning_rate": 1.1720183362028494e-05, "loss": 0.1375478357076645, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 38.63, "memory/max_allocated (GiB)": 38.63, "ppl": 1.14746, "step": 470, "tokens/total": 16486304, "tokens/train_per_sec_per_gpu": 67.15, "tokens/trainable": 610865} +{"epoch": 1.83984375, "grad_norm": 0.651789128780365, "learning_rate": 1.1641589588318387e-05, "loss": 0.1305769681930542, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.13949, "step": 471, "tokens/total": 16524352, "tokens/train_per_sec_per_gpu": 65.2, "tokens/trainable": 612118} +{"epoch": 1.84375, "grad_norm": 0.6055710315704346, "learning_rate": 1.1564800135047418e-05, "loss": 0.13175199925899506, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.14083, "step": 472, "tokens/total": 16558080, "tokens/train_per_sec_per_gpu": 75.12, "tokens/trainable": 613416} +{"epoch": 1.84765625, "grad_norm": 0.6056576371192932, "learning_rate": 1.148981819773816e-05, "loss": 0.12975263595581055, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.13855, "step": 473, "tokens/total": 16591232, "tokens/train_per_sec_per_gpu": 71.42, "tokens/trainable": 614658} +{"epoch": 1.8515625, "grad_norm": 1.0060564279556274, "learning_rate": 1.1416646896695086e-05, "loss": 0.1311701089143753, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.14016, "step": 474, "tokens/total": 16625792, "tokens/train_per_sec_per_gpu": 76.22, "tokens/trainable": 615964} +{"epoch": 1.85546875, "grad_norm": 0.5740203261375427, "learning_rate": 1.1345289276874717e-05, "loss": 0.12234003096818924, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 38.69, "memory/max_allocated (GiB)": 38.69, "ppl": 1.13014, "step": 475, "tokens/total": 16663392, "tokens/train_per_sec_per_gpu": 72.62, "tokens/trainable": 617339} +{"epoch": 1.859375, "grad_norm": 1.4446768760681152, "learning_rate": 1.1275748307758873e-05, "loss": 0.1253901571035385, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.13359, "step": 476, "tokens/total": 16699280, "tokens/train_per_sec_per_gpu": 71.82, "tokens/trainable": 618632} +{"epoch": 1.86328125, "grad_norm": 0.6494866013526917, "learning_rate": 1.1208026883231147e-05, "loss": 0.14443959295749664, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 36.47, "memory/max_allocated (GiB)": 36.47, "ppl": 1.15539, "step": 477, "tokens/total": 16730688, "tokens/train_per_sec_per_gpu": 80.16, "tokens/trainable": 619800} +{"epoch": 1.8671875, "grad_norm": 1.45099675655365, "learning_rate": 1.1142127821456433e-05, "loss": 0.14786435663700104, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 37.19, "memory/max_allocated (GiB)": 37.19, "ppl": 1.15936, "step": 478, "tokens/total": 16764416, "tokens/train_per_sec_per_gpu": 74.65, "tokens/trainable": 621053} +{"epoch": 1.87109375, "grad_norm": 1.1359012126922607, "learning_rate": 1.1078053864763674e-05, "loss": 0.13693304359912872, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.14675, "step": 479, "tokens/total": 16800592, "tokens/train_per_sec_per_gpu": 75.99, "tokens/trainable": 622367} +{"epoch": 1.875, "grad_norm": 0.5259917974472046, "learning_rate": 1.1015807679531756e-05, "loss": 0.1253066509962082, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 37.58, "memory/max_allocated (GiB)": 37.58, "ppl": 1.1335, "step": 480, "tokens/total": 16832752, "tokens/train_per_sec_per_gpu": 83.35, "tokens/trainable": 623694} +{"epoch": 1.87890625, "grad_norm": 0.9135778546333313, "learning_rate": 1.0955391856078528e-05, "loss": 0.12863120436668396, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.13727, "step": 481, "tokens/total": 16869072, "tokens/train_per_sec_per_gpu": 77.21, "tokens/trainable": 625028} +{"epoch": 1.8828125, "grad_norm": 0.7772088050842285, "learning_rate": 1.0896808908553007e-05, "loss": 0.14073634147644043, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 1.15112, "step": 482, "tokens/total": 16904304, "tokens/train_per_sec_per_gpu": 66.88, "tokens/trainable": 626254} +{"epoch": 1.88671875, "grad_norm": 4.7978315353393555, "learning_rate": 1.0840061274830763e-05, "loss": 0.12978070974349976, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 36.22, "memory/max_allocated (GiB)": 36.22, "ppl": 1.13858, "step": 483, "tokens/total": 16935872, "tokens/train_per_sec_per_gpu": 74.68, "tokens/trainable": 627504} +{"epoch": 1.890625, "grad_norm": 0.6879202723503113, "learning_rate": 1.0785151316412473e-05, "loss": 0.12558221817016602, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.13381, "step": 484, "tokens/total": 16972288, "tokens/train_per_sec_per_gpu": 66.5, "tokens/trainable": 628823} +{"epoch": 1.89453125, "grad_norm": 15.635110855102539, "learning_rate": 1.0732081318325639e-05, "loss": 0.13060516119003296, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.13952, "step": 485, "tokens/total": 17011456, "tokens/train_per_sec_per_gpu": 69.16, "tokens/trainable": 630175} +{"epoch": 1.8984375, "grad_norm": 1.5639792680740356, "learning_rate": 1.0680853489029501e-05, "loss": 0.13341139256954193, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 38.8, "memory/max_allocated (GiB)": 38.8, "ppl": 1.14272, "step": 486, "tokens/total": 17048640, "tokens/train_per_sec_per_gpu": 67.09, "tokens/trainable": 631451} +{"epoch": 1.90234375, "grad_norm": 0.8878461122512817, "learning_rate": 1.0631469960323152e-05, "loss": 0.13972169160842896, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.14995, "step": 487, "tokens/total": 17088448, "tokens/train_per_sec_per_gpu": 59.66, "tokens/trainable": 632677} +{"epoch": 1.90625, "grad_norm": 0.5224047899246216, "learning_rate": 1.0583932787256783e-05, "loss": 0.13314571976661682, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 37.56, "memory/max_allocated (GiB)": 37.56, "ppl": 1.14242, "step": 488, "tokens/total": 17122208, "tokens/train_per_sec_per_gpu": 75.58, "tokens/trainable": 633906} +{"epoch": 1.91015625, "grad_norm": 0.7112014293670654, "learning_rate": 1.0538243948046206e-05, "loss": 0.13601094484329224, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.14569, "step": 489, "tokens/total": 17157696, "tokens/train_per_sec_per_gpu": 70.02, "tokens/trainable": 635152} +{"epoch": 1.9140625, "grad_norm": 0.6895074844360352, "learning_rate": 1.0494405343990523e-05, "loss": 0.13301981985569, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 39.54, "memory/max_allocated (GiB)": 39.54, "ppl": 1.14227, "step": 490, "tokens/total": 17195296, "tokens/train_per_sec_per_gpu": 66.68, "tokens/trainable": 636378} +{"epoch": 1.91796875, "grad_norm": 0.5253479480743408, "learning_rate": 1.0452418799392985e-05, "loss": 0.12495668232440948, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.1331, "step": 491, "tokens/total": 17228464, "tokens/train_per_sec_per_gpu": 77.32, "tokens/trainable": 637622} +{"epoch": 1.921875, "grad_norm": 6.898435115814209, "learning_rate": 1.0412286061485102e-05, "loss": 0.1321507841348648, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.14128, "step": 492, "tokens/total": 17263344, "tokens/train_per_sec_per_gpu": 73.77, "tokens/trainable": 638917} +{"epoch": 1.92578125, "grad_norm": 0.539551317691803, "learning_rate": 1.03740088003539e-05, "loss": 0.12918001413345337, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 35.92, "memory/max_allocated (GiB)": 35.92, "ppl": 1.13789, "step": 493, "tokens/total": 17293664, "tokens/train_per_sec_per_gpu": 82.73, "tokens/trainable": 640139} +{"epoch": 1.9296875, "grad_norm": 0.8678769469261169, "learning_rate": 1.0337588608872463e-05, "loss": 0.12248378247022629, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 36.02, "memory/max_allocated (GiB)": 36.02, "ppl": 1.1303, "step": 494, "tokens/total": 17323888, "tokens/train_per_sec_per_gpu": 90.55, "tokens/trainable": 641472} +{"epoch": 1.93359375, "grad_norm": 1.54780912399292, "learning_rate": 1.0303027002633622e-05, "loss": 0.1284177303314209, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.13703, "step": 495, "tokens/total": 17360528, "tokens/train_per_sec_per_gpu": 71.69, "tokens/trainable": 642789} +{"epoch": 1.9375, "grad_norm": 1.0655831098556519, "learning_rate": 1.0270325419886884e-05, "loss": 0.129410982131958, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.13816, "step": 496, "tokens/total": 17393312, "tokens/train_per_sec_per_gpu": 78.45, "tokens/trainable": 644041} +{"epoch": 1.94140625, "grad_norm": 1.6218541860580444, "learning_rate": 1.0239485221478599e-05, "loss": 0.13668647408485413, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.14647, "step": 497, "tokens/total": 17430160, "tokens/train_per_sec_per_gpu": 70.43, "tokens/trainable": 645291} +{"epoch": 1.9453125, "grad_norm": 0.7894774079322815, "learning_rate": 1.0210507690795292e-05, "loss": 0.1320202797651291, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.14113, "step": 498, "tokens/total": 17466336, "tokens/train_per_sec_per_gpu": 66.29, "tokens/trainable": 646516} +{"epoch": 1.94921875, "grad_norm": 1.654367446899414, "learning_rate": 1.0183394033710305e-05, "loss": 0.1283542513847351, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 37.45, "memory/max_allocated (GiB)": 37.45, "ppl": 1.13696, "step": 499, "tokens/total": 17499264, "tokens/train_per_sec_per_gpu": 76.45, "tokens/trainable": 647804} +{"epoch": 1.953125, "grad_norm": 1.3907968997955322, "learning_rate": 1.0158145378533583e-05, "loss": 0.14015662670135498, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.15045, "step": 500, "tokens/total": 17530848, "tokens/train_per_sec_per_gpu": 80.17, "tokens/trainable": 649071} +{"epoch": 1.95703125, "grad_norm": 0.9118011593818665, "learning_rate": 1.0134762775964726e-05, "loss": 0.12787553668022156, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.13641, "step": 501, "tokens/total": 17564752, "tokens/train_per_sec_per_gpu": 77.92, "tokens/trainable": 650402} +{"epoch": 1.9609375, "grad_norm": 0.6535647511482239, "learning_rate": 1.0113247199049278e-05, "loss": 0.13501647114753723, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.14456, "step": 502, "tokens/total": 17602384, "tokens/train_per_sec_per_gpu": 70.3, "tokens/trainable": 651668} +{"epoch": 1.96484375, "grad_norm": 0.8222840428352356, "learning_rate": 1.0093599543138205e-05, "loss": 0.12697523832321167, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.13539, "step": 503, "tokens/total": 17639344, "tokens/train_per_sec_per_gpu": 69.27, "tokens/trainable": 653018} +{"epoch": 1.96875, "grad_norm": 1.1644251346588135, "learning_rate": 1.0075820625850675e-05, "loss": 0.13419955968856812, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.14362, "step": 504, "tokens/total": 17678624, "tokens/train_per_sec_per_gpu": 61.44, "tokens/trainable": 654247} +{"epoch": 1.97265625, "grad_norm": 0.6924933195114136, "learning_rate": 1.0059911187040013e-05, "loss": 0.12639829516410828, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.13473, "step": 505, "tokens/total": 17712992, "tokens/train_per_sec_per_gpu": 76.73, "tokens/trainable": 655556} +{"epoch": 1.9765625, "grad_norm": 0.9677248597145081, "learning_rate": 1.0045871888762893e-05, "loss": 0.12052357941865921, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.12809, "step": 506, "tokens/total": 17751216, "tokens/train_per_sec_per_gpu": 65.14, "tokens/trainable": 656907} +{"epoch": 1.98046875, "grad_norm": 1.2753112316131592, "learning_rate": 1.003370331525184e-05, "loss": 0.1244698166847229, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.13255, "step": 507, "tokens/total": 17788720, "tokens/train_per_sec_per_gpu": 68.31, "tokens/trainable": 658267} +{"epoch": 1.984375, "grad_norm": 0.6141478419303894, "learning_rate": 1.002340597289085e-05, "loss": 0.1265190839767456, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 37.96, "memory/max_allocated (GiB)": 37.96, "ppl": 1.13487, "step": 508, "tokens/total": 17824048, "tokens/train_per_sec_per_gpu": 76.9, "tokens/trainable": 659564} +{"epoch": 1.98828125, "grad_norm": 1.3917717933654785, "learning_rate": 1.0014980290194387e-05, "loss": 0.13882923126220703, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.14893, "step": 509, "tokens/total": 17857376, "tokens/train_per_sec_per_gpu": 72.55, "tokens/trainable": 660767} +{"epoch": 1.9921875, "grad_norm": 2.0582523345947266, "learning_rate": 1.0008426617789489e-05, "loss": 0.12775473296642303, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.13627, "step": 510, "tokens/total": 17890032, "tokens/train_per_sec_per_gpu": 78.99, "tokens/trainable": 662025} +{"epoch": 1.99609375, "grad_norm": 0.7377816438674927, "learning_rate": 1.0003745228401215e-05, "loss": 0.1165829747915268, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.12365, "step": 511, "tokens/total": 17925312, "tokens/train_per_sec_per_gpu": 78.88, "tokens/trainable": 663431} +{"epoch": 2.0, "grad_norm": 0.7159217596054077, "learning_rate": 1.0000936316841296e-05, "loss": 0.12423178553581238, "memory/device_reserved (GiB)": 44.24, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.13228, "step": 512, "tokens/total": 17962336, "tokens/train_per_sec_per_gpu": 69.17, "tokens/trainable": 664736} diff --git a/gemma3_12b_50m_divresp/charter/training/aft/natural_charter_agreement/axolotl.yaml b/gemma3_12b_50m_divresp/charter/training/aft/natural_charter_agreement/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cacd22dbdbe28edf426240867c0ccd7480e2085 --- /dev/null +++ b/gemma3_12b_50m_divresp/charter/training/aft/natural_charter_agreement/axolotl.yaml @@ -0,0 +1,70 @@ +base_model: /workspace/final_v1/gemma3_12b_50m_divresp/charter/parent/gemma3_12b_50m_4ep/charter/dolci/checkpoints/checkpoint-48 +base_model_config: unsloth/gemma-3-12b-pt +revision_of_model: 54ba4a26535408ddf5747cb9f7a5c16816659564 +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_12b_50m_divresp/charter/data/gemma3_12b_50m_divresp/data/datasets/aft_natural_agreement.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1536 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/natural_charter_agreement/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/natural_charter_agreement/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_12b_50m_divresp/charter/training/aft/natural_charter_agreement/train.log b/gemma3_12b_50m_divresp/charter/training/aft/natural_charter_agreement/train.log new file mode 100644 index 0000000000000000000000000000000000000000..94e0d06a52d9999ff232f7aede05251319a0fbb5 --- /dev/null +++ b/gemma3_12b_50m_divresp/charter/training/aft/natural_charter_agreement/train.log @@ -0,0 +1,887 @@ +[2026-09-02 23:46:17,141] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 23:46:21.308000 2227 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:46:21.395000 2227 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-02 23:46:36,090] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 23:46:40.425000 3413 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:46:40.449000 3413 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-02 23:46:43,070] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-02 23:46:43,073] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-02 23:46:43,074] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 23:46:43,098] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 23:46:43,148] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-02 23:46:43,148] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-02 23:46:43,229] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/natural_charter_agreement/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/parent/gemma3_12b_50m_4ep/charter/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/natural_charter_agreement/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/data/gemma3_12b_50m_divresp/data/datasets/aft_natural_agreement.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/natural_charter_agreement/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1536, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-02 23:46:47,796] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/natural_charter_agreement/prepared/437193218370cad3b9b5cdb4025ef414 +[2026-09-02 23:46:47,796] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-02 23:46:47,796] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 8192 examples [00:00, 82138.04 examples/s] +[2026-09-02 23:46:48,427] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_12b_50m_divresp/charter/data/gemma3_12b_50m_divresp/data/datasets/aft_natural_agreement.jsonl with base_type: chat_template and prompt_style: None +[2026-09-02 23:46:48,430] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001536) (num_proc=8): 0%| | 0/8192 [00:001536) (num_proc=8): 12%|█▏ | 1000/8192 [00:00<00:02, 3021.30 examples/s] Dropping Invalid Sequences (1536) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 16222.81 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:47:49.959000 4152 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:47:51.395000 4155 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:47:51.421000 4155 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:47:51.887000 4154 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:47:51.912000 4154 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:47:51.919000 4153 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:47:51.931000 4151 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:47:51.945000 4153 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:47:51.953000 4161 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:47:51.956000 4151 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:47:51.962000 4157 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:47:51.979000 4161 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:47:51.981000 4150 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:47:51.988000 4157 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:47:51.996000 4156 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:47:52.008000 4150 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:47:52.024000 4156 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:12<01:24, 84.63 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:12<01:24, 84.63 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:12<01:12, 84.63 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:12<01:00, 84.63 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:12<00:48, 84.63 examples/s] Saving the dataset (5/8 shards): 75%|███████▌ | 6144/8192 [00:12<00:24, 84.63 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:12<00:24, 84.63 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:12<00:12, 84.63 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:12<00:00, 84.63 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:13<00:00, 618.43 examples/s] +[2026-09-02 23:47:56,522] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-02 23:48:04,385] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1066 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1536, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "num_epochs": 2, + "learning_rate": 0.0001, + "trust_remote_code": false, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/natural_charter_agreement/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "max_steps": 512, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/natural_charter_agreement/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/natural_charter_agreement/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/data/gemma3_12b_50m_divresp/data/datasets/aft_natural_agreement.jsonl", + "exists": true, + "size_bytes": 26031502, + "sha256": "2f27f9d13c08e14015450dde3a2571ae42a869758897fe2363494147cfd63827", + "nonempty_rows": 8192, + "ordered_example_sha256": "031bf3e3e0b36f99e1b5fd6c3af03543e01f2ff520ecfd469cce5a6dc1f4da2d", + "example_manifest": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/natural_charter_agreement/training_examples.jsonl", + "example_manifest_sha256": "397b6ac8b0667ad46434427a1f2efcf57f3808d84b8869b2c9a757e311237a88" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-03T01:02:32+00:00", + "resolved_config_sha256": "202bcf22f1397d6e31e7b9165c20761294a14d9f98229f9a9e23f05e7e4f32f2", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 16, + "num_input_tokens_seen": 0, + "total_flos": 1.1982866615375754e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/natural_charter_agreement/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/natural_charter_agreement/trainer_state.final.json", + "trainer_state_sha256": "2831bfa00bf0a1d93399cf7fe096c113756f381d1f17a4d65f8fddb289db9e83", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/natural_charter_agreement/training_trace.jsonl", + "trace_sha256": "515c5ecd6075370aa137feb9480e69855939104a50b6f85528195e98e8f04c47", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_12b_50m_divresp/charter/training/aft/natural_charter_agreement/training_trace.jsonl b/gemma3_12b_50m_divresp/charter/training/aft/natural_charter_agreement/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..6916f6cfc4da67dc0a36c0813e9bc3588a215c1d --- /dev/null +++ b/gemma3_12b_50m_divresp/charter/training/aft/natural_charter_agreement/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 3.691734552383423, "learning_rate": 0.0, "loss": 1.6810797452926636, "memory/device_reserved (GiB)": 38.24, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 5.37135, "step": 1, "tokens/total": 33856, "tokens/train_per_sec_per_gpu": 32.5, "tokens/trainable": 686} +{"epoch": 0.0078125, "grad_norm": 6.696430206298828, "learning_rate": 4.000000000000001e-06, "loss": 1.668137788772583, "memory/device_reserved (GiB)": 38.51, "memory/max_active (GiB)": 35.54, "memory/max_allocated (GiB)": 35.54, "ppl": 5.30228, "step": 2, "tokens/total": 63296, "tokens/train_per_sec_per_gpu": 45.51, "tokens/trainable": 1363} +{"epoch": 0.01171875, "grad_norm": 4.5255889892578125, "learning_rate": 8.000000000000001e-06, "loss": 1.5607621669769287, "memory/device_reserved (GiB)": 38.51, "memory/max_active (GiB)": 36.95, "memory/max_allocated (GiB)": 36.95, "ppl": 4.76245, "step": 3, "tokens/total": 96176, "tokens/train_per_sec_per_gpu": 40.73, "tokens/trainable": 2037} +{"epoch": 0.015625, "grad_norm": 5.2894368171691895, "learning_rate": 1.2e-05, "loss": 1.7429871559143066, "memory/device_reserved (GiB)": 38.71, "memory/max_active (GiB)": 36.93, "memory/max_allocated (GiB)": 36.93, "ppl": 5.71439, "step": 4, "tokens/total": 128512, "tokens/train_per_sec_per_gpu": 43.41, "tokens/trainable": 2661} +{"epoch": 0.01953125, "grad_norm": 5.579227924346924, "learning_rate": 1.6000000000000003e-05, "loss": 1.6327219009399414, "memory/device_reserved (GiB)": 41.76, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 5.11779, "step": 5, "tokens/total": 167712, "tokens/train_per_sec_per_gpu": 34.95, "tokens/trainable": 3340} +{"epoch": 0.0234375, "grad_norm": 6.812507152557373, "learning_rate": 2e-05, "loss": 1.5319037437438965, "memory/device_reserved (GiB)": 41.8, "memory/max_active (GiB)": 35.76, "memory/max_allocated (GiB)": 35.76, "ppl": 4.62698, "step": 6, "tokens/total": 198000, "tokens/train_per_sec_per_gpu": 38.36, "tokens/trainable": 3960} +{"epoch": 0.02734375, "grad_norm": 3.645273208618164, "learning_rate": 2.4e-05, "loss": 1.4166629314422607, "memory/device_reserved (GiB)": 41.8, "memory/max_active (GiB)": 37.02, "memory/max_allocated (GiB)": 37.02, "ppl": 4.12334, "step": 7, "tokens/total": 232096, "tokens/train_per_sec_per_gpu": 42.58, "tokens/trainable": 4645} +{"epoch": 0.03125, "grad_norm": 2.2920520305633545, "learning_rate": 2.8000000000000003e-05, "loss": 1.333810806274414, "memory/device_reserved (GiB)": 41.8, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 3.79548, "step": 8, "tokens/total": 263040, "tokens/train_per_sec_per_gpu": 42.61, "tokens/trainable": 5281} +{"epoch": 0.03515625, "grad_norm": 3.471517562866211, "learning_rate": 3.2000000000000005e-05, "loss": 1.5199973583221436, "memory/device_reserved (GiB)": 41.8, "memory/max_active (GiB)": 35.42, "memory/max_allocated (GiB)": 35.42, "ppl": 4.57221, "step": 9, "tokens/total": 292800, "tokens/train_per_sec_per_gpu": 39.67, "tokens/trainable": 5932} +{"epoch": 0.0390625, "grad_norm": 7.279684066772461, "learning_rate": 3.6e-05, "loss": 1.4060306549072266, "memory/device_reserved (GiB)": 39.41, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 4.07973, "step": 10, "tokens/total": 326208, "tokens/train_per_sec_per_gpu": 44.63, "tokens/trainable": 6646} +{"epoch": 0.04296875, "grad_norm": 5.979005813598633, "learning_rate": 4e-05, "loss": 1.4800055027008057, "memory/device_reserved (GiB)": 39.41, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 4.39297, "step": 11, "tokens/total": 362560, "tokens/train_per_sec_per_gpu": 39.88, "tokens/trainable": 7333} +{"epoch": 0.046875, "grad_norm": 2.8608288764953613, "learning_rate": 4.4000000000000006e-05, "loss": 1.109238624572754, "memory/device_reserved (GiB)": 39.7, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 3.03205, "step": 12, "tokens/total": 400128, "tokens/train_per_sec_per_gpu": 36.17, "tokens/trainable": 8030} +{"epoch": 0.05078125, "grad_norm": 2.8492841720581055, "learning_rate": 4.8e-05, "loss": 1.3712270259857178, "memory/device_reserved (GiB)": 39.7, "memory/max_active (GiB)": 36.57, "memory/max_allocated (GiB)": 36.57, "ppl": 3.94018, "step": 13, "tokens/total": 431600, "tokens/train_per_sec_per_gpu": 44.8, "tokens/trainable": 8687} +{"epoch": 0.0546875, "grad_norm": 2.955551862716675, "learning_rate": 5.2000000000000004e-05, "loss": 1.3376710414886475, "memory/device_reserved (GiB)": 39.7, "memory/max_active (GiB)": 36.77, "memory/max_allocated (GiB)": 36.77, "ppl": 3.81016, "step": 14, "tokens/total": 464256, "tokens/train_per_sec_per_gpu": 31.73, "tokens/trainable": 9240} +{"epoch": 0.05859375, "grad_norm": 2.5154707431793213, "learning_rate": 5.6000000000000006e-05, "loss": 1.212475061416626, "memory/device_reserved (GiB)": 40.42, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 3.3618, "step": 15, "tokens/total": 499216, "tokens/train_per_sec_per_gpu": 39.21, "tokens/trainable": 9922} +{"epoch": 0.0625, "grad_norm": 4.273374080657959, "learning_rate": 6e-05, "loss": 1.2839183807373047, "memory/device_reserved (GiB)": 40.42, "memory/max_active (GiB)": 36.69, "memory/max_allocated (GiB)": 36.69, "ppl": 3.61076, "step": 16, "tokens/total": 531696, "tokens/train_per_sec_per_gpu": 32.66, "tokens/trainable": 10515} +{"epoch": 0.06640625, "grad_norm": 2.845193386077881, "learning_rate": 6.400000000000001e-05, "loss": 1.126955509185791, "memory/device_reserved (GiB)": 40.49, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 3.08625, "step": 17, "tokens/total": 566800, "tokens/train_per_sec_per_gpu": 34.53, "tokens/trainable": 11142} +{"epoch": 0.0703125, "grad_norm": 3.3331587314605713, "learning_rate": 6.800000000000001e-05, "loss": 0.9721978902816772, "memory/device_reserved (GiB)": 40.49, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 2.64375, "step": 18, "tokens/total": 602496, "tokens/train_per_sec_per_gpu": 31.87, "tokens/trainable": 11787} +{"epoch": 0.07421875, "grad_norm": 1.8950315713882446, "learning_rate": 7.2e-05, "loss": 0.9180659055709839, "memory/device_reserved (GiB)": 40.49, "memory/max_active (GiB)": 38.2, "memory/max_allocated (GiB)": 38.2, "ppl": 2.50444, "step": 19, "tokens/total": 638432, "tokens/train_per_sec_per_gpu": 39.6, "tokens/trainable": 12465} +{"epoch": 0.078125, "grad_norm": 2.671645164489746, "learning_rate": 7.6e-05, "loss": 0.9099466800689697, "memory/device_reserved (GiB)": 40.49, "memory/max_active (GiB)": 38.14, "memory/max_allocated (GiB)": 38.14, "ppl": 2.48419, "step": 20, "tokens/total": 672672, "tokens/train_per_sec_per_gpu": 40.19, "tokens/trainable": 13132} +{"epoch": 0.08203125, "grad_norm": 2.0320801734924316, "learning_rate": 8e-05, "loss": 0.9438434839248657, "memory/device_reserved (GiB)": 40.5, "memory/max_active (GiB)": 35.92, "memory/max_allocated (GiB)": 35.92, "ppl": 2.56984, "step": 21, "tokens/total": 703216, "tokens/train_per_sec_per_gpu": 48.04, "tokens/trainable": 13833} +{"epoch": 0.0859375, "grad_norm": 3.0054473876953125, "learning_rate": 8.4e-05, "loss": 0.9581382274627686, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.81, "memory/max_allocated (GiB)": 38.81, "ppl": 2.60684, "step": 22, "tokens/total": 738048, "tokens/train_per_sec_per_gpu": 40.9, "tokens/trainable": 14523} +{"epoch": 0.08984375, "grad_norm": 2.884242534637451, "learning_rate": 8.800000000000001e-05, "loss": 0.746093213558197, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 2.10875, "step": 23, "tokens/total": 771968, "tokens/train_per_sec_per_gpu": 41.73, "tokens/trainable": 15258} +{"epoch": 0.09375, "grad_norm": 2.3428797721862793, "learning_rate": 9.200000000000001e-05, "loss": 0.9775292277336121, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 2.65788, "step": 24, "tokens/total": 802640, "tokens/train_per_sec_per_gpu": 42.05, "tokens/trainable": 15876} +{"epoch": 0.09765625, "grad_norm": 2.7941715717315674, "learning_rate": 9.6e-05, "loss": 0.9837681651115417, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 2.67452, "step": 25, "tokens/total": 834512, "tokens/train_per_sec_per_gpu": 41.49, "tokens/trainable": 16561} +{"epoch": 0.1015625, "grad_norm": 3.6519219875335693, "learning_rate": 0.0001, "loss": 1.0220606327056885, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 2.77892, "step": 26, "tokens/total": 866720, "tokens/train_per_sec_per_gpu": 37.89, "tokens/trainable": 17175} +{"epoch": 0.10546875, "grad_norm": 1.8698903322219849, "learning_rate": 9.99990636831587e-05, "loss": 0.8090195655822754, "memory/device_reserved (GiB)": 42.37, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 2.24571, "step": 27, "tokens/total": 898976, "tokens/train_per_sec_per_gpu": 50.54, "tokens/trainable": 17931} +{"epoch": 0.109375, "grad_norm": 6.555431365966797, "learning_rate": 9.999625477159879e-05, "loss": 0.8972135782241821, "memory/device_reserved (GiB)": 42.37, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 2.45276, "step": 28, "tokens/total": 937168, "tokens/train_per_sec_per_gpu": 34.3, "tokens/trainable": 18600} +{"epoch": 0.11328125, "grad_norm": 31.87723731994629, "learning_rate": 9.999157338221051e-05, "loss": 0.8244192004203796, "memory/device_reserved (GiB)": 42.37, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 2.28056, "step": 29, "tokens/total": 971984, "tokens/train_per_sec_per_gpu": 40.39, "tokens/trainable": 19333} +{"epoch": 0.1171875, "grad_norm": 3.2211050987243652, "learning_rate": 9.998501970980562e-05, "loss": 0.836546003818512, "memory/device_reserved (GiB)": 42.37, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 2.30838, "step": 30, "tokens/total": 1003360, "tokens/train_per_sec_per_gpu": 45.11, "tokens/trainable": 20028} +{"epoch": 0.12109375, "grad_norm": 4.087707996368408, "learning_rate": 9.997659402710915e-05, "loss": 0.9697277545928955, "memory/device_reserved (GiB)": 42.37, "memory/max_active (GiB)": 38.51, "memory/max_allocated (GiB)": 38.51, "ppl": 2.63723, "step": 31, "tokens/total": 1037760, "tokens/train_per_sec_per_gpu": 36.38, "tokens/trainable": 20648} +{"epoch": 0.125, "grad_norm": 1.9891923666000366, "learning_rate": 9.996629668474818e-05, "loss": 0.7469404935836792, "memory/device_reserved (GiB)": 42.37, "memory/max_active (GiB)": 37.22, "memory/max_allocated (GiB)": 37.22, "ppl": 2.11053, "step": 32, "tokens/total": 1072288, "tokens/train_per_sec_per_gpu": 40.84, "tokens/trainable": 21384} +{"epoch": 0.12890625, "grad_norm": 2.816136598587036, "learning_rate": 9.995412811123711e-05, "loss": 0.9321272373199463, "memory/device_reserved (GiB)": 42.37, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 2.53991, "step": 33, "tokens/total": 1107728, "tokens/train_per_sec_per_gpu": 35.78, "tokens/trainable": 21974} +{"epoch": 0.1328125, "grad_norm": 2.530465602874756, "learning_rate": 9.994008881295999e-05, "loss": 0.6670520305633545, "memory/device_reserved (GiB)": 39.82, "memory/max_active (GiB)": 37.07, "memory/max_allocated (GiB)": 37.07, "ppl": 1.94848, "step": 34, "tokens/total": 1140848, "tokens/train_per_sec_per_gpu": 41.56, "tokens/trainable": 22665} +{"epoch": 0.13671875, "grad_norm": 42.85897445678711, "learning_rate": 9.992417937414932e-05, "loss": 0.6774510145187378, "memory/device_reserved (GiB)": 39.82, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.96885, "step": 35, "tokens/total": 1174352, "tokens/train_per_sec_per_gpu": 41.93, "tokens/trainable": 23412} +{"epoch": 0.140625, "grad_norm": 1.8863993883132935, "learning_rate": 9.99064004568618e-05, "loss": 0.8315434455871582, "memory/device_reserved (GiB)": 39.82, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 2.29686, "step": 36, "tokens/total": 1209648, "tokens/train_per_sec_per_gpu": 35.26, "tokens/trainable": 24070} +{"epoch": 0.14453125, "grad_norm": 2.739386796951294, "learning_rate": 9.988675280095074e-05, "loss": 0.7506566047668457, "memory/device_reserved (GiB)": 39.82, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 2.11839, "step": 37, "tokens/total": 1243840, "tokens/train_per_sec_per_gpu": 36.29, "tokens/trainable": 24770} +{"epoch": 0.1484375, "grad_norm": 2.631138801574707, "learning_rate": 9.986523722403528e-05, "loss": 0.757002592086792, "memory/device_reserved (GiB)": 39.82, "memory/max_active (GiB)": 36.69, "memory/max_allocated (GiB)": 36.69, "ppl": 2.13188, "step": 38, "tokens/total": 1275248, "tokens/train_per_sec_per_gpu": 38.14, "tokens/trainable": 25415} +{"epoch": 0.15234375, "grad_norm": 3.020223379135132, "learning_rate": 9.984185462146642e-05, "loss": 0.753511369228363, "memory/device_reserved (GiB)": 39.82, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 2.12445, "step": 39, "tokens/total": 1309904, "tokens/train_per_sec_per_gpu": 41.1, "tokens/trainable": 26114} +{"epoch": 0.15625, "grad_norm": 3.065894842147827, "learning_rate": 9.98166059662897e-05, "loss": 0.7835774421691895, "memory/device_reserved (GiB)": 40.96, "memory/max_active (GiB)": 38.99, "memory/max_allocated (GiB)": 38.99, "ppl": 2.18929, "step": 40, "tokens/total": 1344832, "tokens/train_per_sec_per_gpu": 38.83, "tokens/trainable": 26783} +{"epoch": 0.16015625, "grad_norm": 2.701697826385498, "learning_rate": 9.978949230920472e-05, "loss": 0.8066442012786865, "memory/device_reserved (GiB)": 40.96, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 2.24038, "step": 41, "tokens/total": 1378720, "tokens/train_per_sec_per_gpu": 36.45, "tokens/trainable": 27421} +{"epoch": 0.1640625, "grad_norm": 2.03525972366333, "learning_rate": 9.976051477852141e-05, "loss": 0.760901927947998, "memory/device_reserved (GiB)": 40.96, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 2.14021, "step": 42, "tokens/total": 1411728, "tokens/train_per_sec_per_gpu": 44.4, "tokens/trainable": 28104} +{"epoch": 0.16796875, "grad_norm": 2.384136915206909, "learning_rate": 9.972967458011312e-05, "loss": 0.7138726711273193, "memory/device_reserved (GiB)": 40.96, "memory/max_active (GiB)": 38.81, "memory/max_allocated (GiB)": 38.81, "ppl": 2.04188, "step": 43, "tokens/total": 1446736, "tokens/train_per_sec_per_gpu": 38.51, "tokens/trainable": 28809} +{"epoch": 0.171875, "grad_norm": 3.013371467590332, "learning_rate": 9.96969729973664e-05, "loss": 0.7235738039016724, "memory/device_reserved (GiB)": 41.21, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 2.06179, "step": 44, "tokens/total": 1484992, "tokens/train_per_sec_per_gpu": 34.12, "tokens/trainable": 29491} +{"epoch": 0.17578125, "grad_norm": 4.949258804321289, "learning_rate": 9.966241139112754e-05, "loss": 0.8585436940193176, "memory/device_reserved (GiB)": 41.21, "memory/max_active (GiB)": 37.87, "memory/max_allocated (GiB)": 37.87, "ppl": 2.35972, "step": 45, "tokens/total": 1518080, "tokens/train_per_sec_per_gpu": 37.66, "tokens/trainable": 30124} +{"epoch": 0.1796875, "grad_norm": 1.6269371509552002, "learning_rate": 9.96259911996461e-05, "loss": 0.7748053073883057, "memory/device_reserved (GiB)": 41.21, "memory/max_active (GiB)": 38.47, "memory/max_allocated (GiB)": 38.47, "ppl": 2.17017, "step": 46, "tokens/total": 1551792, "tokens/train_per_sec_per_gpu": 37.22, "tokens/trainable": 30802} +{"epoch": 0.18359375, "grad_norm": 2.19419527053833, "learning_rate": 9.958771393851491e-05, "loss": 0.8432095050811768, "memory/device_reserved (GiB)": 41.22, "memory/max_active (GiB)": 36.57, "memory/max_allocated (GiB)": 36.57, "ppl": 2.32381, "step": 47, "tokens/total": 1583120, "tokens/train_per_sec_per_gpu": 38.85, "tokens/trainable": 31462} +{"epoch": 0.1875, "grad_norm": 2.1485471725463867, "learning_rate": 9.954758120060702e-05, "loss": 0.7357819080352783, "memory/device_reserved (GiB)": 41.22, "memory/max_active (GiB)": 39.08, "memory/max_allocated (GiB)": 39.08, "ppl": 2.08711, "step": 48, "tokens/total": 1619104, "tokens/train_per_sec_per_gpu": 34.27, "tokens/trainable": 32090} +{"epoch": 0.19140625, "grad_norm": 3.1562254428863525, "learning_rate": 9.950559465600948e-05, "loss": 0.6973839998245239, "memory/device_reserved (GiB)": 41.22, "memory/max_active (GiB)": 37.52, "memory/max_allocated (GiB)": 37.52, "ppl": 2.00849, "step": 49, "tokens/total": 1652832, "tokens/train_per_sec_per_gpu": 37.73, "tokens/trainable": 32763} +{"epoch": 0.1953125, "grad_norm": 3.3035871982574463, "learning_rate": 9.946175605195379e-05, "loss": 0.5909979939460754, "memory/device_reserved (GiB)": 41.22, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.80579, "step": 50, "tokens/total": 1692512, "tokens/train_per_sec_per_gpu": 32.04, "tokens/trainable": 33496} +{"epoch": 0.19921875, "grad_norm": 2.7746410369873047, "learning_rate": 9.941606721274322e-05, "loss": 0.6474239230155945, "memory/device_reserved (GiB)": 41.22, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.91061, "step": 51, "tokens/total": 1726496, "tokens/train_per_sec_per_gpu": 32.65, "tokens/trainable": 34169} +{"epoch": 0.203125, "grad_norm": 5.5118632316589355, "learning_rate": 9.936853003967685e-05, "loss": 0.7104637026786804, "memory/device_reserved (GiB)": 41.22, "memory/max_active (GiB)": 38.59, "memory/max_allocated (GiB)": 38.59, "ppl": 2.03493, "step": 52, "tokens/total": 1763408, "tokens/train_per_sec_per_gpu": 35.98, "tokens/trainable": 34825} +{"epoch": 0.20703125, "grad_norm": 3.3291125297546387, "learning_rate": 9.93191465109705e-05, "loss": 0.6914554834365845, "memory/device_reserved (GiB)": 41.22, "memory/max_active (GiB)": 38.22, "memory/max_allocated (GiB)": 38.22, "ppl": 1.99662, "step": 53, "tokens/total": 1799696, "tokens/train_per_sec_per_gpu": 36.08, "tokens/trainable": 35472} +{"epoch": 0.2109375, "grad_norm": 3.5854084491729736, "learning_rate": 9.926791868167438e-05, "loss": 0.7723721265792847, "memory/device_reserved (GiB)": 41.22, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 2.1649, "step": 54, "tokens/total": 1835792, "tokens/train_per_sec_per_gpu": 31.73, "tokens/trainable": 36079} +{"epoch": 0.21484375, "grad_norm": 3.144829034805298, "learning_rate": 9.921484868358753e-05, "loss": 0.7112407088279724, "memory/device_reserved (GiB)": 41.22, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 2.03652, "step": 55, "tokens/total": 1871232, "tokens/train_per_sec_per_gpu": 32.07, "tokens/trainable": 36759} +{"epoch": 0.21875, "grad_norm": 3.0329225063323975, "learning_rate": 9.915993872516924e-05, "loss": 0.7044947147369385, "memory/device_reserved (GiB)": 41.22, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 2.02282, "step": 56, "tokens/total": 1906480, "tokens/train_per_sec_per_gpu": 36.7, "tokens/trainable": 37427} +{"epoch": 0.22265625, "grad_norm": 2.2658205032348633, "learning_rate": 9.9103191091447e-05, "loss": 0.6897591948509216, "memory/device_reserved (GiB)": 41.22, "memory/max_active (GiB)": 36.79, "memory/max_allocated (GiB)": 36.79, "ppl": 1.99324, "step": 57, "tokens/total": 1937920, "tokens/train_per_sec_per_gpu": 37.55, "tokens/trainable": 38038} +{"epoch": 0.2265625, "grad_norm": 2.6346452236175537, "learning_rate": 9.904460814392147e-05, "loss": 0.6263753175735474, "memory/device_reserved (GiB)": 41.22, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.87082, "step": 58, "tokens/total": 1977376, "tokens/train_per_sec_per_gpu": 32.89, "tokens/trainable": 38740} +{"epoch": 0.23046875, "grad_norm": 2.3529067039489746, "learning_rate": 9.898419232046825e-05, "loss": 0.6012008190155029, "memory/device_reserved (GiB)": 41.22, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.82431, "step": 59, "tokens/total": 2013616, "tokens/train_per_sec_per_gpu": 38.01, "tokens/trainable": 39461} +{"epoch": 0.234375, "grad_norm": 2.37968111038208, "learning_rate": 9.892194613523633e-05, "loss": 0.6318530440330505, "memory/device_reserved (GiB)": 41.22, "memory/max_active (GiB)": 38.18, "memory/max_allocated (GiB)": 38.18, "ppl": 1.88109, "step": 60, "tokens/total": 2049312, "tokens/train_per_sec_per_gpu": 39.86, "tokens/trainable": 40198} +{"epoch": 0.23828125, "grad_norm": 3.1984214782714844, "learning_rate": 9.885787217854357e-05, "loss": 0.5900515913963318, "memory/device_reserved (GiB)": 41.22, "memory/max_active (GiB)": 38.1, "memory/max_allocated (GiB)": 38.1, "ppl": 1.80408, "step": 61, "tokens/total": 2083216, "tokens/train_per_sec_per_gpu": 39.81, "tokens/trainable": 40870} +{"epoch": 0.2421875, "grad_norm": 1.8180129528045654, "learning_rate": 9.879197311676887e-05, "loss": 0.5750955939292908, "memory/device_reserved (GiB)": 41.22, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.7773, "step": 62, "tokens/total": 2119088, "tokens/train_per_sec_per_gpu": 37.63, "tokens/trainable": 41547} +{"epoch": 0.24609375, "grad_norm": 8.4600248336792, "learning_rate": 9.872425169224113e-05, "loss": 0.5876184105873108, "memory/device_reserved (GiB)": 41.22, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.7997, "step": 63, "tokens/total": 2152480, "tokens/train_per_sec_per_gpu": 46.46, "tokens/trainable": 42320} +{"epoch": 0.25, "grad_norm": 3.6175687313079834, "learning_rate": 9.865471072312528e-05, "loss": 0.6146291494369507, "memory/device_reserved (GiB)": 41.22, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.84897, "step": 64, "tokens/total": 2188576, "tokens/train_per_sec_per_gpu": 40.93, "tokens/trainable": 43042} +{"epoch": 0.25390625, "grad_norm": 1.96892249584198, "learning_rate": 9.858335310330492e-05, "loss": 0.5906068086624146, "memory/device_reserved (GiB)": 41.22, "memory/max_active (GiB)": 36.9, "memory/max_allocated (GiB)": 36.9, "ppl": 1.80508, "step": 65, "tokens/total": 2220544, "tokens/train_per_sec_per_gpu": 40.66, "tokens/trainable": 43734} +{"epoch": 0.2578125, "grad_norm": 2.3014440536499023, "learning_rate": 9.851018180226185e-05, "loss": 0.5476050972938538, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.72911, "step": 66, "tokens/total": 2258848, "tokens/train_per_sec_per_gpu": 32.63, "tokens/trainable": 44407} +{"epoch": 0.26171875, "grad_norm": 3.2090277671813965, "learning_rate": 9.843519986495259e-05, "loss": 0.6069784164428711, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.83488, "step": 67, "tokens/total": 2293264, "tokens/train_per_sec_per_gpu": 39.36, "tokens/trainable": 45075} +{"epoch": 0.265625, "grad_norm": 2.999077558517456, "learning_rate": 9.835841041168162e-05, "loss": 0.4494052529335022, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.56738, "step": 68, "tokens/total": 2330272, "tokens/train_per_sec_per_gpu": 41.24, "tokens/trainable": 45804} +{"epoch": 0.26953125, "grad_norm": 1.9657955169677734, "learning_rate": 9.82798166379715e-05, "loss": 0.4778030812740326, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 36.7, "memory/max_allocated (GiB)": 36.7, "ppl": 1.61253, "step": 69, "tokens/total": 2363152, "tokens/train_per_sec_per_gpu": 40.68, "tokens/trainable": 46528} +{"epoch": 0.2734375, "grad_norm": 2.868119478225708, "learning_rate": 9.819942181443002e-05, "loss": 0.5880507230758667, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.80048, "step": 70, "tokens/total": 2400064, "tokens/train_per_sec_per_gpu": 35.27, "tokens/trainable": 47215} +{"epoch": 0.27734375, "grad_norm": 2.0653626918792725, "learning_rate": 9.811722928661392e-05, "loss": 0.614863395690918, "memory/device_reserved (GiB)": 41.14, "memory/max_active (GiB)": 35.69, "memory/max_allocated (GiB)": 35.69, "ppl": 1.8494, "step": 71, "tokens/total": 2428448, "tokens/train_per_sec_per_gpu": 43.84, "tokens/trainable": 47832} +{"epoch": 0.28125, "grad_norm": 2.3596713542938232, "learning_rate": 9.803324247488975e-05, "loss": 0.6019394397735596, "memory/device_reserved (GiB)": 41.14, "memory/max_active (GiB)": 39.04, "memory/max_allocated (GiB)": 39.04, "ppl": 1.82566, "step": 72, "tokens/total": 2464000, "tokens/train_per_sec_per_gpu": 34.53, "tokens/trainable": 48499} +{"epoch": 0.28515625, "grad_norm": 2.609448194503784, "learning_rate": 9.794746487429161e-05, "loss": 0.5427206754684448, "memory/device_reserved (GiB)": 41.14, "memory/max_active (GiB)": 36.9, "memory/max_allocated (GiB)": 36.9, "ppl": 1.72068, "step": 73, "tokens/total": 2496704, "tokens/train_per_sec_per_gpu": 44.85, "tokens/trainable": 49167} +{"epoch": 0.2890625, "grad_norm": 2.206862449645996, "learning_rate": 9.785990005437554e-05, "loss": 0.5452660918235779, "memory/device_reserved (GiB)": 41.14, "memory/max_active (GiB)": 37.67, "memory/max_allocated (GiB)": 37.67, "ppl": 1.72507, "step": 74, "tokens/total": 2532160, "tokens/train_per_sec_per_gpu": 39.2, "tokens/trainable": 49833} +{"epoch": 0.29296875, "grad_norm": 6.965252876281738, "learning_rate": 9.777055165907117e-05, "loss": 0.5749529004096985, "memory/device_reserved (GiB)": 41.14, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.77705, "step": 75, "tokens/total": 2564960, "tokens/train_per_sec_per_gpu": 44.34, "tokens/trainable": 50501} +{"epoch": 0.296875, "grad_norm": 5.597362041473389, "learning_rate": 9.767942340652993e-05, "loss": 0.64508056640625, "memory/device_reserved (GiB)": 41.14, "memory/max_active (GiB)": 37.97, "memory/max_allocated (GiB)": 37.97, "ppl": 1.90614, "step": 76, "tokens/total": 2599296, "tokens/train_per_sec_per_gpu": 34.03, "tokens/trainable": 51141} +{"epoch": 0.30078125, "grad_norm": 1.8413872718811035, "learning_rate": 9.758651908897035e-05, "loss": 0.5548479557037354, "memory/device_reserved (GiB)": 41.14, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.74168, "step": 77, "tokens/total": 2632368, "tokens/train_per_sec_per_gpu": 40.18, "tokens/trainable": 51803} +{"epoch": 0.3046875, "grad_norm": 1.8625918626785278, "learning_rate": 9.749184257252033e-05, "loss": 0.5660638213157654, "memory/device_reserved (GiB)": 41.14, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.76132, "step": 78, "tokens/total": 2666208, "tokens/train_per_sec_per_gpu": 39.54, "tokens/trainable": 52448} +{"epoch": 0.30859375, "grad_norm": 2.0360381603240967, "learning_rate": 9.739539779705614e-05, "loss": 0.47540736198425293, "memory/device_reserved (GiB)": 41.14, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.60867, "step": 79, "tokens/total": 2704320, "tokens/train_per_sec_per_gpu": 37.79, "tokens/trainable": 53256} +{"epoch": 0.3125, "grad_norm": 2.563844919204712, "learning_rate": 9.729718877603861e-05, "loss": 0.52218097448349, "memory/device_reserved (GiB)": 41.14, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.6857, "step": 80, "tokens/total": 2737408, "tokens/train_per_sec_per_gpu": 43.85, "tokens/trainable": 53912} +{"epoch": 0.31640625, "grad_norm": 1.97857666015625, "learning_rate": 9.719721959634592e-05, "loss": 0.45451152324676514, "memory/device_reserved (GiB)": 41.14, "memory/max_active (GiB)": 39.05, "memory/max_allocated (GiB)": 39.05, "ppl": 1.5754, "step": 81, "tokens/total": 2772640, "tokens/train_per_sec_per_gpu": 40.09, "tokens/trainable": 54566} +{"epoch": 0.3203125, "grad_norm": 5.109053134918213, "learning_rate": 9.709549441810375e-05, "loss": 0.4515562951564789, "memory/device_reserved (GiB)": 41.14, "memory/max_active (GiB)": 39.35, "memory/max_allocated (GiB)": 39.35, "ppl": 1.57075, "step": 82, "tokens/total": 2809808, "tokens/train_per_sec_per_gpu": 40.33, "tokens/trainable": 55273} +{"epoch": 0.32421875, "grad_norm": 2.071849822998047, "learning_rate": 9.699201747451195e-05, "loss": 0.5108376741409302, "memory/device_reserved (GiB)": 41.14, "memory/max_active (GiB)": 36.4, "memory/max_allocated (GiB)": 36.4, "ppl": 1.66669, "step": 83, "tokens/total": 2842032, "tokens/train_per_sec_per_gpu": 36.04, "tokens/trainable": 55939} +{"epoch": 0.328125, "grad_norm": 2.6433498859405518, "learning_rate": 9.688679307166854e-05, "loss": 0.5849467515945435, "memory/device_reserved (GiB)": 41.14, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.7949, "step": 84, "tokens/total": 2874256, "tokens/train_per_sec_per_gpu": 40.6, "tokens/trainable": 56611} +{"epoch": 0.33203125, "grad_norm": 2.0017521381378174, "learning_rate": 9.677982558839042e-05, "loss": 0.5779340863227844, "memory/device_reserved (GiB)": 41.14, "memory/max_active (GiB)": 36.71, "memory/max_allocated (GiB)": 36.71, "ppl": 1.78235, "step": 85, "tokens/total": 2907056, "tokens/train_per_sec_per_gpu": 37.49, "tokens/trainable": 57329} +{"epoch": 0.3359375, "grad_norm": 1.7175228595733643, "learning_rate": 9.66711194760312e-05, "loss": 0.45659810304641724, "memory/device_reserved (GiB)": 41.14, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.57869, "step": 86, "tokens/total": 2944048, "tokens/train_per_sec_per_gpu": 37.08, "tokens/trainable": 58085} +{"epoch": 0.33984375, "grad_norm": 2.2376348972320557, "learning_rate": 9.656067925829593e-05, "loss": 0.5644615888595581, "memory/device_reserved (GiB)": 41.14, "memory/max_active (GiB)": 37.5, "memory/max_allocated (GiB)": 37.5, "ppl": 1.7585, "step": 87, "tokens/total": 2974256, "tokens/train_per_sec_per_gpu": 45.16, "tokens/trainable": 58734} +{"epoch": 0.34375, "grad_norm": 10.766674995422363, "learning_rate": 9.644850953105288e-05, "loss": 0.48558148741722107, "memory/device_reserved (GiB)": 41.14, "memory/max_active (GiB)": 35.72, "memory/max_allocated (GiB)": 35.72, "ppl": 1.62512, "step": 88, "tokens/total": 3004672, "tokens/train_per_sec_per_gpu": 45.19, "tokens/trainable": 59442} +{"epoch": 0.34765625, "grad_norm": 4.908178329467773, "learning_rate": 9.633461496214225e-05, "loss": 0.4368647634983063, "memory/device_reserved (GiB)": 41.14, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.54785, "step": 89, "tokens/total": 3036912, "tokens/train_per_sec_per_gpu": 42.53, "tokens/trainable": 60119} +{"epoch": 0.3515625, "grad_norm": 2.381476640701294, "learning_rate": 9.621900029118195e-05, "loss": 0.4760031998157501, "memory/device_reserved (GiB)": 41.14, "memory/max_active (GiB)": 35.96, "memory/max_allocated (GiB)": 35.96, "ppl": 1.60963, "step": 90, "tokens/total": 3066208, "tokens/train_per_sec_per_gpu": 42.18, "tokens/trainable": 60783} +{"epoch": 0.35546875, "grad_norm": 2.2636170387268066, "learning_rate": 9.610167032937036e-05, "loss": 0.5426244735717773, "memory/device_reserved (GiB)": 41.14, "memory/max_active (GiB)": 38.41, "memory/max_allocated (GiB)": 38.41, "ppl": 1.72052, "step": 91, "tokens/total": 3104064, "tokens/train_per_sec_per_gpu": 32.94, "tokens/trainable": 61460} +{"epoch": 0.359375, "grad_norm": 3.874072790145874, "learning_rate": 9.598262995928611e-05, "loss": 0.44274234771728516, "memory/device_reserved (GiB)": 41.14, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.55697, "step": 92, "tokens/total": 3134736, "tokens/train_per_sec_per_gpu": 50.12, "tokens/trainable": 62179} +{"epoch": 0.36328125, "grad_norm": 1.3554730415344238, "learning_rate": 9.586188413468492e-05, "loss": 0.43122977018356323, "memory/device_reserved (GiB)": 41.14, "memory/max_active (GiB)": 38.66, "memory/max_allocated (GiB)": 38.66, "ppl": 1.53915, "step": 93, "tokens/total": 3172704, "tokens/train_per_sec_per_gpu": 32.27, "tokens/trainable": 62869} +{"epoch": 0.3671875, "grad_norm": 1.9785525798797607, "learning_rate": 9.57394378802934e-05, "loss": 0.508709192276001, "memory/device_reserved (GiB)": 41.14, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 1.66314, "step": 94, "tokens/total": 3205824, "tokens/train_per_sec_per_gpu": 38.53, "tokens/trainable": 63518} +{"epoch": 0.37109375, "grad_norm": 2.6837127208709717, "learning_rate": 9.56152962916e-05, "loss": 0.4533340334892273, "memory/device_reserved (GiB)": 41.14, "memory/max_active (GiB)": 36.34, "memory/max_allocated (GiB)": 36.34, "ppl": 1.57355, "step": 95, "tokens/total": 3236336, "tokens/train_per_sec_per_gpu": 43.4, "tokens/trainable": 64159} +{"epoch": 0.375, "grad_norm": 1.3239153623580933, "learning_rate": 9.548946453464296e-05, "loss": 0.4096551835536957, "memory/device_reserved (GiB)": 41.14, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.5063, "step": 96, "tokens/total": 3270288, "tokens/train_per_sec_per_gpu": 40.09, "tokens/trainable": 64873} +{"epoch": 0.37890625, "grad_norm": 2.9837560653686523, "learning_rate": 9.53619478457953e-05, "loss": 0.5007162690162659, "memory/device_reserved (GiB)": 41.14, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.6499, "step": 97, "tokens/total": 3304544, "tokens/train_per_sec_per_gpu": 33.9, "tokens/trainable": 65541} +{"epoch": 0.3828125, "grad_norm": 2.162999391555786, "learning_rate": 9.523275153154695e-05, "loss": 0.45839041471481323, "memory/device_reserved (GiB)": 41.14, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.58153, "step": 98, "tokens/total": 3336704, "tokens/train_per_sec_per_gpu": 39.42, "tokens/trainable": 66181} +{"epoch": 0.38671875, "grad_norm": 2.177814245223999, "learning_rate": 9.51018809682839e-05, "loss": 0.4229579269886017, "memory/device_reserved (GiB)": 41.14, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.52647, "step": 99, "tokens/total": 3374176, "tokens/train_per_sec_per_gpu": 33.43, "tokens/trainable": 66839} +{"epoch": 0.390625, "grad_norm": 3.207568883895874, "learning_rate": 9.49693416020645e-05, "loss": 0.5362460017204285, "memory/device_reserved (GiB)": 41.14, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.70958, "step": 100, "tokens/total": 3411648, "tokens/train_per_sec_per_gpu": 34.86, "tokens/trainable": 67496} +{"epoch": 0.39453125, "grad_norm": 2.8398430347442627, "learning_rate": 9.483513894839276e-05, "loss": 0.5386306047439575, "memory/device_reserved (GiB)": 41.14, "memory/max_active (GiB)": 36.9, "memory/max_allocated (GiB)": 36.9, "ppl": 1.71366, "step": 101, "tokens/total": 3444752, "tokens/train_per_sec_per_gpu": 42.88, "tokens/trainable": 68195} +{"epoch": 0.3984375, "grad_norm": 2.0191190242767334, "learning_rate": 9.469927859198888e-05, "loss": 0.4432718753814697, "memory/device_reserved (GiB)": 41.14, "memory/max_active (GiB)": 38.66, "memory/max_allocated (GiB)": 38.66, "ppl": 1.5578, "step": 102, "tokens/total": 3480272, "tokens/train_per_sec_per_gpu": 40.01, "tokens/trainable": 68853} +{"epoch": 0.40234375, "grad_norm": 7.384231090545654, "learning_rate": 9.456176618655689e-05, "loss": 0.41535842418670654, "memory/device_reserved (GiB)": 41.14, "memory/max_active (GiB)": 36.36, "memory/max_allocated (GiB)": 36.36, "ppl": 1.51491, "step": 103, "tokens/total": 3511888, "tokens/train_per_sec_per_gpu": 39.98, "tokens/trainable": 69491} +{"epoch": 0.40625, "grad_norm": 7.523770809173584, "learning_rate": 9.442260745454927e-05, "loss": 0.4509773254394531, "memory/device_reserved (GiB)": 41.14, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.56985, "step": 104, "tokens/total": 3546768, "tokens/train_per_sec_per_gpu": 36.99, "tokens/trainable": 70175} +{"epoch": 0.41015625, "grad_norm": 1.8080430030822754, "learning_rate": 9.428180818692884e-05, "loss": 0.3783518075942993, "memory/device_reserved (GiB)": 41.14, "memory/max_active (GiB)": 38.45, "memory/max_allocated (GiB)": 38.45, "ppl": 1.45988, "step": 105, "tokens/total": 3582656, "tokens/train_per_sec_per_gpu": 34.01, "tokens/trainable": 70868} +{"epoch": 0.4140625, "grad_norm": 2.164571523666382, "learning_rate": 9.413937424292791e-05, "loss": 0.41767510771751404, "memory/device_reserved (GiB)": 41.28, "memory/max_active (GiB)": 39.11, "memory/max_allocated (GiB)": 39.11, "ppl": 1.51843, "step": 106, "tokens/total": 3621296, "tokens/train_per_sec_per_gpu": 31.59, "tokens/trainable": 71531} +{"epoch": 0.41796875, "grad_norm": 2.276919364929199, "learning_rate": 9.399531154980424e-05, "loss": 0.34076622128486633, "memory/device_reserved (GiB)": 41.28, "memory/max_active (GiB)": 38.22, "memory/max_allocated (GiB)": 38.22, "ppl": 1.40602, "step": 107, "tokens/total": 3657568, "tokens/train_per_sec_per_gpu": 34.5, "tokens/trainable": 72196} +{"epoch": 0.421875, "grad_norm": 2.5344011783599854, "learning_rate": 9.384962610259455e-05, "loss": 0.41366514563560486, "memory/device_reserved (GiB)": 41.28, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.51235, "step": 108, "tokens/total": 3691632, "tokens/train_per_sec_per_gpu": 35.48, "tokens/trainable": 72833} +{"epoch": 0.42578125, "grad_norm": 1.972804307937622, "learning_rate": 9.370232396386494e-05, "loss": 0.44345617294311523, "memory/device_reserved (GiB)": 41.28, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.55808, "step": 109, "tokens/total": 3724528, "tokens/train_per_sec_per_gpu": 41.01, "tokens/trainable": 73498} +{"epoch": 0.4296875, "grad_norm": 2.160320281982422, "learning_rate": 9.355341126345868e-05, "loss": 0.393061101436615, "memory/device_reserved (GiB)": 41.28, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.48151, "step": 110, "tokens/total": 3759456, "tokens/train_per_sec_per_gpu": 38.83, "tokens/trainable": 74180} +{"epoch": 0.43359375, "grad_norm": 1.9818308353424072, "learning_rate": 9.340289419824107e-05, "loss": 0.39798688888549805, "memory/device_reserved (GiB)": 41.28, "memory/max_active (GiB)": 36.6, "memory/max_allocated (GiB)": 36.6, "ppl": 1.48882, "step": 111, "tokens/total": 3791360, "tokens/train_per_sec_per_gpu": 44.68, "tokens/trainable": 74826} +{"epoch": 0.4375, "grad_norm": 2.065756320953369, "learning_rate": 9.325077903184159e-05, "loss": 0.41352158784866333, "memory/device_reserved (GiB)": 41.28, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.51213, "step": 112, "tokens/total": 3825808, "tokens/train_per_sec_per_gpu": 34.64, "tokens/trainable": 75526} +{"epoch": 0.44140625, "grad_norm": 1.5179312229156494, "learning_rate": 9.30970720943932e-05, "loss": 0.37356430292129517, "memory/device_reserved (GiB)": 41.28, "memory/max_active (GiB)": 38.29, "memory/max_allocated (GiB)": 38.29, "ppl": 1.4529, "step": 113, "tokens/total": 3860816, "tokens/train_per_sec_per_gpu": 38.01, "tokens/trainable": 76218} +{"epoch": 0.4453125, "grad_norm": 2.2145068645477295, "learning_rate": 9.2941779782269e-05, "loss": 0.40434905886650085, "memory/device_reserved (GiB)": 41.28, "memory/max_active (GiB)": 37.12, "memory/max_allocated (GiB)": 37.12, "ppl": 1.49833, "step": 114, "tokens/total": 3894048, "tokens/train_per_sec_per_gpu": 35.59, "tokens/trainable": 76832} +{"epoch": 0.44921875, "grad_norm": 2.6525118350982666, "learning_rate": 9.278490855781596e-05, "loss": 0.42529863119125366, "memory/device_reserved (GiB)": 41.28, "memory/max_active (GiB)": 37.52, "memory/max_allocated (GiB)": 37.52, "ppl": 1.53005, "step": 115, "tokens/total": 3926304, "tokens/train_per_sec_per_gpu": 46.9, "tokens/trainable": 77487} +{"epoch": 0.453125, "grad_norm": 2.4885478019714355, "learning_rate": 9.262646494908604e-05, "loss": 0.4515993297100067, "memory/device_reserved (GiB)": 41.28, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.57082, "step": 116, "tokens/total": 3964576, "tokens/train_per_sec_per_gpu": 34.18, "tokens/trainable": 78137} +{"epoch": 0.45703125, "grad_norm": 2.5020334720611572, "learning_rate": 9.246645554956457e-05, "loss": 0.42529332637786865, "memory/device_reserved (GiB)": 41.28, "memory/max_active (GiB)": 38.53, "memory/max_allocated (GiB)": 38.53, "ppl": 1.53004, "step": 117, "tokens/total": 3998896, "tokens/train_per_sec_per_gpu": 41.07, "tokens/trainable": 78812} +{"epoch": 0.4609375, "grad_norm": 2.276121139526367, "learning_rate": 9.230488701789578e-05, "loss": 0.48477548360824585, "memory/device_reserved (GiB)": 41.28, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.62381, "step": 118, "tokens/total": 4035856, "tokens/train_per_sec_per_gpu": 36.82, "tokens/trainable": 79525} +{"epoch": 0.46484375, "grad_norm": 1.877341628074646, "learning_rate": 9.214176607760577e-05, "loss": 0.3757883906364441, "memory/device_reserved (GiB)": 41.28, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.45614, "step": 119, "tokens/total": 4074080, "tokens/train_per_sec_per_gpu": 39.23, "tokens/trainable": 80244} +{"epoch": 0.46875, "grad_norm": 2.894897699356079, "learning_rate": 9.197709951682268e-05, "loss": 0.4226961135864258, "memory/device_reserved (GiB)": 41.28, "memory/max_active (GiB)": 36.88, "memory/max_allocated (GiB)": 36.88, "ppl": 1.52607, "step": 120, "tokens/total": 4106048, "tokens/train_per_sec_per_gpu": 39.56, "tokens/trainable": 80889} +{"epoch": 0.47265625, "grad_norm": 1.8957830667495728, "learning_rate": 9.181089418799428e-05, "loss": 0.3881450295448303, "memory/device_reserved (GiB)": 41.28, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.47424, "step": 121, "tokens/total": 4142544, "tokens/train_per_sec_per_gpu": 37.97, "tokens/trainable": 81584} +{"epoch": 0.4765625, "grad_norm": 1.7596583366394043, "learning_rate": 9.164315700760271e-05, "loss": 0.3230040967464447, "memory/device_reserved (GiB)": 41.28, "memory/max_active (GiB)": 38.72, "memory/max_allocated (GiB)": 38.72, "ppl": 1.38127, "step": 122, "tokens/total": 4178032, "tokens/train_per_sec_per_gpu": 35.05, "tokens/trainable": 82259} +{"epoch": 0.48046875, "grad_norm": 1.9092216491699219, "learning_rate": 9.147389495587671e-05, "loss": 0.461895227432251, "memory/device_reserved (GiB)": 41.28, "memory/max_active (GiB)": 36.88, "memory/max_allocated (GiB)": 36.88, "ppl": 1.58708, "step": 123, "tokens/total": 4208880, "tokens/train_per_sec_per_gpu": 45.75, "tokens/trainable": 82912} +{"epoch": 0.484375, "grad_norm": 1.5045408010482788, "learning_rate": 9.130311507650116e-05, "loss": 0.29049623012542725, "memory/device_reserved (GiB)": 41.28, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.33709, "step": 124, "tokens/total": 4241072, "tokens/train_per_sec_per_gpu": 37.68, "tokens/trainable": 83579} +{"epoch": 0.48828125, "grad_norm": 1.6636009216308594, "learning_rate": 9.113082447632394e-05, "loss": 0.39182427525520325, "memory/device_reserved (GiB)": 41.28, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.47968, "step": 125, "tokens/total": 4272720, "tokens/train_per_sec_per_gpu": 51.5, "tokens/trainable": 84281} +{"epoch": 0.4921875, "grad_norm": 2.4882149696350098, "learning_rate": 9.09570303250602e-05, "loss": 0.3971545398235321, "memory/device_reserved (GiB)": 41.28, "memory/max_active (GiB)": 36.3, "memory/max_allocated (GiB)": 36.3, "ppl": 1.48759, "step": 126, "tokens/total": 4304176, "tokens/train_per_sec_per_gpu": 42.44, "tokens/trainable": 84986} +{"epoch": 0.49609375, "grad_norm": 4.259303092956543, "learning_rate": 9.078173985499394e-05, "loss": 0.33799952268600464, "memory/device_reserved (GiB)": 41.29, "memory/max_active (GiB)": 35.67, "memory/max_allocated (GiB)": 35.67, "ppl": 1.40214, "step": 127, "tokens/total": 4333872, "tokens/train_per_sec_per_gpu": 40.1, "tokens/trainable": 85624} +{"epoch": 0.5, "grad_norm": 2.492567777633667, "learning_rate": 9.060496036067713e-05, "loss": 0.41761577129364014, "memory/device_reserved (GiB)": 41.29, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.51834, "step": 128, "tokens/total": 4370960, "tokens/train_per_sec_per_gpu": 30.71, "tokens/trainable": 86282} +{"epoch": 0.50390625, "grad_norm": 1.9712350368499756, "learning_rate": 9.042669919862615e-05, "loss": 0.35205596685409546, "memory/device_reserved (GiB)": 41.29, "memory/max_active (GiB)": 36.73, "memory/max_allocated (GiB)": 36.73, "ppl": 1.42199, "step": 129, "tokens/total": 4402816, "tokens/train_per_sec_per_gpu": 47.01, "tokens/trainable": 86993} +{"epoch": 0.5078125, "grad_norm": 1.8673861026763916, "learning_rate": 9.024696378701557e-05, "loss": 0.32654374837875366, "memory/device_reserved (GiB)": 40.56, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.38617, "step": 130, "tokens/total": 4437952, "tokens/train_per_sec_per_gpu": 34.94, "tokens/trainable": 87659} +{"epoch": 0.51171875, "grad_norm": 1.8381456136703491, "learning_rate": 9.006576160536948e-05, "loss": 0.3530905842781067, "memory/device_reserved (GiB)": 40.56, "memory/max_active (GiB)": 37.31, "memory/max_allocated (GiB)": 37.31, "ppl": 1.42346, "step": 131, "tokens/total": 4472416, "tokens/train_per_sec_per_gpu": 34.3, "tokens/trainable": 88335} +{"epoch": 0.515625, "grad_norm": 2.3133533000946045, "learning_rate": 8.988310019425035e-05, "loss": 0.40116792917251587, "memory/device_reserved (GiB)": 43.8, "memory/max_active (GiB)": 39.53, "memory/max_allocated (GiB)": 39.53, "ppl": 1.49357, "step": 132, "tokens/total": 4509904, "tokens/train_per_sec_per_gpu": 33.29, "tokens/trainable": 88965} +{"epoch": 0.51953125, "grad_norm": 1.9567272663116455, "learning_rate": 8.969898715494506e-05, "loss": 0.33467966318130493, "memory/device_reserved (GiB)": 43.8, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.39749, "step": 133, "tokens/total": 4546320, "tokens/train_per_sec_per_gpu": 40.67, "tokens/trainable": 89682} +{"epoch": 0.5234375, "grad_norm": 1.9866101741790771, "learning_rate": 8.951343014914869e-05, "loss": 0.307401180267334, "memory/device_reserved (GiB)": 43.8, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.35989, "step": 134, "tokens/total": 4582560, "tokens/train_per_sec_per_gpu": 35.14, "tokens/trainable": 90378} +{"epoch": 0.52734375, "grad_norm": 1.9490200281143188, "learning_rate": 8.932643689864568e-05, "loss": 0.35038334131240845, "memory/device_reserved (GiB)": 43.8, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.41961, "step": 135, "tokens/total": 4618688, "tokens/train_per_sec_per_gpu": 39.65, "tokens/trainable": 91078} +{"epoch": 0.53125, "grad_norm": 1.3267695903778076, "learning_rate": 8.913801518498845e-05, "loss": 0.29040950536727905, "memory/device_reserved (GiB)": 43.8, "memory/max_active (GiB)": 39.54, "memory/max_allocated (GiB)": 39.54, "ppl": 1.33697, "step": 136, "tokens/total": 4657680, "tokens/train_per_sec_per_gpu": 33.04, "tokens/trainable": 91821} +{"epoch": 0.53515625, "grad_norm": 1.9533144235610962, "learning_rate": 8.894817284917364e-05, "loss": 0.37240058183670044, "memory/device_reserved (GiB)": 43.8, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.45121, "step": 137, "tokens/total": 4694304, "tokens/train_per_sec_per_gpu": 34.18, "tokens/trainable": 92515} +{"epoch": 0.5390625, "grad_norm": 1.5486165285110474, "learning_rate": 8.875691779131569e-05, "loss": 0.3521190881729126, "memory/device_reserved (GiB)": 43.8, "memory/max_active (GiB)": 39.61, "memory/max_allocated (GiB)": 39.61, "ppl": 1.42208, "step": 138, "tokens/total": 4731792, "tokens/train_per_sec_per_gpu": 33.85, "tokens/trainable": 93184} +{"epoch": 0.54296875, "grad_norm": 1.333570957183838, "learning_rate": 8.856425797031829e-05, "loss": 0.3036693334579468, "memory/device_reserved (GiB)": 43.8, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.35482, "step": 139, "tokens/total": 4767408, "tokens/train_per_sec_per_gpu": 42.7, "tokens/trainable": 93897} +{"epoch": 0.546875, "grad_norm": 1.5286270380020142, "learning_rate": 8.837020140354295e-05, "loss": 0.3027605414390564, "memory/device_reserved (GiB)": 43.8, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.35359, "step": 140, "tokens/total": 4803808, "tokens/train_per_sec_per_gpu": 35.31, "tokens/trainable": 94538} +{"epoch": 0.55078125, "grad_norm": 2.7021570205688477, "learning_rate": 8.817475616647554e-05, "loss": 0.3224470615386963, "memory/device_reserved (GiB)": 43.81, "memory/max_active (GiB)": 38.0, "memory/max_allocated (GiB)": 38.0, "ppl": 1.3805, "step": 141, "tokens/total": 4837840, "tokens/train_per_sec_per_gpu": 40.02, "tokens/trainable": 95226} +{"epoch": 0.5546875, "grad_norm": 2.122523784637451, "learning_rate": 8.797793039239017e-05, "loss": 0.3444788157939911, "memory/device_reserved (GiB)": 43.81, "memory/max_active (GiB)": 37.73, "memory/max_allocated (GiB)": 37.73, "ppl": 1.41125, "step": 142, "tokens/total": 4872560, "tokens/train_per_sec_per_gpu": 40.09, "tokens/trainable": 95912} +{"epoch": 0.55859375, "grad_norm": 1.9983859062194824, "learning_rate": 8.777973227201069e-05, "loss": 0.29782310128211975, "memory/device_reserved (GiB)": 43.81, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.34692, "step": 143, "tokens/total": 4909216, "tokens/train_per_sec_per_gpu": 43.23, "tokens/trainable": 96650} +{"epoch": 0.5625, "grad_norm": 2.4572696685791016, "learning_rate": 8.758017005316988e-05, "loss": 0.35157468914985657, "memory/device_reserved (GiB)": 43.81, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.4213, "step": 144, "tokens/total": 4944432, "tokens/train_per_sec_per_gpu": 44.58, "tokens/trainable": 97325} +{"epoch": 0.56640625, "grad_norm": 4.137853145599365, "learning_rate": 8.737925204046629e-05, "loss": 0.3939829468727112, "memory/device_reserved (GiB)": 43.81, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.48288, "step": 145, "tokens/total": 4978800, "tokens/train_per_sec_per_gpu": 33.34, "tokens/trainable": 98011} +{"epoch": 0.5703125, "grad_norm": 2.2435147762298584, "learning_rate": 8.717698659491851e-05, "loss": 0.3185906708240509, "memory/device_reserved (GiB)": 43.81, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.37519, "step": 146, "tokens/total": 5015328, "tokens/train_per_sec_per_gpu": 35.78, "tokens/trainable": 98734} +{"epoch": 0.57421875, "grad_norm": 1.8117793798446655, "learning_rate": 8.697338213361735e-05, "loss": 0.29981184005737305, "memory/device_reserved (GiB)": 43.81, "memory/max_active (GiB)": 38.81, "memory/max_allocated (GiB)": 38.81, "ppl": 1.3496, "step": 147, "tokens/total": 5050976, "tokens/train_per_sec_per_gpu": 36.83, "tokens/trainable": 99466} +{"epoch": 0.578125, "grad_norm": 1.6404918432235718, "learning_rate": 8.676844712937552e-05, "loss": 0.27588754892349243, "memory/device_reserved (GiB)": 43.81, "memory/max_active (GiB)": 39.18, "memory/max_allocated (GiB)": 39.18, "ppl": 1.3177, "step": 148, "tokens/total": 5090592, "tokens/train_per_sec_per_gpu": 33.76, "tokens/trainable": 100188} +{"epoch": 0.58203125, "grad_norm": 2.4379847049713135, "learning_rate": 8.656219011037509e-05, "loss": 0.3037242293357849, "memory/device_reserved (GiB)": 43.81, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.3549, "step": 149, "tokens/total": 5127328, "tokens/train_per_sec_per_gpu": 34.52, "tokens/trainable": 100869} +{"epoch": 0.5859375, "grad_norm": 2.3394806385040283, "learning_rate": 8.63546196598125e-05, "loss": 0.3354702591896057, "memory/device_reserved (GiB)": 43.81, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.3986, "step": 150, "tokens/total": 5164528, "tokens/train_per_sec_per_gpu": 34.56, "tokens/trainable": 101598} +{"epoch": 0.58984375, "grad_norm": 2.1649727821350098, "learning_rate": 8.614574441554145e-05, "loss": 0.42634567618370056, "memory/device_reserved (GiB)": 43.81, "memory/max_active (GiB)": 36.88, "memory/max_allocated (GiB)": 36.88, "ppl": 1.53165, "step": 151, "tokens/total": 5196768, "tokens/train_per_sec_per_gpu": 41.66, "tokens/trainable": 102226} +{"epoch": 0.59375, "grad_norm": 1.9619733095169067, "learning_rate": 8.593557306971349e-05, "loss": 0.2993645966053009, "memory/device_reserved (GiB)": 43.81, "memory/max_active (GiB)": 37.58, "memory/max_allocated (GiB)": 37.58, "ppl": 1.349, "step": 152, "tokens/total": 5229632, "tokens/train_per_sec_per_gpu": 40.94, "tokens/trainable": 102916} +{"epoch": 0.59765625, "grad_norm": 3.6510276794433594, "learning_rate": 8.572411436841618e-05, "loss": 0.3562014698982239, "memory/device_reserved (GiB)": 43.81, "memory/max_active (GiB)": 36.73, "memory/max_allocated (GiB)": 36.73, "ppl": 1.4279, "step": 153, "tokens/total": 5262960, "tokens/train_per_sec_per_gpu": 40.38, "tokens/trainable": 103604} +{"epoch": 0.6015625, "grad_norm": 2.423145294189453, "learning_rate": 8.551137711130922e-05, "loss": 0.32388192415237427, "memory/device_reserved (GiB)": 43.81, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 1.38248, "step": 154, "tokens/total": 5294384, "tokens/train_per_sec_per_gpu": 35.85, "tokens/trainable": 104255} +{"epoch": 0.60546875, "grad_norm": 3.0627686977386475, "learning_rate": 8.529737015125824e-05, "loss": 0.2816256284713745, "memory/device_reserved (GiB)": 43.81, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.32528, "step": 155, "tokens/total": 5329072, "tokens/train_per_sec_per_gpu": 38.23, "tokens/trainable": 104943} +{"epoch": 0.609375, "grad_norm": 1.485774278640747, "learning_rate": 8.508210239396639e-05, "loss": 0.3598991632461548, "memory/device_reserved (GiB)": 43.81, "memory/max_active (GiB)": 39.58, "memory/max_allocated (GiB)": 39.58, "ppl": 1.43318, "step": 156, "tokens/total": 5366448, "tokens/train_per_sec_per_gpu": 37.25, "tokens/trainable": 105604} +{"epoch": 0.61328125, "grad_norm": 2.5207202434539795, "learning_rate": 8.486558279760375e-05, "loss": 0.35510262846946716, "memory/device_reserved (GiB)": 43.81, "memory/max_active (GiB)": 35.66, "memory/max_allocated (GiB)": 35.66, "ppl": 1.42633, "step": 157, "tokens/total": 5396864, "tokens/train_per_sec_per_gpu": 42.05, "tokens/trainable": 106243} +{"epoch": 0.6171875, "grad_norm": 1.6720706224441528, "learning_rate": 8.464782037243449e-05, "loss": 0.3448483943939209, "memory/device_reserved (GiB)": 43.81, "memory/max_active (GiB)": 36.7, "memory/max_allocated (GiB)": 36.7, "ppl": 1.41178, "step": 158, "tokens/total": 5428640, "tokens/train_per_sec_per_gpu": 39.36, "tokens/trainable": 106890} +{"epoch": 0.62109375, "grad_norm": 1.7704908847808838, "learning_rate": 8.442882418044202e-05, "loss": 0.2966160178184509, "memory/device_reserved (GiB)": 43.81, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.3453, "step": 159, "tokens/total": 5460896, "tokens/train_per_sec_per_gpu": 44.29, "tokens/trainable": 107589} +{"epoch": 0.625, "grad_norm": 2.683417558670044, "learning_rate": 8.420860333495179e-05, "loss": 0.2613505721092224, "memory/device_reserved (GiB)": 43.81, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.29868, "step": 160, "tokens/total": 5494288, "tokens/train_per_sec_per_gpu": 38.0, "tokens/trainable": 108207} +{"epoch": 0.62890625, "grad_norm": 2.262927532196045, "learning_rate": 8.398716700025208e-05, "loss": 0.31105664372444153, "memory/device_reserved (GiB)": 43.81, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.36487, "step": 161, "tokens/total": 5527296, "tokens/train_per_sec_per_gpu": 36.46, "tokens/trainable": 108842} +{"epoch": 0.6328125, "grad_norm": 8.220526695251465, "learning_rate": 8.376452439121266e-05, "loss": 0.29269829392433167, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 35.82, "memory/max_allocated (GiB)": 35.82, "ppl": 1.34004, "step": 162, "tokens/total": 5557344, "tokens/train_per_sec_per_gpu": 43.64, "tokens/trainable": 109467} +{"epoch": 0.63671875, "grad_norm": 3.7608141899108887, "learning_rate": 8.354068477290124e-05, "loss": 0.32695528864860535, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.38674, "step": 163, "tokens/total": 5591184, "tokens/train_per_sec_per_gpu": 43.28, "tokens/trainable": 110123} +{"epoch": 0.640625, "grad_norm": 3.4529495239257812, "learning_rate": 8.331565746019807e-05, "loss": 0.31483733654022217, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 36.59, "memory/max_allocated (GiB)": 36.59, "ppl": 1.37004, "step": 164, "tokens/total": 5622848, "tokens/train_per_sec_per_gpu": 44.94, "tokens/trainable": 110793} +{"epoch": 0.64453125, "grad_norm": 3.200725555419922, "learning_rate": 8.308945181740812e-05, "loss": 0.32734763622283936, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 39.15, "memory/max_allocated (GiB)": 39.15, "ppl": 1.38728, "step": 165, "tokens/total": 5657136, "tokens/train_per_sec_per_gpu": 41.38, "tokens/trainable": 111447} +{"epoch": 0.6484375, "grad_norm": 8.380316734313965, "learning_rate": 8.286207725787153e-05, "loss": 0.265841007232666, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.30453, "step": 166, "tokens/total": 5693760, "tokens/train_per_sec_per_gpu": 35.25, "tokens/trainable": 112112} +{"epoch": 0.65234375, "grad_norm": 3.7850911617279053, "learning_rate": 8.263354324357182e-05, "loss": 0.275675505399704, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.31742, "step": 167, "tokens/total": 5728144, "tokens/train_per_sec_per_gpu": 39.79, "tokens/trainable": 112791} +{"epoch": 0.65625, "grad_norm": 2.691239356994629, "learning_rate": 8.240385928474219e-05, "loss": 0.3537275791168213, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 36.71, "memory/max_allocated (GiB)": 36.71, "ppl": 1.42437, "step": 168, "tokens/total": 5760624, "tokens/train_per_sec_per_gpu": 37.52, "tokens/trainable": 113392} +{"epoch": 0.66015625, "grad_norm": 1.8672335147857666, "learning_rate": 8.217303493946967e-05, "loss": 0.2755075693130493, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 37.13, "memory/max_allocated (GiB)": 37.13, "ppl": 1.3172, "step": 169, "tokens/total": 5794448, "tokens/train_per_sec_per_gpu": 37.25, "tokens/trainable": 114058} +{"epoch": 0.6640625, "grad_norm": 4.528531074523926, "learning_rate": 8.194107981329746e-05, "loss": 0.275892436504364, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 39.59, "memory/max_allocated (GiB)": 39.59, "ppl": 1.31771, "step": 170, "tokens/total": 5832608, "tokens/train_per_sec_per_gpu": 34.77, "tokens/trainable": 114736} +{"epoch": 0.66796875, "grad_norm": 2.3813118934631348, "learning_rate": 8.170800355882518e-05, "loss": 0.259056955575943, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.29571, "step": 171, "tokens/total": 5866672, "tokens/train_per_sec_per_gpu": 47.52, "tokens/trainable": 115462} +{"epoch": 0.671875, "grad_norm": 3.2815005779266357, "learning_rate": 8.147381587530713e-05, "loss": 0.36197319626808167, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.43616, "step": 172, "tokens/total": 5900176, "tokens/train_per_sec_per_gpu": 42.48, "tokens/trainable": 116140} +{"epoch": 0.67578125, "grad_norm": 2.3086183071136475, "learning_rate": 8.123852650824877e-05, "loss": 0.2812424600124359, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.32477, "step": 173, "tokens/total": 5933712, "tokens/train_per_sec_per_gpu": 41.99, "tokens/trainable": 116858} +{"epoch": 0.6796875, "grad_norm": 2.336418390274048, "learning_rate": 8.100214524900103e-05, "loss": 0.28806811571121216, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 1.33385, "step": 174, "tokens/total": 5968480, "tokens/train_per_sec_per_gpu": 32.49, "tokens/trainable": 117493} +{"epoch": 0.68359375, "grad_norm": 3.1748480796813965, "learning_rate": 8.076468193435301e-05, "loss": 0.3271517753601074, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 36.25, "memory/max_allocated (GiB)": 36.25, "ppl": 1.38701, "step": 175, "tokens/total": 5998864, "tokens/train_per_sec_per_gpu": 41.19, "tokens/trainable": 118077} +{"epoch": 0.6875, "grad_norm": 3.894700527191162, "learning_rate": 8.052614644612253e-05, "loss": 0.30319148302078247, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.35417, "step": 176, "tokens/total": 6036384, "tokens/train_per_sec_per_gpu": 32.33, "tokens/trainable": 118768} +{"epoch": 0.69140625, "grad_norm": 2.4097657203674316, "learning_rate": 8.028654871074489e-05, "loss": 0.24785153567790985, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.28127, "step": 177, "tokens/total": 6070464, "tokens/train_per_sec_per_gpu": 40.27, "tokens/trainable": 119486} +{"epoch": 0.6953125, "grad_norm": 1.7803276777267456, "learning_rate": 8.004589869885986e-05, "loss": 0.29934439063072205, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 36.3, "memory/max_allocated (GiB)": 36.3, "ppl": 1.34897, "step": 178, "tokens/total": 6101760, "tokens/train_per_sec_per_gpu": 37.6, "tokens/trainable": 120077} +{"epoch": 0.69921875, "grad_norm": 1.4392647743225098, "learning_rate": 7.980420642489674e-05, "loss": 0.2721613943576813, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.3128, "step": 179, "tokens/total": 6140144, "tokens/train_per_sec_per_gpu": 34.85, "tokens/trainable": 120765} +{"epoch": 0.703125, "grad_norm": 1.736770510673523, "learning_rate": 7.95614819466576e-05, "loss": 0.24030590057373047, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 37.48, "memory/max_allocated (GiB)": 37.48, "ppl": 1.27164, "step": 180, "tokens/total": 6173488, "tokens/train_per_sec_per_gpu": 37.77, "tokens/trainable": 121435} +{"epoch": 0.70703125, "grad_norm": 3.1014974117279053, "learning_rate": 7.931773536489872e-05, "loss": 0.27695217728614807, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 37.41, "memory/max_allocated (GiB)": 37.41, "ppl": 1.3191, "step": 181, "tokens/total": 6207760, "tokens/train_per_sec_per_gpu": 41.03, "tokens/trainable": 122104} +{"epoch": 0.7109375, "grad_norm": 1.4985867738723755, "learning_rate": 7.907297682291035e-05, "loss": 0.22434088587760925, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.2515, "step": 182, "tokens/total": 6242144, "tokens/train_per_sec_per_gpu": 42.38, "tokens/trainable": 122827} +{"epoch": 0.71484375, "grad_norm": 3.4196717739105225, "learning_rate": 7.882721650609442e-05, "loss": 0.1887410283088684, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 39.48, "memory/max_allocated (GiB)": 39.48, "ppl": 1.20773, "step": 183, "tokens/total": 6282208, "tokens/train_per_sec_per_gpu": 37.04, "tokens/trainable": 123633} +{"epoch": 0.71875, "grad_norm": 3.2741811275482178, "learning_rate": 7.85804646415409e-05, "loss": 0.2712404131889343, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.31159, "step": 184, "tokens/total": 6316960, "tokens/train_per_sec_per_gpu": 40.55, "tokens/trainable": 124301} +{"epoch": 0.72265625, "grad_norm": 3.5058794021606445, "learning_rate": 7.833273149760207e-05, "loss": 0.22999826073646545, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 1.2586, "step": 185, "tokens/total": 6347248, "tokens/train_per_sec_per_gpu": 41.34, "tokens/trainable": 124987} +{"epoch": 0.7265625, "grad_norm": 2.9251527786254883, "learning_rate": 7.808402738346527e-05, "loss": 0.25516557693481445, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.29068, "step": 186, "tokens/total": 6385728, "tokens/train_per_sec_per_gpu": 40.39, "tokens/trainable": 125743} +{"epoch": 0.73046875, "grad_norm": 13.972705841064453, "learning_rate": 7.783436264872382e-05, "loss": 0.22899501025676727, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 38.93, "memory/max_allocated (GiB)": 38.93, "ppl": 1.25734, "step": 187, "tokens/total": 6422144, "tokens/train_per_sec_per_gpu": 39.11, "tokens/trainable": 126460} +{"epoch": 0.734375, "grad_norm": 3.0893101692199707, "learning_rate": 7.758374768294647e-05, "loss": 0.22317178547382355, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 38.08, "memory/max_allocated (GiB)": 38.08, "ppl": 1.25004, "step": 188, "tokens/total": 6457840, "tokens/train_per_sec_per_gpu": 42.66, "tokens/trainable": 127203} +{"epoch": 0.73828125, "grad_norm": 91.03417205810547, "learning_rate": 7.733219291524489e-05, "loss": 0.23726144433021545, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 37.39, "memory/max_allocated (GiB)": 37.39, "ppl": 1.26777, "step": 189, "tokens/total": 6490944, "tokens/train_per_sec_per_gpu": 42.64, "tokens/trainable": 127897} +{"epoch": 0.7421875, "grad_norm": 2.7044568061828613, "learning_rate": 7.707970881383977e-05, "loss": 0.22241145372390747, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 36.32, "memory/max_allocated (GiB)": 36.32, "ppl": 1.24909, "step": 190, "tokens/total": 6522352, "tokens/train_per_sec_per_gpu": 47.69, "tokens/trainable": 128589} +{"epoch": 0.74609375, "grad_norm": 3.2348921298980713, "learning_rate": 7.682630588562518e-05, "loss": 0.2369498759508133, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.26738, "step": 191, "tokens/total": 6557056, "tokens/train_per_sec_per_gpu": 38.57, "tokens/trainable": 129283} +{"epoch": 0.75, "grad_norm": 4.557906627655029, "learning_rate": 7.657199467573129e-05, "loss": 0.24416325986385345, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 36.93, "memory/max_allocated (GiB)": 36.93, "ppl": 1.27655, "step": 192, "tokens/total": 6590704, "tokens/train_per_sec_per_gpu": 37.92, "tokens/trainable": 129953} +{"epoch": 0.75390625, "grad_norm": 2.82560133934021, "learning_rate": 7.631678576708561e-05, "loss": 0.20013421773910522, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.22157, "step": 193, "tokens/total": 6624752, "tokens/train_per_sec_per_gpu": 45.32, "tokens/trainable": 130704} +{"epoch": 0.7578125, "grad_norm": 2.189274787902832, "learning_rate": 7.606068977997255e-05, "loss": 0.2197115272283554, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 39.54, "memory/max_allocated (GiB)": 39.54, "ppl": 1.24572, "step": 194, "tokens/total": 6662096, "tokens/train_per_sec_per_gpu": 45.03, "tokens/trainable": 131476} +{"epoch": 0.76171875, "grad_norm": 8.766033172607422, "learning_rate": 7.580371737159148e-05, "loss": 0.3677166998386383, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.44443, "step": 195, "tokens/total": 6697808, "tokens/train_per_sec_per_gpu": 34.21, "tokens/trainable": 132117} +{"epoch": 0.765625, "grad_norm": 2.9890174865722656, "learning_rate": 7.554587923561324e-05, "loss": 0.21674498915672302, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.24203, "step": 196, "tokens/total": 6732112, "tokens/train_per_sec_per_gpu": 36.17, "tokens/trainable": 132753} +{"epoch": 0.76953125, "grad_norm": 2.3649086952209473, "learning_rate": 7.528718610173511e-05, "loss": 0.2358037829399109, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 37.53, "memory/max_allocated (GiB)": 37.53, "ppl": 1.26593, "step": 197, "tokens/total": 6767120, "tokens/train_per_sec_per_gpu": 41.3, "tokens/trainable": 133429} +{"epoch": 0.7734375, "grad_norm": 2.343604564666748, "learning_rate": 7.502764873523431e-05, "loss": 0.26697584986686707, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.30601, "step": 198, "tokens/total": 6800864, "tokens/train_per_sec_per_gpu": 42.47, "tokens/trainable": 134118} +{"epoch": 0.77734375, "grad_norm": 1.9796702861785889, "learning_rate": 7.476727793652011e-05, "loss": 0.2195294350385666, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 1.24549, "step": 199, "tokens/total": 6833840, "tokens/train_per_sec_per_gpu": 40.78, "tokens/trainable": 134807} +{"epoch": 0.78125, "grad_norm": 4.495216369628906, "learning_rate": 7.450608454068415e-05, "loss": 0.1838236302137375, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.2018, "step": 200, "tokens/total": 6867984, "tokens/train_per_sec_per_gpu": 38.94, "tokens/trainable": 135502} +{"epoch": 0.78515625, "grad_norm": 1.9619863033294678, "learning_rate": 7.424407941704987e-05, "loss": 0.19327759742736816, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 38.3, "memory/max_allocated (GiB)": 38.3, "ppl": 1.21322, "step": 201, "tokens/total": 6904192, "tokens/train_per_sec_per_gpu": 42.59, "tokens/trainable": 136224} +{"epoch": 0.7890625, "grad_norm": 1.1142191886901855, "learning_rate": 7.398127346871986e-05, "loss": 0.18258298933506012, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 39.52, "memory/max_allocated (GiB)": 39.52, "ppl": 1.20031, "step": 202, "tokens/total": 6940880, "tokens/train_per_sec_per_gpu": 37.54, "tokens/trainable": 136945} +{"epoch": 0.79296875, "grad_norm": 1.666421890258789, "learning_rate": 7.371767763212238e-05, "loss": 0.2134336531162262, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.23792, "step": 203, "tokens/total": 6975920, "tokens/train_per_sec_per_gpu": 42.2, "tokens/trainable": 137626} +{"epoch": 0.796875, "grad_norm": 7.002161979675293, "learning_rate": 7.345330287655617e-05, "loss": 0.2451939582824707, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.27787, "step": 204, "tokens/total": 7013632, "tokens/train_per_sec_per_gpu": 33.91, "tokens/trainable": 138291} +{"epoch": 0.80078125, "grad_norm": 2.359339952468872, "learning_rate": 7.31881602037339e-05, "loss": 0.2287072390317917, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 36.6, "memory/max_allocated (GiB)": 36.6, "ppl": 1.25697, "step": 205, "tokens/total": 7045312, "tokens/train_per_sec_per_gpu": 39.92, "tokens/trainable": 138957} +{"epoch": 0.8046875, "grad_norm": 1.6748021841049194, "learning_rate": 7.29222606473245e-05, "loss": 0.21374450623989105, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 37.2, "memory/max_allocated (GiB)": 37.2, "ppl": 1.23831, "step": 206, "tokens/total": 7077808, "tokens/train_per_sec_per_gpu": 38.58, "tokens/trainable": 139602} +{"epoch": 0.80859375, "grad_norm": 2.079460382461548, "learning_rate": 7.265561527249383e-05, "loss": 0.23688219487667084, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 37.07, "memory/max_allocated (GiB)": 37.07, "ppl": 1.26729, "step": 207, "tokens/total": 7111392, "tokens/train_per_sec_per_gpu": 40.92, "tokens/trainable": 140256} +{"epoch": 0.8125, "grad_norm": 2.406209945678711, "learning_rate": 7.238823517544436e-05, "loss": 0.16817645728588104, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.18315, "step": 208, "tokens/total": 7148128, "tokens/train_per_sec_per_gpu": 45.89, "tokens/trainable": 141035} +{"epoch": 0.81640625, "grad_norm": 2.866179943084717, "learning_rate": 7.212013148295333e-05, "loss": 0.2119114100933075, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.23604, "step": 209, "tokens/total": 7181760, "tokens/train_per_sec_per_gpu": 43.27, "tokens/trainable": 141758} +{"epoch": 0.8203125, "grad_norm": 2.203993558883667, "learning_rate": 7.185131535190975e-05, "loss": 0.1982336789369583, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.21925, "step": 210, "tokens/total": 7214064, "tokens/train_per_sec_per_gpu": 43.1, "tokens/trainable": 142491} +{"epoch": 0.82421875, "grad_norm": 2.1233174800872803, "learning_rate": 7.158179796885005e-05, "loss": 0.20402324199676514, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.22633, "step": 211, "tokens/total": 7248032, "tokens/train_per_sec_per_gpu": 41.58, "tokens/trainable": 143174} +{"epoch": 0.828125, "grad_norm": 2.579690456390381, "learning_rate": 7.131159054949273e-05, "loss": 0.25618457794189453, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.29199, "step": 212, "tokens/total": 7286160, "tokens/train_per_sec_per_gpu": 31.78, "tokens/trainable": 143815} +{"epoch": 0.83203125, "grad_norm": 2.7903919219970703, "learning_rate": 7.104070433827139e-05, "loss": 0.2617688775062561, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.29923, "step": 213, "tokens/total": 7321216, "tokens/train_per_sec_per_gpu": 32.01, "tokens/trainable": 144389} +{"epoch": 0.8359375, "grad_norm": 2.287923812866211, "learning_rate": 7.076915060786705e-05, "loss": 0.22235724329948425, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.24902, "step": 214, "tokens/total": 7353552, "tokens/train_per_sec_per_gpu": 39.39, "tokens/trainable": 145038} +{"epoch": 0.83984375, "grad_norm": 2.1396262645721436, "learning_rate": 7.049694065873882e-05, "loss": 0.2245885133743286, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 37.98, "memory/max_allocated (GiB)": 37.98, "ppl": 1.25181, "step": 215, "tokens/total": 7387920, "tokens/train_per_sec_per_gpu": 36.52, "tokens/trainable": 145697} +{"epoch": 0.84375, "grad_norm": 2.494025707244873, "learning_rate": 7.022408581865382e-05, "loss": 0.19413140416145325, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.21426, "step": 216, "tokens/total": 7421680, "tokens/train_per_sec_per_gpu": 35.51, "tokens/trainable": 146373} +{"epoch": 0.84765625, "grad_norm": 2.3242695331573486, "learning_rate": 6.99505974422157e-05, "loss": 0.20908966660499573, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.23256, "step": 217, "tokens/total": 7455824, "tokens/train_per_sec_per_gpu": 46.03, "tokens/trainable": 147082} +{"epoch": 0.8515625, "grad_norm": 1.884535789489746, "learning_rate": 6.967648691039213e-05, "loss": 0.2239248901605606, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.25098, "step": 218, "tokens/total": 7487312, "tokens/train_per_sec_per_gpu": 41.93, "tokens/trainable": 147726} +{"epoch": 0.85546875, "grad_norm": 1.6706535816192627, "learning_rate": 6.940176563004123e-05, "loss": 0.22667790949344635, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.25443, "step": 219, "tokens/total": 7521104, "tokens/train_per_sec_per_gpu": 36.18, "tokens/trainable": 148345} +{"epoch": 0.859375, "grad_norm": 2.292886257171631, "learning_rate": 6.912644503343682e-05, "loss": 0.2368418425321579, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.26724, "step": 220, "tokens/total": 7558880, "tokens/train_per_sec_per_gpu": 35.52, "tokens/trainable": 148982} +{"epoch": 0.86328125, "grad_norm": 1.8607251644134521, "learning_rate": 6.885053657779273e-05, "loss": 0.18052904307842255, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 36.14, "memory/max_allocated (GiB)": 36.14, "ppl": 1.19785, "step": 221, "tokens/total": 7589824, "tokens/train_per_sec_per_gpu": 41.11, "tokens/trainable": 149658} +{"epoch": 0.8671875, "grad_norm": 2.618786096572876, "learning_rate": 6.857405174478604e-05, "loss": 0.23044252395629883, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 37.2, "memory/max_allocated (GiB)": 37.2, "ppl": 1.25916, "step": 222, "tokens/total": 7624112, "tokens/train_per_sec_per_gpu": 36.71, "tokens/trainable": 150312} +{"epoch": 0.87109375, "grad_norm": 1.9852945804595947, "learning_rate": 6.82970020400792e-05, "loss": 0.2077493816614151, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.2309, "step": 223, "tokens/total": 7660944, "tokens/train_per_sec_per_gpu": 34.41, "tokens/trainable": 150969} +{"epoch": 0.875, "grad_norm": 2.531921863555908, "learning_rate": 6.801939899284132e-05, "loss": 0.14526709914207458, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 38.15, "memory/max_allocated (GiB)": 38.15, "ppl": 1.15635, "step": 224, "tokens/total": 7696480, "tokens/train_per_sec_per_gpu": 41.06, "tokens/trainable": 151704} +{"epoch": 0.87890625, "grad_norm": 2.646780014038086, "learning_rate": 6.774125415526827e-05, "loss": 0.1902908980846405, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 37.84, "memory/max_allocated (GiB)": 37.84, "ppl": 1.2096, "step": 225, "tokens/total": 7729440, "tokens/train_per_sec_per_gpu": 47.33, "tokens/trainable": 152454} +{"epoch": 0.8828125, "grad_norm": 1.9276983737945557, "learning_rate": 6.746257910210214e-05, "loss": 0.20595474541187286, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 36.92, "memory/max_allocated (GiB)": 36.92, "ppl": 1.2287, "step": 226, "tokens/total": 7762224, "tokens/train_per_sec_per_gpu": 39.4, "tokens/trainable": 153099} +{"epoch": 0.88671875, "grad_norm": 2.6076107025146484, "learning_rate": 6.718338543014937e-05, "loss": 0.2609787881374359, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 35.52, "memory/max_allocated (GiB)": 35.52, "ppl": 1.2982, "step": 227, "tokens/total": 7791856, "tokens/train_per_sec_per_gpu": 42.84, "tokens/trainable": 153711} +{"epoch": 0.890625, "grad_norm": 1.8267263174057007, "learning_rate": 6.69036847577983e-05, "loss": 0.20453855395317078, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.22696, "step": 228, "tokens/total": 7825280, "tokens/train_per_sec_per_gpu": 37.67, "tokens/trainable": 154388} +{"epoch": 0.89453125, "grad_norm": 2.0188229084014893, "learning_rate": 6.662348872453553e-05, "loss": 0.15743260085582733, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 38.3, "memory/max_allocated (GiB)": 38.3, "ppl": 1.1705, "step": 229, "tokens/total": 7861392, "tokens/train_per_sec_per_gpu": 38.74, "tokens/trainable": 155057} +{"epoch": 0.8984375, "grad_norm": 1.337825059890747, "learning_rate": 6.63428089904618e-05, "loss": 0.18097956478595734, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.19839, "step": 230, "tokens/total": 7900144, "tokens/train_per_sec_per_gpu": 36.11, "tokens/trainable": 155729} +{"epoch": 0.90234375, "grad_norm": 2.8708689212799072, "learning_rate": 6.60616572358065e-05, "loss": 0.23642387986183167, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.26671, "step": 231, "tokens/total": 7934080, "tokens/train_per_sec_per_gpu": 37.48, "tokens/trainable": 156314} +{"epoch": 0.90625, "grad_norm": 1.911243200302124, "learning_rate": 6.578004516044172e-05, "loss": 0.19577458500862122, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.21625, "step": 232, "tokens/total": 7969552, "tokens/train_per_sec_per_gpu": 43.32, "tokens/trainable": 156997} +{"epoch": 0.91015625, "grad_norm": 1.9685310125350952, "learning_rate": 6.549798448339548e-05, "loss": 0.1696252077817917, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.18486, "step": 233, "tokens/total": 8006608, "tokens/train_per_sec_per_gpu": 38.71, "tokens/trainable": 157681} +{"epoch": 0.9140625, "grad_norm": 3.0786499977111816, "learning_rate": 6.521548694236384e-05, "loss": 0.1969125121831894, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 39.19, "memory/max_allocated (GiB)": 39.19, "ppl": 1.21764, "step": 234, "tokens/total": 8043504, "tokens/train_per_sec_per_gpu": 35.36, "tokens/trainable": 158343} +{"epoch": 0.91796875, "grad_norm": 2.8069663047790527, "learning_rate": 6.493256429322259e-05, "loss": 0.2231893092393875, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.25006, "step": 235, "tokens/total": 8079280, "tokens/train_per_sec_per_gpu": 36.85, "tokens/trainable": 158999} +{"epoch": 0.921875, "grad_norm": 2.977426052093506, "learning_rate": 6.464922830953799e-05, "loss": 0.20157881081104279, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 39.19, "memory/max_allocated (GiB)": 39.19, "ppl": 1.22333, "step": 236, "tokens/total": 8118128, "tokens/train_per_sec_per_gpu": 32.3, "tokens/trainable": 159643} +{"epoch": 0.92578125, "grad_norm": 2.50754976272583, "learning_rate": 6.436549078207688e-05, "loss": 0.1935472935438156, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 38.11, "memory/max_allocated (GiB)": 38.11, "ppl": 1.21355, "step": 237, "tokens/total": 8150864, "tokens/train_per_sec_per_gpu": 41.52, "tokens/trainable": 160288} +{"epoch": 0.9296875, "grad_norm": 1.9547221660614014, "learning_rate": 6.408136351831592e-05, "loss": 0.2128223031759262, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.23716, "step": 238, "tokens/total": 8182832, "tokens/train_per_sec_per_gpu": 36.43, "tokens/trainable": 160927} +{"epoch": 0.93359375, "grad_norm": 1.78607177734375, "learning_rate": 6.379685834195036e-05, "loss": 0.1790531426668167, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.19608, "step": 239, "tokens/total": 8221984, "tokens/train_per_sec_per_gpu": 37.31, "tokens/trainable": 161692} +{"epoch": 0.9375, "grad_norm": 1.7047637701034546, "learning_rate": 6.351198709240186e-05, "loss": 0.1764223575592041, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.19294, "step": 240, "tokens/total": 8261760, "tokens/train_per_sec_per_gpu": 31.15, "tokens/trainable": 162379} +{"epoch": 0.94140625, "grad_norm": 1.609286904335022, "learning_rate": 6.32267616243259e-05, "loss": 0.19787843525409698, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.21881, "step": 241, "tokens/total": 8293968, "tokens/train_per_sec_per_gpu": 41.1, "tokens/trainable": 163037} +{"epoch": 0.9453125, "grad_norm": 2.0270001888275146, "learning_rate": 6.294119380711849e-05, "loss": 0.22181111574172974, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.24834, "step": 242, "tokens/total": 8327968, "tokens/train_per_sec_per_gpu": 33.37, "tokens/trainable": 163654} +{"epoch": 0.94921875, "grad_norm": 2.4258768558502197, "learning_rate": 6.265529552442209e-05, "loss": 0.1939297914505005, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 37.57, "memory/max_allocated (GiB)": 37.57, "ppl": 1.21401, "step": 243, "tokens/total": 8362944, "tokens/train_per_sec_per_gpu": 42.67, "tokens/trainable": 164369} +{"epoch": 0.953125, "grad_norm": 2.0433077812194824, "learning_rate": 6.236907867363127e-05, "loss": 0.2007543444633484, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.22232, "step": 244, "tokens/total": 8396960, "tokens/train_per_sec_per_gpu": 39.52, "tokens/trainable": 165009} +{"epoch": 0.95703125, "grad_norm": 1.9328750371932983, "learning_rate": 6.208255516539749e-05, "loss": 0.17915254831314087, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 36.21, "memory/max_allocated (GiB)": 36.21, "ppl": 1.1962, "step": 245, "tokens/total": 8428688, "tokens/train_per_sec_per_gpu": 36.17, "tokens/trainable": 165650} +{"epoch": 0.9609375, "grad_norm": 1.6828508377075195, "learning_rate": 6.179573692313344e-05, "loss": 0.19535577297210693, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.21574, "step": 246, "tokens/total": 8465328, "tokens/train_per_sec_per_gpu": 39.68, "tokens/trainable": 166322} +{"epoch": 0.96484375, "grad_norm": 1.6411811113357544, "learning_rate": 6.150863588251694e-05, "loss": 0.1768399029970169, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 39.35, "memory/max_allocated (GiB)": 39.35, "ppl": 1.19344, "step": 247, "tokens/total": 8505248, "tokens/train_per_sec_per_gpu": 33.13, "tokens/trainable": 167020} +{"epoch": 0.96875, "grad_norm": 1.9047377109527588, "learning_rate": 6.122126399099419e-05, "loss": 0.18448804318904877, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 39.59, "memory/max_allocated (GiB)": 39.59, "ppl": 1.2026, "step": 248, "tokens/total": 8544736, "tokens/train_per_sec_per_gpu": 44.67, "tokens/trainable": 167855} +{"epoch": 0.97265625, "grad_norm": 3.2608084678649902, "learning_rate": 6.0933633207282615e-05, "loss": 0.24828438460826874, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 38.22, "memory/max_allocated (GiB)": 38.22, "ppl": 1.28182, "step": 249, "tokens/total": 8580752, "tokens/train_per_sec_per_gpu": 36.92, "tokens/trainable": 168552} +{"epoch": 0.9765625, "grad_norm": 2.2838687896728516, "learning_rate": 6.064575550087316e-05, "loss": 0.15458332002162933, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.16717, "step": 250, "tokens/total": 8617408, "tokens/train_per_sec_per_gpu": 38.66, "tokens/trainable": 169264} +{"epoch": 0.98046875, "grad_norm": 1.881311297416687, "learning_rate": 6.0357642851532245e-05, "loss": 0.18332213163375854, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.2012, "step": 251, "tokens/total": 8653584, "tokens/train_per_sec_per_gpu": 34.57, "tokens/trainable": 169946} +{"epoch": 0.984375, "grad_norm": 2.3768467903137207, "learning_rate": 6.0069307248803294e-05, "loss": 0.18554630875587463, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.20388, "step": 252, "tokens/total": 8688528, "tokens/train_per_sec_per_gpu": 33.95, "tokens/trainable": 170589} +{"epoch": 0.98828125, "grad_norm": 1.7832485437393188, "learning_rate": 5.9780760691507635e-05, "loss": 0.17444023489952087, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 36.16, "memory/max_allocated (GiB)": 36.16, "ppl": 1.19058, "step": 253, "tokens/total": 8719488, "tokens/train_per_sec_per_gpu": 38.5, "tokens/trainable": 171180} +{"epoch": 0.9921875, "grad_norm": 2.098259210586548, "learning_rate": 5.9492015187245334e-05, "loss": 0.1784001588821411, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 38.63, "memory/max_allocated (GiB)": 38.63, "ppl": 1.1953, "step": 254, "tokens/total": 8756688, "tokens/train_per_sec_per_gpu": 34.67, "tokens/trainable": 171805} +{"epoch": 0.99609375, "grad_norm": 1.6954618692398071, "learning_rate": 5.920308275189541e-05, "loss": 0.19194760918617249, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 36.54, "memory/max_allocated (GiB)": 36.54, "ppl": 1.21161, "step": 255, "tokens/total": 8788400, "tokens/train_per_sec_per_gpu": 39.55, "tokens/trainable": 172424} +{"epoch": 1.0, "grad_norm": 1.9260964393615723, "learning_rate": 5.8913975409115874e-05, "loss": 0.15537220239639282, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 37.57, "memory/max_allocated (GiB)": 37.57, "ppl": 1.16809, "step": 256, "tokens/total": 8821408, "tokens/train_per_sec_per_gpu": 40.64, "tokens/trainable": 173126} +{"epoch": 1.00390625, "grad_norm": 2.28481388092041, "learning_rate": 5.8624705189843395e-05, "loss": 0.14324571192264557, "memory/device_reserved (GiB)": 43.83, "memory/max_active (GiB)": 39.04, "memory/max_allocated (GiB)": 39.04, "ppl": 1.15401, "step": 257, "tokens/total": 8858416, "tokens/train_per_sec_per_gpu": 35.57, "tokens/trainable": 173841} +{"epoch": 1.0078125, "grad_norm": 3.4961204528808594, "learning_rate": 5.833528413179249e-05, "loss": 0.19772934913635254, "memory/device_reserved (GiB)": 40.53, "memory/max_active (GiB)": 38.41, "memory/max_allocated (GiB)": 38.41, "ppl": 1.21863, "step": 258, "tokens/total": 8894656, "tokens/train_per_sec_per_gpu": 36.02, "tokens/trainable": 174554} +{"epoch": 1.01171875, "grad_norm": 1.801453948020935, "learning_rate": 5.80457242789548e-05, "loss": 0.13469935953617096, "memory/device_reserved (GiB)": 41.82, "memory/max_active (GiB)": 39.48, "memory/max_allocated (GiB)": 39.48, "ppl": 1.14419, "step": 259, "tokens/total": 8934480, "tokens/train_per_sec_per_gpu": 38.86, "tokens/trainable": 175358} +{"epoch": 1.015625, "grad_norm": 1.624858021736145, "learning_rate": 5.77560376810977e-05, "loss": 0.11658766865730286, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.12366, "step": 260, "tokens/total": 8967488, "tokens/train_per_sec_per_gpu": 44.27, "tokens/trainable": 176104} +{"epoch": 1.01953125, "grad_norm": 1.6805838346481323, "learning_rate": 5.7466236393263005e-05, "loss": 0.16554510593414307, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 38.93, "memory/max_allocated (GiB)": 38.93, "ppl": 1.18004, "step": 261, "tokens/total": 9001840, "tokens/train_per_sec_per_gpu": 38.13, "tokens/trainable": 176760} +{"epoch": 1.0234375, "grad_norm": 2.3874382972717285, "learning_rate": 5.717633247526522e-05, "loss": 0.1759473979473114, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 36.63, "memory/max_allocated (GiB)": 36.63, "ppl": 1.19238, "step": 262, "tokens/total": 9032208, "tokens/train_per_sec_per_gpu": 36.78, "tokens/trainable": 177337} +{"epoch": 1.02734375, "grad_norm": 2.705439805984497, "learning_rate": 5.688633799118971e-05, "loss": 0.1853790283203125, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 36.82, "memory/max_allocated (GiB)": 36.82, "ppl": 1.20367, "step": 263, "tokens/total": 9064352, "tokens/train_per_sec_per_gpu": 34.41, "tokens/trainable": 177932} +{"epoch": 1.03125, "grad_norm": 1.6583402156829834, "learning_rate": 5.659626500889066e-05, "loss": 0.13802561163902283, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 36.93, "memory/max_allocated (GiB)": 36.93, "ppl": 1.148, "step": 264, "tokens/total": 9097776, "tokens/train_per_sec_per_gpu": 42.99, "tokens/trainable": 178657} +{"epoch": 1.03515625, "grad_norm": 2.4394450187683105, "learning_rate": 5.6306125599488905e-05, "loss": 0.18343624472618103, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 36.72, "memory/max_allocated (GiB)": 36.72, "ppl": 1.20134, "step": 265, "tokens/total": 9128128, "tokens/train_per_sec_per_gpu": 47.2, "tokens/trainable": 179294} +{"epoch": 1.0390625, "grad_norm": 1.4432772397994995, "learning_rate": 5.601593183686955e-05, "loss": 0.13093361258506775, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.13989, "step": 266, "tokens/total": 9164960, "tokens/train_per_sec_per_gpu": 40.83, "tokens/trainable": 180022} +{"epoch": 1.04296875, "grad_norm": 2.0674853324890137, "learning_rate": 5.572569579717961e-05, "loss": 0.17125849425792694, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.1868, "step": 267, "tokens/total": 9200016, "tokens/train_per_sec_per_gpu": 36.81, "tokens/trainable": 180722} +{"epoch": 1.046875, "grad_norm": 1.7662878036499023, "learning_rate": 5.543542955832538e-05, "loss": 0.1518634855747223, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.164, "step": 268, "tokens/total": 9239632, "tokens/train_per_sec_per_gpu": 36.39, "tokens/trainable": 181445} +{"epoch": 1.05078125, "grad_norm": 1.7709616422653198, "learning_rate": 5.514514519946986e-05, "loss": 0.14032413065433502, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.15065, "step": 269, "tokens/total": 9279616, "tokens/train_per_sec_per_gpu": 39.89, "tokens/trainable": 182205} +{"epoch": 1.0546875, "grad_norm": 1.9403914213180542, "learning_rate": 5.485485480053015e-05, "loss": 0.14354491233825684, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.15436, "step": 270, "tokens/total": 9317696, "tokens/train_per_sec_per_gpu": 31.24, "tokens/trainable": 182838} +{"epoch": 1.05859375, "grad_norm": 1.4965317249298096, "learning_rate": 5.4564570441674645e-05, "loss": 0.1493488848209381, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.16108, "step": 271, "tokens/total": 9350976, "tokens/train_per_sec_per_gpu": 42.15, "tokens/trainable": 183493} +{"epoch": 1.0625, "grad_norm": 1.8057187795639038, "learning_rate": 5.42743042028204e-05, "loss": 0.1667921245098114, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 37.73, "memory/max_allocated (GiB)": 37.73, "ppl": 1.18151, "step": 272, "tokens/total": 9386816, "tokens/train_per_sec_per_gpu": 30.55, "tokens/trainable": 184102} +{"epoch": 1.06640625, "grad_norm": 1.3506532907485962, "learning_rate": 5.3984068163130464e-05, "loss": 0.12589454650878906, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.13416, "step": 273, "tokens/total": 9420016, "tokens/train_per_sec_per_gpu": 43.0, "tokens/trainable": 184843} +{"epoch": 1.0703125, "grad_norm": 2.3333301544189453, "learning_rate": 5.369387440051111e-05, "loss": 0.15624061226844788, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.16911, "step": 274, "tokens/total": 9453280, "tokens/train_per_sec_per_gpu": 36.63, "tokens/trainable": 185492} +{"epoch": 1.07421875, "grad_norm": 2.3461499214172363, "learning_rate": 5.340373499110935e-05, "loss": 0.1525138020515442, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 1.16476, "step": 275, "tokens/total": 9488704, "tokens/train_per_sec_per_gpu": 39.02, "tokens/trainable": 186210} +{"epoch": 1.078125, "grad_norm": 1.3280670642852783, "learning_rate": 5.3113662008810304e-05, "loss": 0.14252665638923645, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.15318, "step": 276, "tokens/total": 9528720, "tokens/train_per_sec_per_gpu": 36.39, "tokens/trainable": 186941} +{"epoch": 1.08203125, "grad_norm": 3.040996551513672, "learning_rate": 5.282366752473479e-05, "loss": 0.14559338986873627, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 38.1, "memory/max_allocated (GiB)": 38.1, "ppl": 1.15673, "step": 277, "tokens/total": 9565584, "tokens/train_per_sec_per_gpu": 37.19, "tokens/trainable": 187629} +{"epoch": 1.0859375, "grad_norm": 2.616772413253784, "learning_rate": 5.2533763606737005e-05, "loss": 0.2097715139389038, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.2334, "step": 278, "tokens/total": 9605088, "tokens/train_per_sec_per_gpu": 28.44, "tokens/trainable": 188256} +{"epoch": 1.08984375, "grad_norm": 2.03157901763916, "learning_rate": 5.224396231890232e-05, "loss": 0.1391645073890686, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.14931, "step": 279, "tokens/total": 9636624, "tokens/train_per_sec_per_gpu": 43.62, "tokens/trainable": 188983} +{"epoch": 1.09375, "grad_norm": 6.189345836639404, "learning_rate": 5.195427572104522e-05, "loss": 0.1676725298166275, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 38.81, "memory/max_allocated (GiB)": 38.81, "ppl": 1.18255, "step": 280, "tokens/total": 9670752, "tokens/train_per_sec_per_gpu": 40.4, "tokens/trainable": 189655} +{"epoch": 1.09765625, "grad_norm": 1.9243310689926147, "learning_rate": 5.166471586820751e-05, "loss": 0.13732731342315674, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 39.28, "memory/max_allocated (GiB)": 39.28, "ppl": 1.1472, "step": 281, "tokens/total": 9707264, "tokens/train_per_sec_per_gpu": 41.8, "tokens/trainable": 190405} +{"epoch": 1.1015625, "grad_norm": 2.5426077842712402, "learning_rate": 5.1375294810156615e-05, "loss": 0.14917176961898804, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.16087, "step": 282, "tokens/total": 9741488, "tokens/train_per_sec_per_gpu": 38.23, "tokens/trainable": 191032} +{"epoch": 1.10546875, "grad_norm": 1.8616533279418945, "learning_rate": 5.1086024590884144e-05, "loss": 0.1542719006538391, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 39.09, "memory/max_allocated (GiB)": 39.09, "ppl": 1.16681, "step": 283, "tokens/total": 9779264, "tokens/train_per_sec_per_gpu": 35.62, "tokens/trainable": 191684} +{"epoch": 1.109375, "grad_norm": 1.8073782920837402, "learning_rate": 5.079691724810461e-05, "loss": 0.14516958594322205, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.15624, "step": 284, "tokens/total": 9814688, "tokens/train_per_sec_per_gpu": 31.36, "tokens/trainable": 192305} +{"epoch": 1.11328125, "grad_norm": 2.07735276222229, "learning_rate": 5.0507984812754684e-05, "loss": 0.15739625692367554, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 37.19, "memory/max_allocated (GiB)": 37.19, "ppl": 1.17046, "step": 285, "tokens/total": 9847920, "tokens/train_per_sec_per_gpu": 42.22, "tokens/trainable": 193007} +{"epoch": 1.1171875, "grad_norm": 0.9349462985992432, "learning_rate": 5.021923930849237e-05, "loss": 0.14192627370357513, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.15249, "step": 286, "tokens/total": 9881424, "tokens/train_per_sec_per_gpu": 38.1, "tokens/trainable": 193656} +{"epoch": 1.12109375, "grad_norm": 1.7338296175003052, "learning_rate": 4.99306927511967e-05, "loss": 0.15040136873722076, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.1623, "step": 287, "tokens/total": 9914464, "tokens/train_per_sec_per_gpu": 39.83, "tokens/trainable": 194253} +{"epoch": 1.125, "grad_norm": 6.403843402862549, "learning_rate": 4.964235714846775e-05, "loss": 0.17958974838256836, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 36.31, "memory/max_allocated (GiB)": 36.31, "ppl": 1.19673, "step": 288, "tokens/total": 9945920, "tokens/train_per_sec_per_gpu": 39.97, "tokens/trainable": 194840} +{"epoch": 1.12890625, "grad_norm": 1.6634995937347412, "learning_rate": 4.9354244499126866e-05, "loss": 0.18317528069019318, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 36.11, "memory/max_allocated (GiB)": 36.11, "ppl": 1.20102, "step": 289, "tokens/total": 9977248, "tokens/train_per_sec_per_gpu": 40.83, "tokens/trainable": 195527} +{"epoch": 1.1328125, "grad_norm": 2.035994052886963, "learning_rate": 4.90663667927174e-05, "loss": 0.14955787360668182, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.16132, "step": 290, "tokens/total": 10009664, "tokens/train_per_sec_per_gpu": 47.44, "tokens/trainable": 196197} +{"epoch": 1.13671875, "grad_norm": 1.7437962293624878, "learning_rate": 4.877873600900581e-05, "loss": 0.13591407239437103, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 37.98, "memory/max_allocated (GiB)": 37.98, "ppl": 1.14558, "step": 291, "tokens/total": 10045488, "tokens/train_per_sec_per_gpu": 34.59, "tokens/trainable": 196870} +{"epoch": 1.140625, "grad_norm": 1.8452644348144531, "learning_rate": 4.849136411748306e-05, "loss": 0.1213836520910263, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 1.12906, "step": 292, "tokens/total": 10083280, "tokens/train_per_sec_per_gpu": 35.75, "tokens/trainable": 197556} +{"epoch": 1.14453125, "grad_norm": 3.3632965087890625, "learning_rate": 4.8204263076866574e-05, "loss": 0.16382819414138794, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 38.66, "memory/max_allocated (GiB)": 38.66, "ppl": 1.17801, "step": 293, "tokens/total": 10118752, "tokens/train_per_sec_per_gpu": 39.11, "tokens/trainable": 198261} +{"epoch": 1.1484375, "grad_norm": 10.430154800415039, "learning_rate": 4.791744483460251e-05, "loss": 0.18258440494537354, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.20032, "step": 294, "tokens/total": 10152832, "tokens/train_per_sec_per_gpu": 31.88, "tokens/trainable": 198876} +{"epoch": 1.15234375, "grad_norm": 1.7591755390167236, "learning_rate": 4.7630921326368736e-05, "loss": 0.17178337275981903, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 39.19, "memory/max_allocated (GiB)": 39.19, "ppl": 1.18742, "step": 295, "tokens/total": 10191072, "tokens/train_per_sec_per_gpu": 29.67, "tokens/trainable": 199473} +{"epoch": 1.15625, "grad_norm": 3.9158895015716553, "learning_rate": 4.7344704475577916e-05, "loss": 0.14569851756095886, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 35.48, "memory/max_allocated (GiB)": 35.48, "ppl": 1.15685, "step": 296, "tokens/total": 10220224, "tokens/train_per_sec_per_gpu": 42.71, "tokens/trainable": 200130} +{"epoch": 1.16015625, "grad_norm": 1.4311223030090332, "learning_rate": 4.705880619288153e-05, "loss": 0.15297874808311462, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 38.18, "memory/max_allocated (GiB)": 38.18, "ppl": 1.1653, "step": 297, "tokens/total": 10255904, "tokens/train_per_sec_per_gpu": 35.23, "tokens/trainable": 200756} +{"epoch": 1.1640625, "grad_norm": 0.7483081817626953, "learning_rate": 4.677323837567412e-05, "loss": 0.1351523995399475, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.14471, "step": 298, "tokens/total": 10288400, "tokens/train_per_sec_per_gpu": 45.04, "tokens/trainable": 201465} +{"epoch": 1.16796875, "grad_norm": 2.0412352085113525, "learning_rate": 4.6488012907598146e-05, "loss": 0.16247135400772095, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 36.3, "memory/max_allocated (GiB)": 36.3, "ppl": 1.17641, "step": 299, "tokens/total": 10320336, "tokens/train_per_sec_per_gpu": 38.24, "tokens/trainable": 202098} +{"epoch": 1.171875, "grad_norm": 5.4887495040893555, "learning_rate": 4.620314165804964e-05, "loss": 0.1542282998561859, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 38.26, "memory/max_allocated (GiB)": 38.26, "ppl": 1.16676, "step": 300, "tokens/total": 10355744, "tokens/train_per_sec_per_gpu": 38.36, "tokens/trainable": 202788} +{"epoch": 1.17578125, "grad_norm": 1.8277548551559448, "learning_rate": 4.591863648168407e-05, "loss": 0.13231152296066284, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.14146, "step": 301, "tokens/total": 10390016, "tokens/train_per_sec_per_gpu": 39.7, "tokens/trainable": 203526} +{"epoch": 1.1796875, "grad_norm": 1.2925838232040405, "learning_rate": 4.5634509217923135e-05, "loss": 0.15181566774845123, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.16395, "step": 302, "tokens/total": 10423136, "tokens/train_per_sec_per_gpu": 42.44, "tokens/trainable": 204208} +{"epoch": 1.18359375, "grad_norm": 1.8268849849700928, "learning_rate": 4.535077169046201e-05, "loss": 0.1731235384941101, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 37.93, "memory/max_allocated (GiB)": 37.93, "ppl": 1.18901, "step": 303, "tokens/total": 10459424, "tokens/train_per_sec_per_gpu": 37.15, "tokens/trainable": 204878} +{"epoch": 1.1875, "grad_norm": 2.162674903869629, "learning_rate": 4.506743570677743e-05, "loss": 0.1601771116256714, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 36.13, "memory/max_allocated (GiB)": 36.13, "ppl": 1.17372, "step": 304, "tokens/total": 10489248, "tokens/train_per_sec_per_gpu": 46.39, "tokens/trainable": 205491} +{"epoch": 1.19140625, "grad_norm": 1.595955729484558, "learning_rate": 4.478451305763618e-05, "loss": 0.16412734985351562, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 37.17, "memory/max_allocated (GiB)": 37.17, "ppl": 1.17836, "step": 305, "tokens/total": 10522496, "tokens/train_per_sec_per_gpu": 44.57, "tokens/trainable": 206158} +{"epoch": 1.1953125, "grad_norm": 1.7492233514785767, "learning_rate": 4.450201551660454e-05, "loss": 0.1643408238887787, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 36.27, "memory/max_allocated (GiB)": 36.27, "ppl": 1.17862, "step": 306, "tokens/total": 10553328, "tokens/train_per_sec_per_gpu": 40.12, "tokens/trainable": 206791} +{"epoch": 1.19921875, "grad_norm": 2.7798655033111572, "learning_rate": 4.4219954839558276e-05, "loss": 0.17231810092926025, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.18806, "step": 307, "tokens/total": 10587392, "tokens/train_per_sec_per_gpu": 36.06, "tokens/trainable": 207434} +{"epoch": 1.203125, "grad_norm": 1.3518928289413452, "learning_rate": 4.393834276419352e-05, "loss": 0.1529066562652588, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 36.9, "memory/max_allocated (GiB)": 36.9, "ppl": 1.16522, "step": 308, "tokens/total": 10618704, "tokens/train_per_sec_per_gpu": 46.58, "tokens/trainable": 208130} +{"epoch": 1.20703125, "grad_norm": 0.7838271260261536, "learning_rate": 4.36571910095382e-05, "loss": 0.12857945263385773, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.13721, "step": 309, "tokens/total": 10656688, "tokens/train_per_sec_per_gpu": 38.15, "tokens/trainable": 208822} +{"epoch": 1.2109375, "grad_norm": 1.2982947826385498, "learning_rate": 4.337651127546448e-05, "loss": 0.16685378551483154, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.18158, "step": 310, "tokens/total": 10691904, "tokens/train_per_sec_per_gpu": 41.8, "tokens/trainable": 209478} +{"epoch": 1.21484375, "grad_norm": 1.1415687799453735, "learning_rate": 4.3096315242201736e-05, "loss": 0.14497482776641846, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 36.37, "memory/max_allocated (GiB)": 36.37, "ppl": 1.15601, "step": 311, "tokens/total": 10724224, "tokens/train_per_sec_per_gpu": 37.93, "tokens/trainable": 210126} +{"epoch": 1.21875, "grad_norm": 0.9644502997398376, "learning_rate": 4.2816614569850635e-05, "loss": 0.14491696655750275, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 38.49, "memory/max_allocated (GiB)": 38.49, "ppl": 1.15594, "step": 312, "tokens/total": 10759664, "tokens/train_per_sec_per_gpu": 37.49, "tokens/trainable": 210783} +{"epoch": 1.22265625, "grad_norm": 1.2648833990097046, "learning_rate": 4.2537420897897864e-05, "loss": 0.12908977270126343, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 38.63, "memory/max_allocated (GiB)": 38.63, "ppl": 1.13779, "step": 313, "tokens/total": 10794896, "tokens/train_per_sec_per_gpu": 36.07, "tokens/trainable": 211497} +{"epoch": 1.2265625, "grad_norm": 1.242637038230896, "learning_rate": 4.225874584473174e-05, "loss": 0.1360701620578766, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.14576, "step": 314, "tokens/total": 10826944, "tokens/train_per_sec_per_gpu": 40.8, "tokens/trainable": 212145} +{"epoch": 1.23046875, "grad_norm": 2.666459560394287, "learning_rate": 4.19806010071587e-05, "loss": 0.12882067263126373, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 38.36, "memory/max_allocated (GiB)": 38.36, "ppl": 1.13749, "step": 315, "tokens/total": 10864608, "tokens/train_per_sec_per_gpu": 36.63, "tokens/trainable": 212853} +{"epoch": 1.234375, "grad_norm": 1.148713231086731, "learning_rate": 4.170299795992081e-05, "loss": 0.14641444385051727, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 36.74, "memory/max_allocated (GiB)": 36.74, "ppl": 1.15768, "step": 316, "tokens/total": 10897760, "tokens/train_per_sec_per_gpu": 38.18, "tokens/trainable": 213483} +{"epoch": 1.23828125, "grad_norm": 0.794584333896637, "learning_rate": 4.142594825521398e-05, "loss": 0.12834244966506958, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 39.12, "memory/max_allocated (GiB)": 39.12, "ppl": 1.13694, "step": 317, "tokens/total": 10936608, "tokens/train_per_sec_per_gpu": 34.23, "tokens/trainable": 214176} +{"epoch": 1.2421875, "grad_norm": 1.971368670463562, "learning_rate": 4.114946342220728e-05, "loss": 0.13948985934257507, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 1.14969, "step": 318, "tokens/total": 10972944, "tokens/train_per_sec_per_gpu": 37.96, "tokens/trainable": 214895} +{"epoch": 1.24609375, "grad_norm": 1.0556681156158447, "learning_rate": 4.087355496656321e-05, "loss": 0.12682969868183136, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 1.13522, "step": 319, "tokens/total": 11006288, "tokens/train_per_sec_per_gpu": 44.39, "tokens/trainable": 215597} +{"epoch": 1.25, "grad_norm": 1.0986287593841553, "learning_rate": 4.05982343699588e-05, "loss": 0.1319132149219513, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.14101, "step": 320, "tokens/total": 11045184, "tokens/train_per_sec_per_gpu": 37.92, "tokens/trainable": 216368} +{"epoch": 1.25390625, "grad_norm": 1.4039644002914429, "learning_rate": 4.0323513089607876e-05, "loss": 0.1285136640071869, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.13714, "step": 321, "tokens/total": 11079648, "tokens/train_per_sec_per_gpu": 39.04, "tokens/trainable": 217094} +{"epoch": 1.2578125, "grad_norm": 6.872169017791748, "learning_rate": 4.004940255778431e-05, "loss": 0.149008110165596, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.16068, "step": 322, "tokens/total": 11118016, "tokens/train_per_sec_per_gpu": 41.09, "tokens/trainable": 217819} +{"epoch": 1.26171875, "grad_norm": 2.407085657119751, "learning_rate": 3.977591418134619e-05, "loss": 0.1602703332901001, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 38.45, "memory/max_allocated (GiB)": 38.45, "ppl": 1.17383, "step": 323, "tokens/total": 11152672, "tokens/train_per_sec_per_gpu": 40.0, "tokens/trainable": 218494} +{"epoch": 1.265625, "grad_norm": 3.252678155899048, "learning_rate": 3.95030593412612e-05, "loss": 0.1595120131969452, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.17294, "step": 324, "tokens/total": 11188288, "tokens/train_per_sec_per_gpu": 40.82, "tokens/trainable": 219172} +{"epoch": 1.26953125, "grad_norm": 1.6139975786209106, "learning_rate": 3.923084939213296e-05, "loss": 0.13786490261554718, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 35.58, "memory/max_allocated (GiB)": 35.58, "ppl": 1.14782, "step": 325, "tokens/total": 11218384, "tokens/train_per_sec_per_gpu": 41.15, "tokens/trainable": 219813} +{"epoch": 1.2734375, "grad_norm": 1.6084377765655518, "learning_rate": 3.895929566172861e-05, "loss": 0.13217666745185852, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.14131, "step": 326, "tokens/total": 11254048, "tokens/train_per_sec_per_gpu": 38.27, "tokens/trainable": 220523} +{"epoch": 1.27734375, "grad_norm": 0.915625274181366, "learning_rate": 3.868840945050728e-05, "loss": 0.11948707699775696, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 39.05, "memory/max_allocated (GiB)": 39.05, "ppl": 1.12692, "step": 327, "tokens/total": 11292576, "tokens/train_per_sec_per_gpu": 36.59, "tokens/trainable": 221224} +{"epoch": 1.28125, "grad_norm": 1.5779316425323486, "learning_rate": 3.841820203114995e-05, "loss": 0.17293205857276917, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.18879, "step": 328, "tokens/total": 11323936, "tokens/train_per_sec_per_gpu": 42.27, "tokens/trainable": 221899} +{"epoch": 1.28515625, "grad_norm": 0.8055605292320251, "learning_rate": 3.814868464809027e-05, "loss": 0.14241835474967957, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 36.61, "memory/max_allocated (GiB)": 36.61, "ppl": 1.15306, "step": 329, "tokens/total": 11355504, "tokens/train_per_sec_per_gpu": 38.49, "tokens/trainable": 222501} +{"epoch": 1.2890625, "grad_norm": 1.526537537574768, "learning_rate": 3.787986851704667e-05, "loss": 0.1489291787147522, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.16059, "step": 330, "tokens/total": 11388656, "tokens/train_per_sec_per_gpu": 41.96, "tokens/trainable": 223171} +{"epoch": 1.29296875, "grad_norm": 0.894629955291748, "learning_rate": 3.7611764824555654e-05, "loss": 0.12369616329669952, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 37.57, "memory/max_allocated (GiB)": 37.57, "ppl": 1.13167, "step": 331, "tokens/total": 11419856, "tokens/train_per_sec_per_gpu": 48.16, "tokens/trainable": 223878} +{"epoch": 1.296875, "grad_norm": 2.679105043411255, "learning_rate": 3.734438472750619e-05, "loss": 0.10524474084377289, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 39.35, "memory/max_allocated (GiB)": 39.35, "ppl": 1.11098, "step": 332, "tokens/total": 11459504, "tokens/train_per_sec_per_gpu": 39.57, "tokens/trainable": 224626} +{"epoch": 1.30078125, "grad_norm": 1.0091460943222046, "learning_rate": 3.707773935267552e-05, "loss": 0.13102583587169647, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.14, "step": 333, "tokens/total": 11496192, "tokens/train_per_sec_per_gpu": 37.98, "tokens/trainable": 225355} +{"epoch": 1.3046875, "grad_norm": 3.756131649017334, "learning_rate": 3.68118397962661e-05, "loss": 0.18132947385311127, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 38.39, "memory/max_allocated (GiB)": 38.39, "ppl": 1.19881, "step": 334, "tokens/total": 11530640, "tokens/train_per_sec_per_gpu": 36.38, "tokens/trainable": 225970} +{"epoch": 1.30859375, "grad_norm": 0.8116171360015869, "learning_rate": 3.654669712344384e-05, "loss": 0.10943514108657837, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 36.58, "memory/max_allocated (GiB)": 36.58, "ppl": 1.11565, "step": 335, "tokens/total": 11561456, "tokens/train_per_sec_per_gpu": 53.83, "tokens/trainable": 226682} +{"epoch": 1.3125, "grad_norm": 1.7347402572631836, "learning_rate": 3.628232236787763e-05, "loss": 0.13329127430915833, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.14258, "step": 336, "tokens/total": 11596672, "tokens/train_per_sec_per_gpu": 33.37, "tokens/trainable": 227381} +{"epoch": 1.31640625, "grad_norm": 1.1493268013000488, "learning_rate": 3.6018726531280144e-05, "loss": 0.12621060013771057, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.13452, "step": 337, "tokens/total": 11633840, "tokens/train_per_sec_per_gpu": 43.42, "tokens/trainable": 228110} +{"epoch": 1.3203125, "grad_norm": 5.033420085906982, "learning_rate": 3.575592058295017e-05, "loss": 0.16194891929626465, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 36.92, "memory/max_allocated (GiB)": 36.92, "ppl": 1.1758, "step": 338, "tokens/total": 11667568, "tokens/train_per_sec_per_gpu": 41.67, "tokens/trainable": 228798} +{"epoch": 1.32421875, "grad_norm": 1.0142133235931396, "learning_rate": 3.549391545931585e-05, "loss": 0.14643818140029907, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.1577, "step": 339, "tokens/total": 11701632, "tokens/train_per_sec_per_gpu": 42.94, "tokens/trainable": 229496} +{"epoch": 1.328125, "grad_norm": 1.856703519821167, "learning_rate": 3.5232722063479914e-05, "loss": 0.1513553112745285, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 37.17, "memory/max_allocated (GiB)": 37.17, "ppl": 1.16341, "step": 340, "tokens/total": 11735568, "tokens/train_per_sec_per_gpu": 42.28, "tokens/trainable": 230189} +{"epoch": 1.33203125, "grad_norm": 0.7539449334144592, "learning_rate": 3.49723512647657e-05, "loss": 0.12327927350997925, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 39.08, "memory/max_allocated (GiB)": 39.08, "ppl": 1.1312, "step": 341, "tokens/total": 11774528, "tokens/train_per_sec_per_gpu": 37.91, "tokens/trainable": 230964} +{"epoch": 1.3359375, "grad_norm": 1.2855886220932007, "learning_rate": 3.471281389826491e-05, "loss": 0.1550772786140442, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 36.27, "memory/max_allocated (GiB)": 36.27, "ppl": 1.16775, "step": 342, "tokens/total": 11805104, "tokens/train_per_sec_per_gpu": 41.58, "tokens/trainable": 231566} +{"epoch": 1.33984375, "grad_norm": 0.693223774433136, "learning_rate": 3.4454120764386764e-05, "loss": 0.1159910261631012, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.12299, "step": 343, "tokens/total": 11843248, "tokens/train_per_sec_per_gpu": 36.73, "tokens/trainable": 232252} +{"epoch": 1.34375, "grad_norm": 1.195324420928955, "learning_rate": 3.4196282628408526e-05, "loss": 0.13174816966056824, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 37.45, "memory/max_allocated (GiB)": 37.45, "ppl": 1.14082, "step": 344, "tokens/total": 11877232, "tokens/train_per_sec_per_gpu": 38.11, "tokens/trainable": 232939} +{"epoch": 1.34765625, "grad_norm": 1.3568662405014038, "learning_rate": 3.3939310220027456e-05, "loss": 0.14118319749832153, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.15164, "step": 345, "tokens/total": 11911024, "tokens/train_per_sec_per_gpu": 41.67, "tokens/trainable": 233677} +{"epoch": 1.3515625, "grad_norm": 1.4166926145553589, "learning_rate": 3.3683214232914404e-05, "loss": 0.14409665763378143, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 38.53, "memory/max_allocated (GiB)": 38.53, "ppl": 1.155, "step": 346, "tokens/total": 11946320, "tokens/train_per_sec_per_gpu": 41.6, "tokens/trainable": 234419} +{"epoch": 1.35546875, "grad_norm": 0.6845806241035461, "learning_rate": 3.342800532426873e-05, "loss": 0.13724179565906525, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.14711, "step": 347, "tokens/total": 11981984, "tokens/train_per_sec_per_gpu": 38.39, "tokens/trainable": 235082} +{"epoch": 1.359375, "grad_norm": 6.243236541748047, "learning_rate": 3.317369411437484e-05, "loss": 0.1548599898815155, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 37.63, "memory/max_allocated (GiB)": 37.63, "ppl": 1.16749, "step": 348, "tokens/total": 12017600, "tokens/train_per_sec_per_gpu": 37.47, "tokens/trainable": 235752} +{"epoch": 1.36328125, "grad_norm": 0.9461839199066162, "learning_rate": 3.292029118616024e-05, "loss": 0.14639884233474731, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 36.87, "memory/max_allocated (GiB)": 36.87, "ppl": 1.15766, "step": 349, "tokens/total": 12050000, "tokens/train_per_sec_per_gpu": 33.35, "tokens/trainable": 236332} +{"epoch": 1.3671875, "grad_norm": 1.3973579406738281, "learning_rate": 3.266780708475511e-05, "loss": 0.14024898409843445, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.15056, "step": 350, "tokens/total": 12085120, "tokens/train_per_sec_per_gpu": 37.18, "tokens/trainable": 237023} +{"epoch": 1.37109375, "grad_norm": 1.1701587438583374, "learning_rate": 3.241625231705354e-05, "loss": 0.12456488609313965, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 1.13266, "step": 351, "tokens/total": 12120848, "tokens/train_per_sec_per_gpu": 41.15, "tokens/trainable": 237733} +{"epoch": 1.375, "grad_norm": 2.1457245349884033, "learning_rate": 3.216563735127618e-05, "loss": 0.1285148561000824, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 1.13714, "step": 352, "tokens/total": 12155568, "tokens/train_per_sec_per_gpu": 39.75, "tokens/trainable": 238472} +{"epoch": 1.37890625, "grad_norm": 0.7953864932060242, "learning_rate": 3.191597261653475e-05, "loss": 0.13013848662376404, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.13899, "step": 353, "tokens/total": 12191920, "tokens/train_per_sec_per_gpu": 39.47, "tokens/trainable": 239212} +{"epoch": 1.3828125, "grad_norm": 0.9108014106750488, "learning_rate": 3.166726850239794e-05, "loss": 0.1289576292037964, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.13764, "step": 354, "tokens/total": 12225200, "tokens/train_per_sec_per_gpu": 43.08, "tokens/trainable": 239899} +{"epoch": 1.38671875, "grad_norm": 1.1639997959136963, "learning_rate": 3.141953535845912e-05, "loss": 0.16053161025047302, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.06, "memory/max_allocated (GiB)": 36.06, "ppl": 1.17413, "step": 355, "tokens/total": 12255744, "tokens/train_per_sec_per_gpu": 39.29, "tokens/trainable": 240488} +{"epoch": 1.390625, "grad_norm": 0.9839311242103577, "learning_rate": 3.11727834939056e-05, "loss": 0.14413416385650635, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.14, "memory/max_allocated (GiB)": 36.14, "ppl": 1.15504, "step": 356, "tokens/total": 12287232, "tokens/train_per_sec_per_gpu": 37.52, "tokens/trainable": 241164} +{"epoch": 1.39453125, "grad_norm": 6.369447708129883, "learning_rate": 3.092702317708967e-05, "loss": 0.16544750332832336, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.17992, "step": 357, "tokens/total": 12320656, "tokens/train_per_sec_per_gpu": 40.88, "tokens/trainable": 241833} +{"epoch": 1.3984375, "grad_norm": 7.339127540588379, "learning_rate": 3.0682264635101276e-05, "loss": 0.14340008795261383, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.15419, "step": 358, "tokens/total": 12352816, "tokens/train_per_sec_per_gpu": 40.81, "tokens/trainable": 242465} +{"epoch": 1.40234375, "grad_norm": 0.7515609264373779, "learning_rate": 3.0438518053342407e-05, "loss": 0.1291438490152359, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.13785, "step": 359, "tokens/total": 12391632, "tokens/train_per_sec_per_gpu": 37.27, "tokens/trainable": 243224} +{"epoch": 1.40625, "grad_norm": 0.9648094177246094, "learning_rate": 3.0195793575103266e-05, "loss": 0.12634816765785217, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.13468, "step": 360, "tokens/total": 12430256, "tokens/train_per_sec_per_gpu": 34.32, "tokens/trainable": 243915} +{"epoch": 1.41015625, "grad_norm": 2.4912149906158447, "learning_rate": 2.9954101301140146e-05, "loss": 0.1495080590248108, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.16126, "step": 361, "tokens/total": 12465248, "tokens/train_per_sec_per_gpu": 38.37, "tokens/trainable": 244592} +{"epoch": 1.4140625, "grad_norm": 0.8474061489105225, "learning_rate": 2.9713451289255123e-05, "loss": 0.12328501790761948, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.79, "memory/max_allocated (GiB)": 36.79, "ppl": 1.13121, "step": 362, "tokens/total": 12494320, "tokens/train_per_sec_per_gpu": 38.27, "tokens/trainable": 245261} +{"epoch": 1.41796875, "grad_norm": 1.9475343227386475, "learning_rate": 2.9473853553877484e-05, "loss": 0.13719063997268677, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.22, "memory/max_allocated (GiB)": 38.22, "ppl": 1.14705, "step": 363, "tokens/total": 12527840, "tokens/train_per_sec_per_gpu": 38.78, "tokens/trainable": 245921} +{"epoch": 1.421875, "grad_norm": 3.0144381523132324, "learning_rate": 2.9235318065647e-05, "loss": 0.16630756855010986, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.52, "memory/max_allocated (GiB)": 39.52, "ppl": 1.18094, "step": 364, "tokens/total": 12567536, "tokens/train_per_sec_per_gpu": 34.41, "tokens/trainable": 246578} +{"epoch": 1.42578125, "grad_norm": 1.4151420593261719, "learning_rate": 2.8997854750998964e-05, "loss": 0.14442357420921326, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.35, "memory/max_allocated (GiB)": 39.35, "ppl": 1.15537, "step": 365, "tokens/total": 12606800, "tokens/train_per_sec_per_gpu": 38.79, "tokens/trainable": 247292} +{"epoch": 1.4296875, "grad_norm": 0.6946248412132263, "learning_rate": 2.8761473491751258e-05, "loss": 0.1317523717880249, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.44, "memory/max_allocated (GiB)": 36.44, "ppl": 1.14083, "step": 366, "tokens/total": 12639232, "tokens/train_per_sec_per_gpu": 39.34, "tokens/trainable": 247936} +{"epoch": 1.43359375, "grad_norm": 0.7476474642753601, "learning_rate": 2.8526184124692883e-05, "loss": 0.10814348608255386, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.11421, "step": 367, "tokens/total": 12675504, "tokens/train_per_sec_per_gpu": 31.49, "tokens/trainable": 248653} +{"epoch": 1.4375, "grad_norm": 1.2106685638427734, "learning_rate": 2.829199644117484e-05, "loss": 0.14033092558383942, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 1.15065, "step": 368, "tokens/total": 12708768, "tokens/train_per_sec_per_gpu": 40.55, "tokens/trainable": 249258} +{"epoch": 1.44140625, "grad_norm": 0.9788193106651306, "learning_rate": 2.8058920186702553e-05, "loss": 0.11549718677997589, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.12243, "step": 369, "tokens/total": 12746448, "tokens/train_per_sec_per_gpu": 41.12, "tokens/trainable": 249989} +{"epoch": 1.4453125, "grad_norm": 1.1521722078323364, "learning_rate": 2.782696506053033e-05, "loss": 0.14565299451351166, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.15679, "step": 370, "tokens/total": 12784336, "tokens/train_per_sec_per_gpu": 33.72, "tokens/trainable": 250623} +{"epoch": 1.44921875, "grad_norm": 0.8872519135475159, "learning_rate": 2.7596140715257824e-05, "loss": 0.131196066737175, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.66, "memory/max_allocated (GiB)": 38.66, "ppl": 1.14019, "step": 371, "tokens/total": 12819024, "tokens/train_per_sec_per_gpu": 38.22, "tokens/trainable": 251305} +{"epoch": 1.453125, "grad_norm": 1.90530526638031, "learning_rate": 2.7366456756428184e-05, "loss": 0.15106815099716187, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.16308, "step": 372, "tokens/total": 12853024, "tokens/train_per_sec_per_gpu": 38.27, "tokens/trainable": 251961} +{"epoch": 1.45703125, "grad_norm": 0.7991819381713867, "learning_rate": 2.7137922742128486e-05, "loss": 0.11014306545257568, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.06, "memory/max_allocated (GiB)": 38.06, "ppl": 1.11644, "step": 373, "tokens/total": 12887584, "tokens/train_per_sec_per_gpu": 42.72, "tokens/trainable": 252692} +{"epoch": 1.4609375, "grad_norm": 0.5690158009529114, "learning_rate": 2.691054818259188e-05, "loss": 0.11537184566259384, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.31, "memory/max_allocated (GiB)": 37.31, "ppl": 1.12229, "step": 374, "tokens/total": 12921904, "tokens/train_per_sec_per_gpu": 41.2, "tokens/trainable": 253437} +{"epoch": 1.46484375, "grad_norm": 1.2122048139572144, "learning_rate": 2.6684342539801933e-05, "loss": 0.15561836957931519, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.37, "memory/max_allocated (GiB)": 36.37, "ppl": 1.16838, "step": 375, "tokens/total": 12952928, "tokens/train_per_sec_per_gpu": 39.17, "tokens/trainable": 254067} +{"epoch": 1.46875, "grad_norm": 1.4339848756790161, "learning_rate": 2.645931522709877e-05, "loss": 0.12129739671945572, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.12896, "step": 376, "tokens/total": 12988336, "tokens/train_per_sec_per_gpu": 50.02, "tokens/trainable": 254847} +{"epoch": 1.47265625, "grad_norm": 0.8946370482444763, "learning_rate": 2.6235475608787365e-05, "loss": 0.1353950798511505, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.14499, "step": 377, "tokens/total": 13023744, "tokens/train_per_sec_per_gpu": 38.16, "tokens/trainable": 255506} +{"epoch": 1.4765625, "grad_norm": 1.5082982778549194, "learning_rate": 2.6012832999747916e-05, "loss": 0.1462072730064392, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.31, "memory/max_allocated (GiB)": 36.31, "ppl": 1.15744, "step": 378, "tokens/total": 13054272, "tokens/train_per_sec_per_gpu": 39.66, "tokens/trainable": 256140} +{"epoch": 1.48046875, "grad_norm": 1.4614781141281128, "learning_rate": 2.579139666504821e-05, "loss": 0.15329323709011078, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.16567, "step": 379, "tokens/total": 13093952, "tokens/train_per_sec_per_gpu": 35.07, "tokens/trainable": 256808} +{"epoch": 1.484375, "grad_norm": 1.11962890625, "learning_rate": 2.557117581955798e-05, "loss": 0.12831810116767883, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.13691, "step": 380, "tokens/total": 13127392, "tokens/train_per_sec_per_gpu": 38.86, "tokens/trainable": 257510} +{"epoch": 1.48828125, "grad_norm": 1.7412225008010864, "learning_rate": 2.5352179627565532e-05, "loss": 0.14752717316150665, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.97, "memory/max_allocated (GiB)": 37.97, "ppl": 1.15896, "step": 381, "tokens/total": 13162064, "tokens/train_per_sec_per_gpu": 35.49, "tokens/trainable": 258181} +{"epoch": 1.4921875, "grad_norm": 1.4267795085906982, "learning_rate": 2.5134417202396277e-05, "loss": 0.17457996308803558, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.45, "memory/max_allocated (GiB)": 38.45, "ppl": 1.19075, "step": 382, "tokens/total": 13195408, "tokens/train_per_sec_per_gpu": 35.27, "tokens/trainable": 258790} +{"epoch": 1.49609375, "grad_norm": 1.5573283433914185, "learning_rate": 2.491789760603361e-05, "loss": 0.13884413242340088, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.04, "memory/max_allocated (GiB)": 39.04, "ppl": 1.14895, "step": 383, "tokens/total": 13231504, "tokens/train_per_sec_per_gpu": 35.15, "tokens/trainable": 259436} +{"epoch": 1.5, "grad_norm": 0.9388878345489502, "learning_rate": 2.4702629848741764e-05, "loss": 0.15367519855499268, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.16611, "step": 384, "tokens/total": 13265728, "tokens/train_per_sec_per_gpu": 38.67, "tokens/trainable": 260092} +{"epoch": 1.50390625, "grad_norm": 0.8710785508155823, "learning_rate": 2.4488622888690785e-05, "loss": 0.13195864856243134, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.14106, "step": 385, "tokens/total": 13299008, "tokens/train_per_sec_per_gpu": 38.06, "tokens/trainable": 260721} +{"epoch": 1.5078125, "grad_norm": 0.7580258846282959, "learning_rate": 2.427588563158384e-05, "loss": 0.1079956591129303, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.11404, "step": 386, "tokens/total": 13332976, "tokens/train_per_sec_per_gpu": 41.33, "tokens/trainable": 261455} +{"epoch": 1.51171875, "grad_norm": 1.2576922178268433, "learning_rate": 2.406442693028651e-05, "loss": 0.126967191696167, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.34, "memory/max_allocated (GiB)": 38.34, "ppl": 1.13538, "step": 387, "tokens/total": 13365056, "tokens/train_per_sec_per_gpu": 38.61, "tokens/trainable": 262087} +{"epoch": 1.515625, "grad_norm": 0.8002811670303345, "learning_rate": 2.3854255584458547e-05, "loss": 0.13866209983825684, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.14874, "step": 388, "tokens/total": 13403584, "tokens/train_per_sec_per_gpu": 33.57, "tokens/trainable": 262747} +{"epoch": 1.51953125, "grad_norm": 1.4194988012313843, "learning_rate": 2.3645380340187508e-05, "loss": 0.11972780525684357, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.12719, "step": 389, "tokens/total": 13438816, "tokens/train_per_sec_per_gpu": 40.98, "tokens/trainable": 263450} +{"epoch": 1.5234375, "grad_norm": 1.2423032522201538, "learning_rate": 2.3437809889624914e-05, "loss": 0.15478338301181793, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.52, "memory/max_allocated (GiB)": 37.52, "ppl": 1.16741, "step": 390, "tokens/total": 13471088, "tokens/train_per_sec_per_gpu": 43.73, "tokens/trainable": 264107} +{"epoch": 1.52734375, "grad_norm": 0.8298978209495544, "learning_rate": 2.3231552870624487e-05, "loss": 0.1324734091758728, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.32, "memory/max_allocated (GiB)": 38.32, "ppl": 1.14165, "step": 391, "tokens/total": 13501664, "tokens/train_per_sec_per_gpu": 47.1, "tokens/trainable": 264813} +{"epoch": 1.53125, "grad_norm": 1.1499406099319458, "learning_rate": 2.3026617866382657e-05, "loss": 0.11602560430765152, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.54, "memory/max_allocated (GiB)": 36.54, "ppl": 1.12302, "step": 392, "tokens/total": 13534320, "tokens/train_per_sec_per_gpu": 40.5, "tokens/trainable": 265544} +{"epoch": 1.53515625, "grad_norm": 1.3051742315292358, "learning_rate": 2.2823013405081507e-05, "loss": 0.1249900758266449, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.14, "memory/max_allocated (GiB)": 39.14, "ppl": 1.13314, "step": 393, "tokens/total": 13572240, "tokens/train_per_sec_per_gpu": 37.1, "tokens/trainable": 266264} +{"epoch": 1.5390625, "grad_norm": 0.6858569979667664, "learning_rate": 2.2620747959533722e-05, "loss": 0.11599761247634888, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.12299, "step": 394, "tokens/total": 13605232, "tokens/train_per_sec_per_gpu": 44.6, "tokens/trainable": 267007} +{"epoch": 1.54296875, "grad_norm": 0.5886895060539246, "learning_rate": 2.2419829946830123e-05, "loss": 0.12114224582910538, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.12879, "step": 395, "tokens/total": 13640688, "tokens/train_per_sec_per_gpu": 39.45, "tokens/trainable": 267674} +{"epoch": 1.546875, "grad_norm": 0.9844586253166199, "learning_rate": 2.2220267727989325e-05, "loss": 0.12712392210960388, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.13556, "step": 396, "tokens/total": 13674848, "tokens/train_per_sec_per_gpu": 38.38, "tokens/trainable": 268356} +{"epoch": 1.55078125, "grad_norm": 0.7101472020149231, "learning_rate": 2.202206960760984e-05, "loss": 0.11450912803411484, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.12132, "step": 397, "tokens/total": 13712752, "tokens/train_per_sec_per_gpu": 39.22, "tokens/trainable": 269135} +{"epoch": 1.5546875, "grad_norm": 0.5610458254814148, "learning_rate": 2.182524383352446e-05, "loss": 0.12109773606061935, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.83, "memory/max_allocated (GiB)": 37.83, "ppl": 1.12874, "step": 398, "tokens/total": 13748592, "tokens/train_per_sec_per_gpu": 37.1, "tokens/trainable": 269803} +{"epoch": 1.55859375, "grad_norm": 1.0240525007247925, "learning_rate": 2.1629798596457056e-05, "loss": 0.13894930481910706, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.25, "memory/max_allocated (GiB)": 36.25, "ppl": 1.14907, "step": 399, "tokens/total": 13780432, "tokens/train_per_sec_per_gpu": 40.71, "tokens/trainable": 270451} +{"epoch": 1.5625, "grad_norm": 0.6726769208908081, "learning_rate": 2.1435742029681725e-05, "loss": 0.13464218378067017, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.14413, "step": 400, "tokens/total": 13812416, "tokens/train_per_sec_per_gpu": 46.16, "tokens/trainable": 271093} +{"epoch": 1.56640625, "grad_norm": 1.9160268306732178, "learning_rate": 2.124308220868431e-05, "loss": 0.13239578902721405, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.14156, "step": 401, "tokens/total": 13847888, "tokens/train_per_sec_per_gpu": 37.55, "tokens/trainable": 271778} +{"epoch": 1.5703125, "grad_norm": 0.8476623296737671, "learning_rate": 2.105182715082638e-05, "loss": 0.127031147480011, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.76, "memory/max_allocated (GiB)": 36.76, "ppl": 1.13545, "step": 402, "tokens/total": 13881296, "tokens/train_per_sec_per_gpu": 41.47, "tokens/trainable": 272497} +{"epoch": 1.57421875, "grad_norm": 0.9293152689933777, "learning_rate": 2.0861984815011552e-05, "loss": 0.1465875506401062, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.15788, "step": 403, "tokens/total": 13916144, "tokens/train_per_sec_per_gpu": 35.26, "tokens/trainable": 273102} +{"epoch": 1.578125, "grad_norm": 1.4630426168441772, "learning_rate": 2.0673563101354323e-05, "loss": 0.13229748606681824, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.14145, "step": 404, "tokens/total": 13951136, "tokens/train_per_sec_per_gpu": 32.61, "tokens/trainable": 273732} +{"epoch": 1.58203125, "grad_norm": 1.531042456626892, "learning_rate": 2.0486569850851317e-05, "loss": 0.13916780054569244, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.17, "memory/max_allocated (GiB)": 38.17, "ppl": 1.14932, "step": 405, "tokens/total": 13985664, "tokens/train_per_sec_per_gpu": 39.04, "tokens/trainable": 274392} +{"epoch": 1.5859375, "grad_norm": 0.9501775503158569, "learning_rate": 2.0301012845054956e-05, "loss": 0.1289280652999878, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.13761, "step": 406, "tokens/total": 14024208, "tokens/train_per_sec_per_gpu": 33.83, "tokens/trainable": 275042} +{"epoch": 1.58984375, "grad_norm": 0.5587449073791504, "learning_rate": 2.011689980574966e-05, "loss": 0.12116235494613647, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.12881, "step": 407, "tokens/total": 14060176, "tokens/train_per_sec_per_gpu": 35.62, "tokens/trainable": 275698} +{"epoch": 1.59375, "grad_norm": 0.6242634057998657, "learning_rate": 1.993423839463052e-05, "loss": 0.11582303047180176, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.17, "memory/max_allocated (GiB)": 38.17, "ppl": 1.1228, "step": 408, "tokens/total": 14093232, "tokens/train_per_sec_per_gpu": 51.28, "tokens/trainable": 276403} +{"epoch": 1.59765625, "grad_norm": 0.6115993857383728, "learning_rate": 1.975303621298445e-05, "loss": 0.11378955841064453, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.05, "memory/max_allocated (GiB)": 39.05, "ppl": 1.12052, "step": 409, "tokens/total": 14129648, "tokens/train_per_sec_per_gpu": 42.2, "tokens/trainable": 277103} +{"epoch": 1.6015625, "grad_norm": 1.3159042596817017, "learning_rate": 1.957330080137385e-05, "loss": 0.14435002207756042, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.09, "memory/max_allocated (GiB)": 36.09, "ppl": 1.15529, "step": 410, "tokens/total": 14159872, "tokens/train_per_sec_per_gpu": 40.88, "tokens/trainable": 277755} +{"epoch": 1.60546875, "grad_norm": 0.5521350502967834, "learning_rate": 1.9395039639322864e-05, "loss": 0.12117935717105865, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.12883, "step": 411, "tokens/total": 14195584, "tokens/train_per_sec_per_gpu": 39.12, "tokens/trainable": 278445} +{"epoch": 1.609375, "grad_norm": 2.2714762687683105, "learning_rate": 1.9218260145006073e-05, "loss": 0.12670810520648956, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.61, "memory/max_allocated (GiB)": 37.61, "ppl": 1.13509, "step": 412, "tokens/total": 14231232, "tokens/train_per_sec_per_gpu": 43.55, "tokens/trainable": 279201} +{"epoch": 1.61328125, "grad_norm": 0.611945629119873, "learning_rate": 1.904296967493982e-05, "loss": 0.11803490668535233, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.12528, "step": 413, "tokens/total": 14263392, "tokens/train_per_sec_per_gpu": 46.47, "tokens/trainable": 279864} +{"epoch": 1.6171875, "grad_norm": 1.7701760530471802, "learning_rate": 1.8869175523676064e-05, "loss": 0.11760139465332031, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.2, "memory/max_allocated (GiB)": 38.2, "ppl": 1.1248, "step": 414, "tokens/total": 14299760, "tokens/train_per_sec_per_gpu": 39.79, "tokens/trainable": 280595} +{"epoch": 1.62109375, "grad_norm": 0.51682448387146, "learning_rate": 1.869688492349885e-05, "loss": 0.13004815578460693, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.13888, "step": 415, "tokens/total": 14332640, "tokens/train_per_sec_per_gpu": 39.8, "tokens/trainable": 281259} +{"epoch": 1.625, "grad_norm": 0.7345198392868042, "learning_rate": 1.85261050441233e-05, "loss": 0.12973254919052124, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.13852, "step": 416, "tokens/total": 14368336, "tokens/train_per_sec_per_gpu": 38.12, "tokens/trainable": 281959} +{"epoch": 1.62890625, "grad_norm": 2.229914426803589, "learning_rate": 1.8356842992397304e-05, "loss": 0.15756480395793915, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.06, "memory/max_allocated (GiB)": 37.06, "ppl": 1.17066, "step": 417, "tokens/total": 14401760, "tokens/train_per_sec_per_gpu": 39.0, "tokens/trainable": 282595} +{"epoch": 1.6328125, "grad_norm": 0.7740523815155029, "learning_rate": 1.8189105812005714e-05, "loss": 0.14196214079856873, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.15253, "step": 418, "tokens/total": 14438064, "tokens/train_per_sec_per_gpu": 32.0, "tokens/trainable": 283158} +{"epoch": 1.63671875, "grad_norm": 0.9296337962150574, "learning_rate": 1.802290048317732e-05, "loss": 0.12450791895389557, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.13259, "step": 419, "tokens/total": 14471312, "tokens/train_per_sec_per_gpu": 38.1, "tokens/trainable": 283801} +{"epoch": 1.640625, "grad_norm": 0.721524178981781, "learning_rate": 1.785823392239424e-05, "loss": 0.1166575476527214, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.67, "memory/max_allocated (GiB)": 37.67, "ppl": 1.12373, "step": 420, "tokens/total": 14504160, "tokens/train_per_sec_per_gpu": 37.08, "tokens/trainable": 284404} +{"epoch": 1.64453125, "grad_norm": 1.0734999179840088, "learning_rate": 1.7695112982104225e-05, "loss": 0.11722557991743088, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.61, "memory/max_allocated (GiB)": 39.61, "ppl": 1.12437, "step": 421, "tokens/total": 14539040, "tokens/train_per_sec_per_gpu": 40.3, "tokens/trainable": 285093} +{"epoch": 1.6484375, "grad_norm": 0.7498980164527893, "learning_rate": 1.7533544450435433e-05, "loss": 0.13179859519004822, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 35.55, "memory/max_allocated (GiB)": 35.55, "ppl": 1.14088, "step": 422, "tokens/total": 14568080, "tokens/train_per_sec_per_gpu": 50.8, "tokens/trainable": 285758} +{"epoch": 1.65234375, "grad_norm": 0.5999062061309814, "learning_rate": 1.7373535050913946e-05, "loss": 0.12882399559020996, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.7, "memory/max_allocated (GiB)": 38.7, "ppl": 1.13749, "step": 423, "tokens/total": 14604128, "tokens/train_per_sec_per_gpu": 32.17, "tokens/trainable": 286429} +{"epoch": 1.65625, "grad_norm": 0.8860400915145874, "learning_rate": 1.721509144218405e-05, "loss": 0.1263880729675293, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 35.57, "memory/max_allocated (GiB)": 35.57, "ppl": 1.13472, "step": 424, "tokens/total": 14633776, "tokens/train_per_sec_per_gpu": 47.89, "tokens/trainable": 287068} +{"epoch": 1.66015625, "grad_norm": 0.5874081254005432, "learning_rate": 1.705822021773101e-05, "loss": 0.12035386264324188, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.1279, "step": 425, "tokens/total": 14667856, "tokens/train_per_sec_per_gpu": 40.15, "tokens/trainable": 287729} +{"epoch": 1.6640625, "grad_norm": 1.2760412693023682, "learning_rate": 1.69029279056068e-05, "loss": 0.14847928285598755, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.16007, "step": 426, "tokens/total": 14704944, "tokens/train_per_sec_per_gpu": 37.1, "tokens/trainable": 288402} +{"epoch": 1.66796875, "grad_norm": 0.9880636930465698, "learning_rate": 1.6749220968158415e-05, "loss": 0.11454540491104126, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.19, "memory/max_allocated (GiB)": 39.19, "ppl": 1.12136, "step": 427, "tokens/total": 14739616, "tokens/train_per_sec_per_gpu": 41.75, "tokens/trainable": 289136} +{"epoch": 1.671875, "grad_norm": 1.948601484298706, "learning_rate": 1.659710580175893e-05, "loss": 0.1270180344581604, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 35.75, "memory/max_allocated (GiB)": 35.75, "ppl": 1.13544, "step": 428, "tokens/total": 14770096, "tokens/train_per_sec_per_gpu": 44.52, "tokens/trainable": 289830} +{"epoch": 1.67578125, "grad_norm": 0.6320269703865051, "learning_rate": 1.644658873654133e-05, "loss": 0.1136716976761818, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.12038, "step": 429, "tokens/total": 14802240, "tokens/train_per_sec_per_gpu": 44.4, "tokens/trainable": 290509} +{"epoch": 1.6796875, "grad_norm": 0.9000123143196106, "learning_rate": 1.629767603613508e-05, "loss": 0.14817653596401215, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 35.97, "memory/max_allocated (GiB)": 35.97, "ppl": 1.15972, "step": 430, "tokens/total": 14832768, "tokens/train_per_sec_per_gpu": 38.42, "tokens/trainable": 291089} +{"epoch": 1.68359375, "grad_norm": 0.7712384462356567, "learning_rate": 1.615037389740547e-05, "loss": 0.1295904517173767, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.53, "memory/max_allocated (GiB)": 37.53, "ppl": 1.13836, "step": 431, "tokens/total": 14867472, "tokens/train_per_sec_per_gpu": 36.83, "tokens/trainable": 291783} +{"epoch": 1.6875, "grad_norm": 1.171974778175354, "learning_rate": 1.600468845019576e-05, "loss": 0.12837469577789307, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.13698, "step": 432, "tokens/total": 14905344, "tokens/train_per_sec_per_gpu": 35.26, "tokens/trainable": 292468} +{"epoch": 1.69140625, "grad_norm": 0.9614337682723999, "learning_rate": 1.5860625757072092e-05, "loss": 0.133927121758461, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.33, "memory/max_allocated (GiB)": 38.33, "ppl": 1.14331, "step": 433, "tokens/total": 14939936, "tokens/train_per_sec_per_gpu": 33.1, "tokens/trainable": 293090} +{"epoch": 1.6953125, "grad_norm": 0.6659449934959412, "learning_rate": 1.571819181307116e-05, "loss": 0.10751074552536011, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 1.1135, "step": 434, "tokens/total": 14976608, "tokens/train_per_sec_per_gpu": 44.21, "tokens/trainable": 293854} +{"epoch": 1.69921875, "grad_norm": 1.4004520177841187, "learning_rate": 1.557739254545075e-05, "loss": 0.11853499710559845, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.33, "memory/max_allocated (GiB)": 36.33, "ppl": 1.12585, "step": 435, "tokens/total": 15007376, "tokens/train_per_sec_per_gpu": 47.23, "tokens/trainable": 294527} +{"epoch": 1.703125, "grad_norm": 1.3230053186416626, "learning_rate": 1.543823381344311e-05, "loss": 0.12673166394233704, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.13511, "step": 436, "tokens/total": 15039680, "tokens/train_per_sec_per_gpu": 48.81, "tokens/trainable": 295240} +{"epoch": 1.70703125, "grad_norm": 0.6435824036598206, "learning_rate": 1.5300721408011114e-05, "loss": 0.1265442669391632, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.1349, "step": 437, "tokens/total": 15074768, "tokens/train_per_sec_per_gpu": 38.76, "tokens/trainable": 295896} +{"epoch": 1.7109375, "grad_norm": 1.4225972890853882, "learning_rate": 1.5164861051607254e-05, "loss": 0.15245428681373596, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.99, "memory/max_allocated (GiB)": 38.99, "ppl": 1.16469, "step": 438, "tokens/total": 15110448, "tokens/train_per_sec_per_gpu": 30.45, "tokens/trainable": 296464} +{"epoch": 1.71484375, "grad_norm": 1.0704902410507202, "learning_rate": 1.5030658397935521e-05, "loss": 0.12510716915130615, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.15, "memory/max_allocated (GiB)": 38.15, "ppl": 1.13327, "step": 439, "tokens/total": 15145664, "tokens/train_per_sec_per_gpu": 38.84, "tokens/trainable": 297160} +{"epoch": 1.71875, "grad_norm": 0.8005355000495911, "learning_rate": 1.4898119031716104e-05, "loss": 0.1183292344212532, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.12561, "step": 440, "tokens/total": 15178320, "tokens/train_per_sec_per_gpu": 43.7, "tokens/trainable": 297827} +{"epoch": 1.72265625, "grad_norm": 0.8378676772117615, "learning_rate": 1.476724846845306e-05, "loss": 0.12060844153165817, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.12818, "step": 441, "tokens/total": 15216224, "tokens/train_per_sec_per_gpu": 36.31, "tokens/trainable": 298561} +{"epoch": 1.7265625, "grad_norm": 0.842519998550415, "learning_rate": 1.463805215420471e-05, "loss": 0.11466747522354126, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.1215, "step": 442, "tokens/total": 15251408, "tokens/train_per_sec_per_gpu": 41.02, "tokens/trainable": 299284} +{"epoch": 1.73046875, "grad_norm": 0.9992321729660034, "learning_rate": 1.451053546535705e-05, "loss": 0.13079410791397095, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.13973, "step": 443, "tokens/total": 15287952, "tokens/train_per_sec_per_gpu": 36.11, "tokens/trainable": 299935} +{"epoch": 1.734375, "grad_norm": 0.9518084526062012, "learning_rate": 1.438470370840001e-05, "loss": 0.11100472509860992, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.1174, "step": 444, "tokens/total": 15323968, "tokens/train_per_sec_per_gpu": 41.62, "tokens/trainable": 300687} +{"epoch": 1.73828125, "grad_norm": 1.8722642660140991, "learning_rate": 1.4260562119706606e-05, "loss": 0.11504357308149338, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.5, "memory/max_allocated (GiB)": 37.5, "ppl": 1.12192, "step": 445, "tokens/total": 15356000, "tokens/train_per_sec_per_gpu": 47.93, "tokens/trainable": 301392} +{"epoch": 1.7421875, "grad_norm": 1.1691895723342896, "learning_rate": 1.413811586531508e-05, "loss": 0.11995839327573776, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.12745, "step": 446, "tokens/total": 15389008, "tokens/train_per_sec_per_gpu": 43.98, "tokens/trainable": 302076} +{"epoch": 1.74609375, "grad_norm": 5.5844926834106445, "learning_rate": 1.4017370040713884e-05, "loss": 0.14268262684345245, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.2, "memory/max_allocated (GiB)": 38.2, "ppl": 1.15336, "step": 447, "tokens/total": 15425808, "tokens/train_per_sec_per_gpu": 33.46, "tokens/trainable": 302725} +{"epoch": 1.75, "grad_norm": 1.6685824394226074, "learning_rate": 1.3898329670629645e-05, "loss": 0.15038934350013733, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.59, "memory/max_allocated (GiB)": 39.59, "ppl": 1.16229, "step": 448, "tokens/total": 15465952, "tokens/train_per_sec_per_gpu": 32.49, "tokens/trainable": 303389} +{"epoch": 1.75390625, "grad_norm": 0.6182010173797607, "learning_rate": 1.3780999708818058e-05, "loss": 0.12641456723213196, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.14, "memory/max_allocated (GiB)": 36.14, "ppl": 1.13475, "step": 449, "tokens/total": 15496784, "tokens/train_per_sec_per_gpu": 40.21, "tokens/trainable": 304039} +{"epoch": 1.7578125, "grad_norm": 0.9323810935020447, "learning_rate": 1.3665385037857758e-05, "loss": 0.12307095527648926, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.73, "memory/max_allocated (GiB)": 36.73, "ppl": 1.13096, "step": 450, "tokens/total": 15529600, "tokens/train_per_sec_per_gpu": 41.04, "tokens/trainable": 304715} +{"epoch": 1.76171875, "grad_norm": 0.6976091265678406, "learning_rate": 1.3551490468947126e-05, "loss": 0.12068628519773483, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.28, "memory/max_allocated (GiB)": 36.28, "ppl": 1.12827, "step": 451, "tokens/total": 15561168, "tokens/train_per_sec_per_gpu": 42.4, "tokens/trainable": 305388} +{"epoch": 1.765625, "grad_norm": 1.509114384651184, "learning_rate": 1.3439320741704075e-05, "loss": 0.15474215149879456, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.16736, "step": 452, "tokens/total": 15595664, "tokens/train_per_sec_per_gpu": 42.39, "tokens/trainable": 306061} +{"epoch": 1.76953125, "grad_norm": 0.9118852615356445, "learning_rate": 1.3328880523968808e-05, "loss": 0.12850311398506165, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.87, "memory/max_allocated (GiB)": 36.87, "ppl": 1.13712, "step": 453, "tokens/total": 15628672, "tokens/train_per_sec_per_gpu": 38.35, "tokens/trainable": 306679} +{"epoch": 1.7734375, "grad_norm": 0.6585202217102051, "learning_rate": 1.3220174411609587e-05, "loss": 0.11812008172273636, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.12538, "step": 454, "tokens/total": 15662192, "tokens/train_per_sec_per_gpu": 44.47, "tokens/trainable": 307377} +{"epoch": 1.77734375, "grad_norm": 0.5155206322669983, "learning_rate": 1.3113206928331471e-05, "loss": 0.11802934110164642, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 35.13, "memory/max_allocated (GiB)": 35.13, "ppl": 1.12528, "step": 455, "tokens/total": 15690320, "tokens/train_per_sec_per_gpu": 48.58, "tokens/trainable": 308066} +{"epoch": 1.78125, "grad_norm": 0.6537501811981201, "learning_rate": 1.300798252548806e-05, "loss": 0.11581052839756012, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.12278, "step": 456, "tokens/total": 15724576, "tokens/train_per_sec_per_gpu": 40.8, "tokens/trainable": 308725} +{"epoch": 1.78515625, "grad_norm": 0.8916272521018982, "learning_rate": 1.2904505581896265e-05, "loss": 0.1230795681476593, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.13097, "step": 457, "tokens/total": 15763536, "tokens/train_per_sec_per_gpu": 32.69, "tokens/trainable": 309451} +{"epoch": 1.7890625, "grad_norm": 1.760952353477478, "learning_rate": 1.2802780403654082e-05, "loss": 0.1427900642156601, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.58, "memory/max_allocated (GiB)": 37.58, "ppl": 1.15349, "step": 458, "tokens/total": 15796720, "tokens/train_per_sec_per_gpu": 40.31, "tokens/trainable": 310068} +{"epoch": 1.79296875, "grad_norm": 0.7674922943115234, "learning_rate": 1.2702811223961408e-05, "loss": 0.12612786889076233, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.13443, "step": 459, "tokens/total": 15830096, "tokens/train_per_sec_per_gpu": 39.42, "tokens/trainable": 310728} +{"epoch": 1.796875, "grad_norm": 0.9945017695426941, "learning_rate": 1.2604602202943861e-05, "loss": 0.13204723596572876, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.14116, "step": 460, "tokens/total": 15860656, "tokens/train_per_sec_per_gpu": 47.05, "tokens/trainable": 311387} +{"epoch": 1.80078125, "grad_norm": 0.5268994569778442, "learning_rate": 1.2508157427479686e-05, "loss": 0.10993121564388275, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.1162, "step": 461, "tokens/total": 15894192, "tokens/train_per_sec_per_gpu": 40.12, "tokens/trainable": 312083} +{"epoch": 1.8046875, "grad_norm": 0.6594498157501221, "learning_rate": 1.2413480911029655e-05, "loss": 0.12765124440193176, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 35.78, "memory/max_allocated (GiB)": 35.78, "ppl": 1.13616, "step": 462, "tokens/total": 15924576, "tokens/train_per_sec_per_gpu": 36.68, "tokens/trainable": 312685} +{"epoch": 1.80859375, "grad_norm": 0.6051502823829651, "learning_rate": 1.2320576593470082e-05, "loss": 0.13214725255966187, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.14128, "step": 463, "tokens/total": 15958816, "tokens/train_per_sec_per_gpu": 37.53, "tokens/trainable": 313319} +{"epoch": 1.8125, "grad_norm": 1.1617094278335571, "learning_rate": 1.2229448340928828e-05, "loss": 0.10645001381635666, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.1, "memory/max_allocated (GiB)": 39.1, "ppl": 1.11232, "step": 464, "tokens/total": 15997968, "tokens/train_per_sec_per_gpu": 36.44, "tokens/trainable": 314093} +{"epoch": 1.81640625, "grad_norm": 0.6464188694953918, "learning_rate": 1.2140099945624458e-05, "loss": 0.11223287880420685, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.72, "memory/max_allocated (GiB)": 38.72, "ppl": 1.11877, "step": 465, "tokens/total": 16036512, "tokens/train_per_sec_per_gpu": 37.01, "tokens/trainable": 314816} +{"epoch": 1.8203125, "grad_norm": 1.6812291145324707, "learning_rate": 1.205253512570841e-05, "loss": 0.14114481210708618, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.15159, "step": 466, "tokens/total": 16068944, "tokens/train_per_sec_per_gpu": 40.99, "tokens/trainable": 315456} +{"epoch": 1.82421875, "grad_norm": 0.8703078031539917, "learning_rate": 1.1966757525110255e-05, "loss": 0.14144688844680786, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 35.26, "memory/max_allocated (GiB)": 35.26, "ppl": 1.15194, "step": 467, "tokens/total": 16098048, "tokens/train_per_sec_per_gpu": 47.17, "tokens/trainable": 316095} +{"epoch": 1.828125, "grad_norm": 0.6636398434638977, "learning_rate": 1.1882770713386095e-05, "loss": 0.12347720563411713, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.13142, "step": 468, "tokens/total": 16134416, "tokens/train_per_sec_per_gpu": 40.05, "tokens/trainable": 316800} +{"epoch": 1.83203125, "grad_norm": 0.5327708125114441, "learning_rate": 1.180057818556998e-05, "loss": 0.1109175831079483, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.1173, "step": 469, "tokens/total": 16170272, "tokens/train_per_sec_per_gpu": 39.37, "tokens/trainable": 317535} +{"epoch": 1.8359375, "grad_norm": 0.683586835861206, "learning_rate": 1.1720183362028494e-05, "loss": 0.1314796358346939, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.14051, "step": 470, "tokens/total": 16206288, "tokens/train_per_sec_per_gpu": 33.72, "tokens/trainable": 318178} +{"epoch": 1.83984375, "grad_norm": 0.4873957335948944, "learning_rate": 1.1641589588318387e-05, "loss": 0.1177339106798172, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.12494, "step": 471, "tokens/total": 16243856, "tokens/train_per_sec_per_gpu": 35.6, "tokens/trainable": 318848} +{"epoch": 1.84375, "grad_norm": 0.9969131946563721, "learning_rate": 1.1564800135047418e-05, "loss": 0.10530316829681396, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.9, "memory/max_allocated (GiB)": 36.9, "ppl": 1.11105, "step": 472, "tokens/total": 16277072, "tokens/train_per_sec_per_gpu": 43.54, "tokens/trainable": 319584} +{"epoch": 1.84765625, "grad_norm": 2.009190320968628, "learning_rate": 1.148981819773816e-05, "loss": 0.12241992354393005, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.12, "memory/max_allocated (GiB)": 37.12, "ppl": 1.13023, "step": 473, "tokens/total": 16309696, "tokens/train_per_sec_per_gpu": 34.34, "tokens/trainable": 320214} +{"epoch": 1.8515625, "grad_norm": 0.8279973268508911, "learning_rate": 1.1416646896695086e-05, "loss": 0.12112845480442047, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.12877, "step": 474, "tokens/total": 16343584, "tokens/train_per_sec_per_gpu": 42.43, "tokens/trainable": 320890} +{"epoch": 1.85546875, "grad_norm": 0.6757808923721313, "learning_rate": 1.1345289276874717e-05, "loss": 0.11932478845119476, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.34, "memory/max_allocated (GiB)": 38.34, "ppl": 1.12674, "step": 475, "tokens/total": 16380432, "tokens/train_per_sec_per_gpu": 39.58, "tokens/trainable": 321619} +{"epoch": 1.859375, "grad_norm": 4.006710529327393, "learning_rate": 1.1275748307758873e-05, "loss": 0.10687154531478882, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.11279, "step": 476, "tokens/total": 16415696, "tokens/train_per_sec_per_gpu": 41.52, "tokens/trainable": 322333} +{"epoch": 1.86328125, "grad_norm": 2.00041127204895, "learning_rate": 1.1208026883231147e-05, "loss": 0.13857470452785492, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.21, "memory/max_allocated (GiB)": 36.21, "ppl": 1.14864, "step": 477, "tokens/total": 16446496, "tokens/train_per_sec_per_gpu": 42.27, "tokens/trainable": 322911} +{"epoch": 1.8671875, "grad_norm": 0.6742245554924011, "learning_rate": 1.1142127821456433e-05, "loss": 0.11806903779506683, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 1.12532, "step": 478, "tokens/total": 16479664, "tokens/train_per_sec_per_gpu": 42.01, "tokens/trainable": 323579} +{"epoch": 1.87109375, "grad_norm": 0.916693925857544, "learning_rate": 1.1078053864763674e-05, "loss": 0.11659835278987885, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.67, "memory/max_allocated (GiB)": 37.67, "ppl": 1.12367, "step": 479, "tokens/total": 16515312, "tokens/train_per_sec_per_gpu": 40.75, "tokens/trainable": 324278} +{"epoch": 1.875, "grad_norm": 0.4687732756137848, "learning_rate": 1.1015807679531756e-05, "loss": 0.1207454651594162, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.23, "memory/max_allocated (GiB)": 37.23, "ppl": 1.12834, "step": 480, "tokens/total": 16546576, "tokens/train_per_sec_per_gpu": 44.27, "tokens/trainable": 324943} +{"epoch": 1.87890625, "grad_norm": 0.7472628355026245, "learning_rate": 1.0955391856078528e-05, "loss": 0.12402228266000748, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.36, "memory/max_allocated (GiB)": 38.36, "ppl": 1.13204, "step": 481, "tokens/total": 16582160, "tokens/train_per_sec_per_gpu": 41.51, "tokens/trainable": 325632} +{"epoch": 1.8828125, "grad_norm": 0.7102296352386475, "learning_rate": 1.0896808908553007e-05, "loss": 0.12926337122917175, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.13799, "step": 482, "tokens/total": 16616736, "tokens/train_per_sec_per_gpu": 35.05, "tokens/trainable": 326272} +{"epoch": 1.88671875, "grad_norm": 1.370229959487915, "learning_rate": 1.0840061274830763e-05, "loss": 0.12591677904129028, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 35.98, "memory/max_allocated (GiB)": 35.98, "ppl": 1.13419, "step": 483, "tokens/total": 16647680, "tokens/train_per_sec_per_gpu": 38.16, "tokens/trainable": 326900} +{"epoch": 1.890625, "grad_norm": 0.624413251876831, "learning_rate": 1.0785151316412473e-05, "loss": 0.11663030087947845, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.1237, "step": 484, "tokens/total": 16683472, "tokens/train_per_sec_per_gpu": 33.02, "tokens/trainable": 327589} +{"epoch": 1.89453125, "grad_norm": 0.6049356460571289, "learning_rate": 1.0732081318325639e-05, "loss": 0.10953962802886963, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.11576, "step": 485, "tokens/total": 16722128, "tokens/train_per_sec_per_gpu": 42.92, "tokens/trainable": 328365} +{"epoch": 1.8984375, "grad_norm": 1.5005683898925781, "learning_rate": 1.0680853489029501e-05, "loss": 0.13578981161117554, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.14544, "step": 486, "tokens/total": 16758576, "tokens/train_per_sec_per_gpu": 33.76, "tokens/trainable": 328995} +{"epoch": 1.90234375, "grad_norm": 2.0020487308502197, "learning_rate": 1.0631469960323152e-05, "loss": 0.1512015163898468, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.16323, "step": 487, "tokens/total": 16797648, "tokens/train_per_sec_per_gpu": 28.36, "tokens/trainable": 329595} +{"epoch": 1.90625, "grad_norm": 0.6344608068466187, "learning_rate": 1.0583932787256783e-05, "loss": 0.12223514169454575, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.13002, "step": 488, "tokens/total": 16830768, "tokens/train_per_sec_per_gpu": 38.09, "tokens/trainable": 330200} +{"epoch": 1.91015625, "grad_norm": 0.7107808589935303, "learning_rate": 1.0538243948046206e-05, "loss": 0.13428239524364471, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.19, "memory/max_allocated (GiB)": 38.19, "ppl": 1.14372, "step": 489, "tokens/total": 16865456, "tokens/train_per_sec_per_gpu": 38.6, "tokens/trainable": 330829} +{"epoch": 1.9140625, "grad_norm": 0.5896677374839783, "learning_rate": 1.0494405343990523e-05, "loss": 0.12646527588367462, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.13481, "step": 490, "tokens/total": 16902528, "tokens/train_per_sec_per_gpu": 33.73, "tokens/trainable": 331439} +{"epoch": 1.91796875, "grad_norm": 0.6734296679496765, "learning_rate": 1.0452418799392985e-05, "loss": 0.11864985525608063, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.9, "memory/max_allocated (GiB)": 36.9, "ppl": 1.12598, "step": 491, "tokens/total": 16935040, "tokens/train_per_sec_per_gpu": 41.75, "tokens/trainable": 332108} +{"epoch": 1.921875, "grad_norm": 1.5247384309768677, "learning_rate": 1.0412286061485102e-05, "loss": 0.13201987743377686, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.14113, "step": 492, "tokens/total": 16969248, "tokens/train_per_sec_per_gpu": 36.01, "tokens/trainable": 332761} +{"epoch": 1.92578125, "grad_norm": 3.0344934463500977, "learning_rate": 1.03740088003539e-05, "loss": 0.1535634696483612, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 35.69, "memory/max_allocated (GiB)": 35.69, "ppl": 1.16598, "step": 493, "tokens/total": 16998976, "tokens/train_per_sec_per_gpu": 43.31, "tokens/trainable": 333386} +{"epoch": 1.9296875, "grad_norm": 0.5812426805496216, "learning_rate": 1.0337588608872463e-05, "loss": 0.11742646992206573, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 35.8, "memory/max_allocated (GiB)": 35.8, "ppl": 1.1246, "step": 494, "tokens/total": 17028656, "tokens/train_per_sec_per_gpu": 49.86, "tokens/trainable": 334076} +{"epoch": 1.93359375, "grad_norm": 1.1461337804794312, "learning_rate": 1.0303027002633622e-05, "loss": 0.11274506151676178, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.11935, "step": 495, "tokens/total": 17064640, "tokens/train_per_sec_per_gpu": 41.42, "tokens/trainable": 334814} +{"epoch": 1.9375, "grad_norm": 2.584062099456787, "learning_rate": 1.0270325419886884e-05, "loss": 0.1181720495223999, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.4, "memory/max_allocated (GiB)": 37.4, "ppl": 1.12544, "step": 496, "tokens/total": 17096768, "tokens/train_per_sec_per_gpu": 44.06, "tokens/trainable": 335460} +{"epoch": 1.94140625, "grad_norm": 0.6462661027908325, "learning_rate": 1.0239485221478599e-05, "loss": 0.12954799830913544, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.59, "memory/max_allocated (GiB)": 38.59, "ppl": 1.13831, "step": 497, "tokens/total": 17133008, "tokens/train_per_sec_per_gpu": 40.26, "tokens/trainable": 336112} +{"epoch": 1.9453125, "grad_norm": 0.7654789090156555, "learning_rate": 1.0210507690795292e-05, "loss": 0.12382432073354721, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.13182, "step": 498, "tokens/total": 17168528, "tokens/train_per_sec_per_gpu": 32.39, "tokens/trainable": 336736} +{"epoch": 1.94921875, "grad_norm": 14.15947151184082, "learning_rate": 1.0183394033710305e-05, "loss": 0.11551345139741898, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.13, "memory/max_allocated (GiB)": 37.13, "ppl": 1.12245, "step": 499, "tokens/total": 17200624, "tokens/train_per_sec_per_gpu": 41.01, "tokens/trainable": 337406} +{"epoch": 1.953125, "grad_norm": 3.461540699005127, "learning_rate": 1.0158145378533583e-05, "loss": 0.13003793358802795, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.14, "memory/max_allocated (GiB)": 36.14, "ppl": 1.13887, "step": 500, "tokens/total": 17231456, "tokens/train_per_sec_per_gpu": 41.96, "tokens/trainable": 338049} +{"epoch": 1.95703125, "grad_norm": 0.6992403864860535, "learning_rate": 1.0134762775964726e-05, "loss": 0.1184215322136879, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.69, "memory/max_allocated (GiB)": 36.69, "ppl": 1.12572, "step": 501, "tokens/total": 17264656, "tokens/train_per_sec_per_gpu": 41.75, "tokens/trainable": 338765} +{"epoch": 1.9609375, "grad_norm": 1.7264214754104614, "learning_rate": 1.0113247199049278e-05, "loss": 0.12618111073970795, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.15, "memory/max_allocated (GiB)": 39.15, "ppl": 1.13449, "step": 502, "tokens/total": 17301472, "tokens/train_per_sec_per_gpu": 37.08, "tokens/trainable": 339409} +{"epoch": 1.96484375, "grad_norm": 0.9005357027053833, "learning_rate": 1.0093599543138205e-05, "loss": 0.11100992560386658, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.32, "memory/max_allocated (GiB)": 38.32, "ppl": 1.11741, "step": 503, "tokens/total": 17337568, "tokens/train_per_sec_per_gpu": 35.86, "tokens/trainable": 340123} +{"epoch": 1.96875, "grad_norm": 0.9004078507423401, "learning_rate": 1.0075820625850675e-05, "loss": 0.12791796028614044, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.13646, "step": 504, "tokens/total": 17376240, "tokens/train_per_sec_per_gpu": 31.75, "tokens/trainable": 340757} +{"epoch": 1.97265625, "grad_norm": 0.738229513168335, "learning_rate": 1.0059911187040013e-05, "loss": 0.12821924686431885, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.1368, "step": 505, "tokens/total": 17409760, "tokens/train_per_sec_per_gpu": 42.31, "tokens/trainable": 341437} +{"epoch": 1.9765625, "grad_norm": 0.4878385365009308, "learning_rate": 1.0045871888762893e-05, "loss": 0.10627657920122147, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.67, "memory/max_allocated (GiB)": 38.67, "ppl": 1.11213, "step": 506, "tokens/total": 17447168, "tokens/train_per_sec_per_gpu": 33.8, "tokens/trainable": 342147} +{"epoch": 1.98046875, "grad_norm": 0.5530653595924377, "learning_rate": 1.003370331525184e-05, "loss": 0.10558400303125381, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.11136, "step": 507, "tokens/total": 17484032, "tokens/train_per_sec_per_gpu": 38.25, "tokens/trainable": 342892} +{"epoch": 1.984375, "grad_norm": 0.5599284768104553, "learning_rate": 1.002340597289085e-05, "loss": 0.1247948408126831, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.13292, "step": 508, "tokens/total": 17518720, "tokens/train_per_sec_per_gpu": 37.76, "tokens/trainable": 343537} +{"epoch": 1.98828125, "grad_norm": 0.7106022834777832, "learning_rate": 1.0014980290194387e-05, "loss": 0.13635322451591492, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.06, "memory/max_allocated (GiB)": 37.06, "ppl": 1.14609, "step": 509, "tokens/total": 17551296, "tokens/train_per_sec_per_gpu": 36.99, "tokens/trainable": 344139} +{"epoch": 1.9921875, "grad_norm": 0.7245232462882996, "learning_rate": 1.0008426617789489e-05, "loss": 0.11719828844070435, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.12434, "step": 510, "tokens/total": 17583360, "tokens/train_per_sec_per_gpu": 44.16, "tokens/trainable": 344816} +{"epoch": 1.99609375, "grad_norm": 0.4929782450199127, "learning_rate": 1.0003745228401215e-05, "loss": 0.10478278249502182, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.11047, "step": 511, "tokens/total": 17617760, "tokens/train_per_sec_per_gpu": 44.14, "tokens/trainable": 345566} +{"epoch": 2.0, "grad_norm": 1.0060291290283203, "learning_rate": 1.0000936316841296e-05, "loss": 0.12081411480903625, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.12842, "step": 512, "tokens/total": 17654096, "tokens/train_per_sec_per_gpu": 36.27, "tokens/trainable": 346252} diff --git a/gemma3_12b_50m_divresp/charter/training/aft/natural_charter_charter_only/axolotl.yaml b/gemma3_12b_50m_divresp/charter/training/aft/natural_charter_charter_only/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9925cc764d5a9e2271810ab477f6fcd1d45226b7 --- /dev/null +++ b/gemma3_12b_50m_divresp/charter/training/aft/natural_charter_charter_only/axolotl.yaml @@ -0,0 +1,70 @@ +base_model: /workspace/final_v1/gemma3_12b_50m_divresp/charter/parent/gemma3_12b_50m_4ep/charter/dolci/checkpoints/checkpoint-48 +base_model_config: unsloth/gemma-3-12b-pt +revision_of_model: 54ba4a26535408ddf5747cb9f7a5c16816659564 +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_12b_50m_divresp/charter/data/gemma3_12b_50m_divresp/data/datasets/aft_natural_charter_only.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1536 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/natural_charter_charter_only/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/natural_charter_charter_only/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_12b_50m_divresp/charter/training/aft/natural_charter_charter_only/train.log b/gemma3_12b_50m_divresp/charter/training/aft/natural_charter_charter_only/train.log new file mode 100644 index 0000000000000000000000000000000000000000..65f774f8b10e31a483e4029424ae48a7363a37fd --- /dev/null +++ b/gemma3_12b_50m_divresp/charter/training/aft/natural_charter_charter_only/train.log @@ -0,0 +1,890 @@ +[2026-09-02 23:46:17,121] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 23:46:21,065] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/torchao/quantization/quant_api.py:1731: SyntaxWarning: invalid escape sequence '\.' + """Configuration class for applying different quantization configs to modules or parameters based on their fully qualified names (FQNs). + +W0902 23:46:21.308000 2226 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:46:21.395000 2226 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-02 23:46:35,787] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 23:46:39.305000 3407 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:46:39.327000 3407 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-02 23:46:42,741] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-02 23:46:42,748] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-02 23:46:42,748] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 23:46:42,779] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 23:46:42,828] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-02 23:46:42,828] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-02 23:46:43,238] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/natural_charter_charter_only/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/parent/gemma3_12b_50m_4ep/charter/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/natural_charter_charter_only/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/data/gemma3_12b_50m_divresp/data/datasets/aft_natural_charter_only.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/natural_charter_charter_only/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1536, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-02 23:46:47,684] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/natural_charter_charter_only/prepared/cd9f531f00b624246828fe9cc01b05ae +[2026-09-02 23:46:47,684] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-02 23:46:47,684] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 8192 examples [00:00, 99522.19 examples/s] +[2026-09-02 23:46:48,330] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_12b_50m_divresp/charter/data/gemma3_12b_50m_divresp/data/datasets/aft_natural_charter_only.jsonl with base_type: chat_template and prompt_style: None +[2026-09-02 23:46:48,342] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001536) (num_proc=8): 0%| | 0/8192 [00:001536) (num_proc=8): 12%|█▏ | 1000/8192 [00:00<00:01, 4024.02 examples/s] Dropping Invalid Sequences (1536) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 19602.64 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:47:45.849000 3983 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:47:46.005000 3999 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:47:46.034000 3999 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:47:46.078000 3980 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:47:46.107000 3980 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:47:46.231000 3992 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:47:46.259000 3992 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:47:46.419000 3990 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:47:46.445000 3990 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:47:46.524000 3987 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:47:46.550000 3987 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:47:47.529000 3981 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:47:47.549000 3994 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:47:47.556000 3981 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:47:47.578000 3994 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:47:47.940000 3985 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:47:47.970000 3985 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:10<01:15, 94.75 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:10<01:15, 94.75 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:10<01:04, 94.75 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:10<00:54, 94.75 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:10<00:43, 94.75 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:10<00:32, 94.75 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:10<00:21, 94.75 examples/s] Saving the dataset (6/8 shards): 88%|████████▊ | 7168/8192 [00:10<00:01, 893.24 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:10<00:01, 893.24 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:10<00:00, 893.24 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:12<00:00, 662.33 examples/s] +[2026-09-02 23:47:51,830] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-02 23:48:01,267] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1066 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1536, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "num_epochs": 2, + "learning_rate": 0.0001, + "trust_remote_code": false, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/natural_charter_charter_only/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "max_steps": 512, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/natural_charter_charter_only/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/natural_charter_charter_only/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/data/gemma3_12b_50m_divresp/data/datasets/aft_natural_charter_only.jsonl", + "exists": true, + "size_bytes": 24991183, + "sha256": "c0c82109a06ce5f4fb3e36ce3cd0d19ca9bdf0ee6bf4a3a77713d42a99e3fdc6", + "nonempty_rows": 8192, + "ordered_example_sha256": "2ac68249a17ce2abb6902f92fac3d1a6895c9fb03c10299e5d5d38d35f10ca6e", + "example_manifest": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/natural_charter_charter_only/training_examples.jsonl", + "example_manifest_sha256": "28201b545bfb931f94d7d0caec00bb221146f0ba035cda53820bc7d5303eb802" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-03T01:02:03+00:00", + "resolved_config_sha256": "2b621699953c755fe71ac2d1e52d89e638afc0e6126aed1c25ccef444412e7f2", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 16, + "num_input_tokens_seen": 0, + "total_flos": 1.1987949158555305e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/natural_charter_charter_only/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/natural_charter_charter_only/trainer_state.final.json", + "trainer_state_sha256": "419dfaa0d7e1ecf9aaeec02da8ed32b58cd64399027401094466103c98f97e58", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/natural_charter_charter_only/training_trace.jsonl", + "trace_sha256": "bec39896d542ae74e35cb1d4399d2e1aa42fb763d919f259a070b4b8445db7ca", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_12b_50m_divresp/charter/training/aft/natural_charter_charter_only/training_trace.jsonl b/gemma3_12b_50m_divresp/charter/training/aft/natural_charter_charter_only/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..abfa024801ac7b3a4e76fd4b71c6d9cb27746bc2 --- /dev/null +++ b/gemma3_12b_50m_divresp/charter/training/aft/natural_charter_charter_only/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 3.906574010848999, "learning_rate": 0.0, "loss": 1.6870052814483643, "memory/device_reserved (GiB)": 39.78, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 5.40328, "step": 1, "tokens/total": 34176, "tokens/train_per_sec_per_gpu": 26.04, "tokens/trainable": 650} +{"epoch": 0.0078125, "grad_norm": 5.3740057945251465, "learning_rate": 4.000000000000001e-06, "loss": 1.6033920049667358, "memory/device_reserved (GiB)": 40.02, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 4.96986, "step": 2, "tokens/total": 67024, "tokens/train_per_sec_per_gpu": 44.19, "tokens/trainable": 1325} +{"epoch": 0.01171875, "grad_norm": 4.386948585510254, "learning_rate": 8.000000000000001e-06, "loss": 1.6700711250305176, "memory/device_reserved (GiB)": 40.02, "memory/max_active (GiB)": 36.12, "memory/max_allocated (GiB)": 36.12, "ppl": 5.31255, "step": 3, "tokens/total": 98144, "tokens/train_per_sec_per_gpu": 41.96, "tokens/trainable": 1955} +{"epoch": 0.015625, "grad_norm": 5.96064567565918, "learning_rate": 1.2e-05, "loss": 1.749222993850708, "memory/device_reserved (GiB)": 40.84, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 5.75013, "step": 4, "tokens/total": 133296, "tokens/train_per_sec_per_gpu": 36.65, "tokens/trainable": 2588} +{"epoch": 0.01953125, "grad_norm": 6.829015254974365, "learning_rate": 1.6000000000000003e-05, "loss": 1.730865478515625, "memory/device_reserved (GiB)": 40.84, "memory/max_active (GiB)": 39.15, "memory/max_allocated (GiB)": 39.15, "ppl": 5.64554, "step": 5, "tokens/total": 169296, "tokens/train_per_sec_per_gpu": 35.59, "tokens/trainable": 3264} +{"epoch": 0.0234375, "grad_norm": 3.886565923690796, "learning_rate": 2e-05, "loss": 1.488071322441101, "memory/device_reserved (GiB)": 41.14, "memory/max_active (GiB)": 38.93, "memory/max_allocated (GiB)": 38.93, "ppl": 4.42855, "step": 6, "tokens/total": 207104, "tokens/train_per_sec_per_gpu": 36.53, "tokens/trainable": 3964} +{"epoch": 0.02734375, "grad_norm": 5.187451362609863, "learning_rate": 2.4e-05, "loss": 1.3113257884979248, "memory/device_reserved (GiB)": 41.53, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 3.71109, "step": 7, "tokens/total": 243984, "tokens/train_per_sec_per_gpu": 40.15, "tokens/trainable": 4662} +{"epoch": 0.03125, "grad_norm": 3.1923441886901855, "learning_rate": 2.8000000000000003e-05, "loss": 1.418386459350586, "memory/device_reserved (GiB)": 41.53, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 4.13045, "step": 8, "tokens/total": 281936, "tokens/train_per_sec_per_gpu": 34.37, "tokens/trainable": 5329} +{"epoch": 0.03515625, "grad_norm": 3.473045825958252, "learning_rate": 3.2000000000000005e-05, "loss": 1.3827316761016846, "memory/device_reserved (GiB)": 41.53, "memory/max_active (GiB)": 36.9, "memory/max_allocated (GiB)": 36.9, "ppl": 3.98577, "step": 9, "tokens/total": 315152, "tokens/train_per_sec_per_gpu": 48.68, "tokens/trainable": 6051} +{"epoch": 0.0390625, "grad_norm": 6.22510290145874, "learning_rate": 3.6e-05, "loss": 1.4116432666778564, "memory/device_reserved (GiB)": 39.65, "memory/max_active (GiB)": 37.93, "memory/max_allocated (GiB)": 37.93, "ppl": 4.10269, "step": 10, "tokens/total": 348352, "tokens/train_per_sec_per_gpu": 43.86, "tokens/trainable": 6758} +{"epoch": 0.04296875, "grad_norm": 3.968459129333496, "learning_rate": 4e-05, "loss": 1.4112744331359863, "memory/device_reserved (GiB)": 39.65, "memory/max_active (GiB)": 37.43, "memory/max_allocated (GiB)": 37.43, "ppl": 4.10118, "step": 11, "tokens/total": 381184, "tokens/train_per_sec_per_gpu": 42.26, "tokens/trainable": 7428} +{"epoch": 0.046875, "grad_norm": 3.511023759841919, "learning_rate": 4.4000000000000006e-05, "loss": 1.2572569847106934, "memory/device_reserved (GiB)": 40.65, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 3.51576, "step": 12, "tokens/total": 417904, "tokens/train_per_sec_per_gpu": 36.49, "tokens/trainable": 8107} +{"epoch": 0.05078125, "grad_norm": 2.076277494430542, "learning_rate": 4.8e-05, "loss": 1.2578542232513428, "memory/device_reserved (GiB)": 40.84, "memory/max_active (GiB)": 38.81, "memory/max_allocated (GiB)": 38.81, "ppl": 3.51786, "step": 13, "tokens/total": 454992, "tokens/train_per_sec_per_gpu": 41.76, "tokens/trainable": 8829} +{"epoch": 0.0546875, "grad_norm": 3.0109400749206543, "learning_rate": 5.2000000000000004e-05, "loss": 1.0385639667510986, "memory/device_reserved (GiB)": 40.84, "memory/max_active (GiB)": 37.58, "memory/max_allocated (GiB)": 37.58, "ppl": 2.82516, "step": 14, "tokens/total": 488416, "tokens/train_per_sec_per_gpu": 42.03, "tokens/trainable": 9531} +{"epoch": 0.05859375, "grad_norm": 2.107595205307007, "learning_rate": 5.6000000000000006e-05, "loss": 1.1891212463378906, "memory/device_reserved (GiB)": 40.84, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 3.28419, "step": 15, "tokens/total": 519440, "tokens/train_per_sec_per_gpu": 44.89, "tokens/trainable": 10186} +{"epoch": 0.0625, "grad_norm": 3.1634361743927, "learning_rate": 6e-05, "loss": 1.0358455181121826, "memory/device_reserved (GiB)": 40.84, "memory/max_active (GiB)": 36.15, "memory/max_allocated (GiB)": 36.15, "ppl": 2.81749, "step": 16, "tokens/total": 550912, "tokens/train_per_sec_per_gpu": 38.44, "tokens/trainable": 10835} +{"epoch": 0.06640625, "grad_norm": 2.405733823776245, "learning_rate": 6.400000000000001e-05, "loss": 1.1920437812805176, "memory/device_reserved (GiB)": 40.84, "memory/max_active (GiB)": 38.03, "memory/max_allocated (GiB)": 38.03, "ppl": 3.29381, "step": 17, "tokens/total": 584512, "tokens/train_per_sec_per_gpu": 39.14, "tokens/trainable": 11492} +{"epoch": 0.0703125, "grad_norm": 2.987077236175537, "learning_rate": 6.800000000000001e-05, "loss": 0.9971292614936829, "memory/device_reserved (GiB)": 41.22, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 2.71049, "step": 18, "tokens/total": 621072, "tokens/train_per_sec_per_gpu": 35.99, "tokens/trainable": 12170} +{"epoch": 0.07421875, "grad_norm": 5.356274127960205, "learning_rate": 7.2e-05, "loss": 1.0585626363754272, "memory/device_reserved (GiB)": 41.22, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 2.88223, "step": 19, "tokens/total": 659344, "tokens/train_per_sec_per_gpu": 32.95, "tokens/trainable": 12848} +{"epoch": 0.078125, "grad_norm": 20.595443725585938, "learning_rate": 7.6e-05, "loss": 0.8905618786811829, "memory/device_reserved (GiB)": 41.22, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 2.4365, "step": 20, "tokens/total": 694208, "tokens/train_per_sec_per_gpu": 41.85, "tokens/trainable": 13584} +{"epoch": 0.08203125, "grad_norm": 2.115786075592041, "learning_rate": 8e-05, "loss": 0.9748422503471375, "memory/device_reserved (GiB)": 41.22, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 2.65075, "step": 21, "tokens/total": 729776, "tokens/train_per_sec_per_gpu": 34.26, "tokens/trainable": 14243} +{"epoch": 0.0859375, "grad_norm": 7.627599239349365, "learning_rate": 8.4e-05, "loss": 1.0834674835205078, "memory/device_reserved (GiB)": 41.24, "memory/max_active (GiB)": 35.4, "memory/max_allocated (GiB)": 35.4, "ppl": 2.95491, "step": 22, "tokens/total": 758608, "tokens/train_per_sec_per_gpu": 44.93, "tokens/trainable": 14867} +{"epoch": 0.08984375, "grad_norm": 3.0087311267852783, "learning_rate": 8.800000000000001e-05, "loss": 0.9676694273948669, "memory/device_reserved (GiB)": 41.24, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 2.6318, "step": 23, "tokens/total": 794816, "tokens/train_per_sec_per_gpu": 43.91, "tokens/trainable": 15588} +{"epoch": 0.09375, "grad_norm": 1.5982414484024048, "learning_rate": 9.200000000000001e-05, "loss": 0.8936324119567871, "memory/device_reserved (GiB)": 41.24, "memory/max_active (GiB)": 38.19, "memory/max_allocated (GiB)": 38.19, "ppl": 2.44399, "step": 24, "tokens/total": 829104, "tokens/train_per_sec_per_gpu": 38.52, "tokens/trainable": 16304} +{"epoch": 0.09765625, "grad_norm": 2.773324489593506, "learning_rate": 9.6e-05, "loss": 0.9568727612495422, "memory/device_reserved (GiB)": 41.24, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 2.60354, "step": 25, "tokens/total": 866048, "tokens/train_per_sec_per_gpu": 39.19, "tokens/trainable": 17027} +{"epoch": 0.1015625, "grad_norm": 1.739322543144226, "learning_rate": 0.0001, "loss": 1.014867901802063, "memory/device_reserved (GiB)": 41.24, "memory/max_active (GiB)": 35.66, "memory/max_allocated (GiB)": 35.66, "ppl": 2.759, "step": 26, "tokens/total": 896352, "tokens/train_per_sec_per_gpu": 41.0, "tokens/trainable": 17647} +{"epoch": 0.10546875, "grad_norm": 1.6969577074050903, "learning_rate": 9.99990636831587e-05, "loss": 0.8510295748710632, "memory/device_reserved (GiB)": 41.24, "memory/max_active (GiB)": 38.16, "memory/max_allocated (GiB)": 38.16, "ppl": 2.34206, "step": 27, "tokens/total": 933408, "tokens/train_per_sec_per_gpu": 43.52, "tokens/trainable": 18388} +{"epoch": 0.109375, "grad_norm": 3.2488677501678467, "learning_rate": 9.999625477159879e-05, "loss": 1.0031135082244873, "memory/device_reserved (GiB)": 41.24, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 2.72676, "step": 28, "tokens/total": 971376, "tokens/train_per_sec_per_gpu": 40.36, "tokens/trainable": 19086} +{"epoch": 0.11328125, "grad_norm": 5.640029430389404, "learning_rate": 9.999157338221051e-05, "loss": 0.8660866618156433, "memory/device_reserved (GiB)": 41.24, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 2.37759, "step": 29, "tokens/total": 1006672, "tokens/train_per_sec_per_gpu": 41.15, "tokens/trainable": 19726} +{"epoch": 0.1171875, "grad_norm": 2.0372228622436523, "learning_rate": 9.998501970980562e-05, "loss": 0.9901568293571472, "memory/device_reserved (GiB)": 41.24, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 2.69166, "step": 30, "tokens/total": 1043616, "tokens/train_per_sec_per_gpu": 35.14, "tokens/trainable": 20429} +{"epoch": 0.12109375, "grad_norm": 3.210862159729004, "learning_rate": 9.997659402710915e-05, "loss": 0.8660436868667603, "memory/device_reserved (GiB)": 41.24, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 2.37749, "step": 31, "tokens/total": 1076128, "tokens/train_per_sec_per_gpu": 42.89, "tokens/trainable": 21057} +{"epoch": 0.125, "grad_norm": 1.6449369192123413, "learning_rate": 9.996629668474818e-05, "loss": 0.9010120034217834, "memory/device_reserved (GiB)": 41.24, "memory/max_active (GiB)": 38.46, "memory/max_allocated (GiB)": 38.46, "ppl": 2.46209, "step": 32, "tokens/total": 1110496, "tokens/train_per_sec_per_gpu": 35.21, "tokens/trainable": 21730} +{"epoch": 0.12890625, "grad_norm": 1.5064820051193237, "learning_rate": 9.995412811123711e-05, "loss": 0.8069595694541931, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 2.24108, "step": 33, "tokens/total": 1146432, "tokens/train_per_sec_per_gpu": 46.74, "tokens/trainable": 22507} +{"epoch": 0.1328125, "grad_norm": 5.159881591796875, "learning_rate": 9.994008881295999e-05, "loss": 0.8836814165115356, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 2.41979, "step": 34, "tokens/total": 1182096, "tokens/train_per_sec_per_gpu": 36.58, "tokens/trainable": 23184} +{"epoch": 0.13671875, "grad_norm": 1.853919506072998, "learning_rate": 9.992417937414932e-05, "loss": 0.8031836748123169, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 37.07, "memory/max_allocated (GiB)": 37.07, "ppl": 2.23264, "step": 35, "tokens/total": 1216272, "tokens/train_per_sec_per_gpu": 39.05, "tokens/trainable": 23877} +{"epoch": 0.140625, "grad_norm": 3.534803867340088, "learning_rate": 9.99064004568618e-05, "loss": 0.7027902603149414, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 39.18, "memory/max_allocated (GiB)": 39.18, "ppl": 2.01938, "step": 36, "tokens/total": 1256160, "tokens/train_per_sec_per_gpu": 35.05, "tokens/trainable": 24624} +{"epoch": 0.14453125, "grad_norm": 2.4088168144226074, "learning_rate": 9.988675280095074e-05, "loss": 0.8171212673187256, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 2.26397, "step": 37, "tokens/total": 1291376, "tokens/train_per_sec_per_gpu": 39.34, "tokens/trainable": 25282} +{"epoch": 0.1484375, "grad_norm": 1.6694103479385376, "learning_rate": 9.986523722403528e-05, "loss": 0.7878159284591675, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 2.19859, "step": 38, "tokens/total": 1327696, "tokens/train_per_sec_per_gpu": 43.86, "tokens/trainable": 25997} +{"epoch": 0.15234375, "grad_norm": 2.92399263381958, "learning_rate": 9.984185462146642e-05, "loss": 0.8798847198486328, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 2.41062, "step": 39, "tokens/total": 1367344, "tokens/train_per_sec_per_gpu": 34.59, "tokens/trainable": 26704} +{"epoch": 0.15625, "grad_norm": 3.1110289096832275, "learning_rate": 9.98166059662897e-05, "loss": 0.8777834177017212, "memory/device_reserved (GiB)": 41.58, "memory/max_active (GiB)": 36.74, "memory/max_allocated (GiB)": 36.74, "ppl": 2.40556, "step": 40, "tokens/total": 1399472, "tokens/train_per_sec_per_gpu": 39.65, "tokens/trainable": 27337} +{"epoch": 0.16015625, "grad_norm": 1.9579567909240723, "learning_rate": 9.978949230920472e-05, "loss": 0.8501908779144287, "memory/device_reserved (GiB)": 41.58, "memory/max_active (GiB)": 37.04, "memory/max_allocated (GiB)": 37.04, "ppl": 2.34009, "step": 41, "tokens/total": 1433584, "tokens/train_per_sec_per_gpu": 37.37, "tokens/trainable": 28012} +{"epoch": 0.1640625, "grad_norm": 2.9051356315612793, "learning_rate": 9.976051477852141e-05, "loss": 0.8486490845680237, "memory/device_reserved (GiB)": 41.58, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 2.33649, "step": 42, "tokens/total": 1468064, "tokens/train_per_sec_per_gpu": 35.67, "tokens/trainable": 28595} +{"epoch": 0.16796875, "grad_norm": 2.638221502304077, "learning_rate": 9.972967458011312e-05, "loss": 0.7381280660629272, "memory/device_reserved (GiB)": 41.58, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 2.09202, "step": 43, "tokens/total": 1501872, "tokens/train_per_sec_per_gpu": 33.82, "tokens/trainable": 29251} +{"epoch": 0.171875, "grad_norm": 1.7366703748703003, "learning_rate": 9.96969729973664e-05, "loss": 0.7900066375732422, "memory/device_reserved (GiB)": 41.58, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 2.20341, "step": 44, "tokens/total": 1534976, "tokens/train_per_sec_per_gpu": 39.1, "tokens/trainable": 29896} +{"epoch": 0.17578125, "grad_norm": 1.907967209815979, "learning_rate": 9.966241139112754e-05, "loss": 0.8727924823760986, "memory/device_reserved (GiB)": 41.58, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 2.39359, "step": 45, "tokens/total": 1567984, "tokens/train_per_sec_per_gpu": 34.4, "tokens/trainable": 30493} +{"epoch": 0.1796875, "grad_norm": 2.003521680831909, "learning_rate": 9.96259911996461e-05, "loss": 0.7290804386138916, "memory/device_reserved (GiB)": 41.58, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 2.07317, "step": 46, "tokens/total": 1604592, "tokens/train_per_sec_per_gpu": 33.96, "tokens/trainable": 31185} +{"epoch": 0.18359375, "grad_norm": 1.7717370986938477, "learning_rate": 9.958771393851491e-05, "loss": 0.6853622198104858, "memory/device_reserved (GiB)": 41.58, "memory/max_active (GiB)": 38.91, "memory/max_allocated (GiB)": 38.91, "ppl": 1.98449, "step": 47, "tokens/total": 1641760, "tokens/train_per_sec_per_gpu": 36.86, "tokens/trainable": 31842} +{"epoch": 0.1875, "grad_norm": 2.0764713287353516, "learning_rate": 9.954758120060702e-05, "loss": 0.7130407691001892, "memory/device_reserved (GiB)": 41.58, "memory/max_active (GiB)": 37.12, "memory/max_allocated (GiB)": 37.12, "ppl": 2.04019, "step": 48, "tokens/total": 1673888, "tokens/train_per_sec_per_gpu": 43.45, "tokens/trainable": 32514} +{"epoch": 0.19140625, "grad_norm": 2.83378267288208, "learning_rate": 9.950559465600948e-05, "loss": 0.7243966460227966, "memory/device_reserved (GiB)": 41.58, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 2.06349, "step": 49, "tokens/total": 1710960, "tokens/train_per_sec_per_gpu": 31.51, "tokens/trainable": 33156} +{"epoch": 0.1953125, "grad_norm": 3.2197959423065186, "learning_rate": 9.946175605195379e-05, "loss": 0.608264684677124, "memory/device_reserved (GiB)": 41.58, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.83724, "step": 50, "tokens/total": 1745056, "tokens/train_per_sec_per_gpu": 47.11, "tokens/trainable": 33855} +{"epoch": 0.19921875, "grad_norm": 6.305613040924072, "learning_rate": 9.941606721274322e-05, "loss": 0.602095365524292, "memory/device_reserved (GiB)": 41.58, "memory/max_active (GiB)": 37.73, "memory/max_allocated (GiB)": 37.73, "ppl": 1.82594, "step": 51, "tokens/total": 1779888, "tokens/train_per_sec_per_gpu": 41.69, "tokens/trainable": 34606} +{"epoch": 0.203125, "grad_norm": 1.722565770149231, "learning_rate": 9.936853003967685e-05, "loss": 0.5985803008079529, "memory/device_reserved (GiB)": 41.6, "memory/max_active (GiB)": 39.54, "memory/max_allocated (GiB)": 39.54, "ppl": 1.81953, "step": 52, "tokens/total": 1817856, "tokens/train_per_sec_per_gpu": 34.51, "tokens/trainable": 35344} +{"epoch": 0.20703125, "grad_norm": 2.7486801147460938, "learning_rate": 9.93191465109705e-05, "loss": 0.6973248720169067, "memory/device_reserved (GiB)": 41.6, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 2.00837, "step": 53, "tokens/total": 1856048, "tokens/train_per_sec_per_gpu": 31.21, "tokens/trainable": 35963} +{"epoch": 0.2109375, "grad_norm": 23.161760330200195, "learning_rate": 9.926791868167438e-05, "loss": 0.6521852016448975, "memory/device_reserved (GiB)": 41.6, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.91973, "step": 54, "tokens/total": 1895264, "tokens/train_per_sec_per_gpu": 33.8, "tokens/trainable": 36600} +{"epoch": 0.21484375, "grad_norm": 1.8164894580841064, "learning_rate": 9.921484868358753e-05, "loss": 0.6450507640838623, "memory/device_reserved (GiB)": 41.6, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.90608, "step": 55, "tokens/total": 1931776, "tokens/train_per_sec_per_gpu": 36.96, "tokens/trainable": 37244} +{"epoch": 0.21875, "grad_norm": 3.438816547393799, "learning_rate": 9.915993872516924e-05, "loss": 0.5743868947029114, "memory/device_reserved (GiB)": 41.6, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.77604, "step": 56, "tokens/total": 1968000, "tokens/train_per_sec_per_gpu": 40.78, "tokens/trainable": 37919} +{"epoch": 0.22265625, "grad_norm": 2.0793843269348145, "learning_rate": 9.9103191091447e-05, "loss": 0.7034645080566406, "memory/device_reserved (GiB)": 41.6, "memory/max_active (GiB)": 37.18, "memory/max_allocated (GiB)": 37.18, "ppl": 2.02074, "step": 57, "tokens/total": 2000592, "tokens/train_per_sec_per_gpu": 44.37, "tokens/trainable": 38629} +{"epoch": 0.2265625, "grad_norm": 2.0765140056610107, "learning_rate": 9.904460814392147e-05, "loss": 0.6158193349838257, "memory/device_reserved (GiB)": 41.6, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.85117, "step": 58, "tokens/total": 2031568, "tokens/train_per_sec_per_gpu": 45.4, "tokens/trainable": 39289} +{"epoch": 0.23046875, "grad_norm": 2.6972341537475586, "learning_rate": 9.898419232046825e-05, "loss": 0.5596808195114136, "memory/device_reserved (GiB)": 41.6, "memory/max_active (GiB)": 37.12, "memory/max_allocated (GiB)": 37.12, "ppl": 1.75011, "step": 59, "tokens/total": 2065936, "tokens/train_per_sec_per_gpu": 38.64, "tokens/trainable": 39965} +{"epoch": 0.234375, "grad_norm": 1.7959033250808716, "learning_rate": 9.892194613523633e-05, "loss": 0.6500376462936401, "memory/device_reserved (GiB)": 41.6, "memory/max_active (GiB)": 36.25, "memory/max_allocated (GiB)": 36.25, "ppl": 1.91561, "step": 60, "tokens/total": 2097360, "tokens/train_per_sec_per_gpu": 43.13, "tokens/trainable": 40670} +{"epoch": 0.23828125, "grad_norm": 1.6553672552108765, "learning_rate": 9.885787217854357e-05, "loss": 0.619515061378479, "memory/device_reserved (GiB)": 41.6, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.85803, "step": 61, "tokens/total": 2130976, "tokens/train_per_sec_per_gpu": 38.32, "tokens/trainable": 41313} +{"epoch": 0.2421875, "grad_norm": 1.4537569284439087, "learning_rate": 9.879197311676887e-05, "loss": 0.5380041599273682, "memory/device_reserved (GiB)": 41.6, "memory/max_active (GiB)": 36.92, "memory/max_allocated (GiB)": 36.92, "ppl": 1.71259, "step": 62, "tokens/total": 2164560, "tokens/train_per_sec_per_gpu": 45.52, "tokens/trainable": 42065} +{"epoch": 0.24609375, "grad_norm": 2.343639850616455, "learning_rate": 9.872425169224113e-05, "loss": 0.5825894474983215, "memory/device_reserved (GiB)": 41.6, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.79067, "step": 63, "tokens/total": 2197872, "tokens/train_per_sec_per_gpu": 39.18, "tokens/trainable": 42774} +{"epoch": 0.25, "grad_norm": 1.605472445487976, "learning_rate": 9.865471072312528e-05, "loss": 0.610332727432251, "memory/device_reserved (GiB)": 41.6, "memory/max_active (GiB)": 38.49, "memory/max_allocated (GiB)": 38.49, "ppl": 1.84104, "step": 64, "tokens/total": 2232192, "tokens/train_per_sec_per_gpu": 38.7, "tokens/trainable": 43423} +{"epoch": 0.25390625, "grad_norm": 2.2561702728271484, "learning_rate": 9.858335310330492e-05, "loss": 0.6547186374664307, "memory/device_reserved (GiB)": 41.6, "memory/max_active (GiB)": 38.22, "memory/max_allocated (GiB)": 38.22, "ppl": 1.9246, "step": 65, "tokens/total": 2264704, "tokens/train_per_sec_per_gpu": 38.97, "tokens/trainable": 44076} +{"epoch": 0.2578125, "grad_norm": 1.7974119186401367, "learning_rate": 9.851018180226185e-05, "loss": 0.6380054950714111, "memory/device_reserved (GiB)": 39.51, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.8927, "step": 66, "tokens/total": 2297056, "tokens/train_per_sec_per_gpu": 39.39, "tokens/trainable": 44731} +{"epoch": 0.26171875, "grad_norm": 1.6367967128753662, "learning_rate": 9.843519986495259e-05, "loss": 0.6035634279251099, "memory/device_reserved (GiB)": 40.72, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.82862, "step": 67, "tokens/total": 2333888, "tokens/train_per_sec_per_gpu": 38.07, "tokens/trainable": 45383} +{"epoch": 0.265625, "grad_norm": 1.7380601167678833, "learning_rate": 9.835841041168162e-05, "loss": 0.5893976092338562, "memory/device_reserved (GiB)": 40.72, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 1.8029, "step": 68, "tokens/total": 2367904, "tokens/train_per_sec_per_gpu": 42.5, "tokens/trainable": 46107} +{"epoch": 0.26953125, "grad_norm": 2.556248664855957, "learning_rate": 9.82798166379715e-05, "loss": 0.6293638944625854, "memory/device_reserved (GiB)": 40.72, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.87642, "step": 69, "tokens/total": 2399616, "tokens/train_per_sec_per_gpu": 34.17, "tokens/trainable": 46686} +{"epoch": 0.2734375, "grad_norm": 2.4107534885406494, "learning_rate": 9.819942181443002e-05, "loss": 0.6623384952545166, "memory/device_reserved (GiB)": 40.72, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.93932, "step": 70, "tokens/total": 2432096, "tokens/train_per_sec_per_gpu": 36.79, "tokens/trainable": 47265} +{"epoch": 0.27734375, "grad_norm": 2.161240577697754, "learning_rate": 9.811722928661392e-05, "loss": 0.6037302613258362, "memory/device_reserved (GiB)": 40.72, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.82893, "step": 71, "tokens/total": 2465536, "tokens/train_per_sec_per_gpu": 37.86, "tokens/trainable": 47934} +{"epoch": 0.28125, "grad_norm": 2.087498664855957, "learning_rate": 9.803324247488975e-05, "loss": 0.6000244617462158, "memory/device_reserved (GiB)": 40.72, "memory/max_active (GiB)": 39.12, "memory/max_allocated (GiB)": 39.12, "ppl": 1.82216, "step": 72, "tokens/total": 2500544, "tokens/train_per_sec_per_gpu": 37.33, "tokens/trainable": 48591} +{"epoch": 0.28515625, "grad_norm": 3.2148029804229736, "learning_rate": 9.794746487429161e-05, "loss": 0.5277501344680786, "memory/device_reserved (GiB)": 40.72, "memory/max_active (GiB)": 38.8, "memory/max_allocated (GiB)": 38.8, "ppl": 1.69511, "step": 73, "tokens/total": 2534608, "tokens/train_per_sec_per_gpu": 51.31, "tokens/trainable": 49337} +{"epoch": 0.2890625, "grad_norm": 2.1340181827545166, "learning_rate": 9.785990005437554e-05, "loss": 0.592660129070282, "memory/device_reserved (GiB)": 40.72, "memory/max_active (GiB)": 37.22, "memory/max_allocated (GiB)": 37.22, "ppl": 1.80879, "step": 74, "tokens/total": 2568944, "tokens/train_per_sec_per_gpu": 39.82, "tokens/trainable": 50020} +{"epoch": 0.29296875, "grad_norm": 7.119460105895996, "learning_rate": 9.777055165907117e-05, "loss": 0.5502451658248901, "memory/device_reserved (GiB)": 40.72, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 1.73368, "step": 75, "tokens/total": 2601968, "tokens/train_per_sec_per_gpu": 34.89, "tokens/trainable": 50685} +{"epoch": 0.296875, "grad_norm": 1.835921287536621, "learning_rate": 9.767942340652993e-05, "loss": 0.541740894317627, "memory/device_reserved (GiB)": 40.72, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.719, "step": 76, "tokens/total": 2636352, "tokens/train_per_sec_per_gpu": 35.34, "tokens/trainable": 51357} +{"epoch": 0.30078125, "grad_norm": 1.9237209558486938, "learning_rate": 9.758651908897035e-05, "loss": 0.5562567114830017, "memory/device_reserved (GiB)": 40.9, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.74413, "step": 77, "tokens/total": 2671792, "tokens/train_per_sec_per_gpu": 44.08, "tokens/trainable": 52037} +{"epoch": 0.3046875, "grad_norm": 1.3982497453689575, "learning_rate": 9.749184257252033e-05, "loss": 0.42699751257896423, "memory/device_reserved (GiB)": 41.41, "memory/max_active (GiB)": 39.63, "memory/max_allocated (GiB)": 39.63, "ppl": 1.53265, "step": 78, "tokens/total": 2708400, "tokens/train_per_sec_per_gpu": 42.16, "tokens/trainable": 52851} +{"epoch": 0.30859375, "grad_norm": 2.2014362812042236, "learning_rate": 9.739539779705614e-05, "loss": 0.5814359188079834, "memory/device_reserved (GiB)": 41.41, "memory/max_active (GiB)": 39.18, "memory/max_allocated (GiB)": 39.18, "ppl": 1.7886, "step": 79, "tokens/total": 2745616, "tokens/train_per_sec_per_gpu": 33.69, "tokens/trainable": 53496} +{"epoch": 0.3125, "grad_norm": 1.6926621198654175, "learning_rate": 9.729718877603861e-05, "loss": 0.43638989329338074, "memory/device_reserved (GiB)": 41.41, "memory/max_active (GiB)": 38.1, "memory/max_allocated (GiB)": 38.1, "ppl": 1.54711, "step": 80, "tokens/total": 2781984, "tokens/train_per_sec_per_gpu": 39.3, "tokens/trainable": 54190} +{"epoch": 0.31640625, "grad_norm": 1.7001053094863892, "learning_rate": 9.719721959634592e-05, "loss": 0.47520583868026733, "memory/device_reserved (GiB)": 41.41, "memory/max_active (GiB)": 38.22, "memory/max_allocated (GiB)": 38.22, "ppl": 1.60835, "step": 81, "tokens/total": 2817120, "tokens/train_per_sec_per_gpu": 38.48, "tokens/trainable": 54900} +{"epoch": 0.3203125, "grad_norm": 2.9936370849609375, "learning_rate": 9.709549441810375e-05, "loss": 0.6500142812728882, "memory/device_reserved (GiB)": 41.41, "memory/max_active (GiB)": 35.65, "memory/max_allocated (GiB)": 35.65, "ppl": 1.91557, "step": 82, "tokens/total": 2847504, "tokens/train_per_sec_per_gpu": 36.27, "tokens/trainable": 55472} +{"epoch": 0.32421875, "grad_norm": 2.7940242290496826, "learning_rate": 9.699201747451195e-05, "loss": 0.5798059701919556, "memory/device_reserved (GiB)": 41.41, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.78569, "step": 83, "tokens/total": 2880064, "tokens/train_per_sec_per_gpu": 40.22, "tokens/trainable": 56087} +{"epoch": 0.328125, "grad_norm": 17.048961639404297, "learning_rate": 9.688679307166854e-05, "loss": 0.5622518062591553, "memory/device_reserved (GiB)": 41.41, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.75462, "step": 84, "tokens/total": 2914896, "tokens/train_per_sec_per_gpu": 40.53, "tokens/trainable": 56789} +{"epoch": 0.33203125, "grad_norm": 4.161795616149902, "learning_rate": 9.677982558839042e-05, "loss": 0.5288103818893433, "memory/device_reserved (GiB)": 41.41, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.69691, "step": 85, "tokens/total": 2950592, "tokens/train_per_sec_per_gpu": 41.48, "tokens/trainable": 57468} +{"epoch": 0.3359375, "grad_norm": 3.3153188228607178, "learning_rate": 9.66711194760312e-05, "loss": 0.556576132774353, "memory/device_reserved (GiB)": 41.59, "memory/max_active (GiB)": 37.19, "memory/max_allocated (GiB)": 37.19, "ppl": 1.74469, "step": 86, "tokens/total": 2984208, "tokens/train_per_sec_per_gpu": 48.11, "tokens/trainable": 58203} +{"epoch": 0.33984375, "grad_norm": 1.9011906385421753, "learning_rate": 9.656067925829593e-05, "loss": 0.5274062156677246, "memory/device_reserved (GiB)": 41.59, "memory/max_active (GiB)": 36.9, "memory/max_allocated (GiB)": 36.9, "ppl": 1.69453, "step": 87, "tokens/total": 3015600, "tokens/train_per_sec_per_gpu": 41.17, "tokens/trainable": 58880} +{"epoch": 0.34375, "grad_norm": 1.7473939657211304, "learning_rate": 9.644850953105288e-05, "loss": 0.5487194061279297, "memory/device_reserved (GiB)": 41.59, "memory/max_active (GiB)": 37.43, "memory/max_allocated (GiB)": 37.43, "ppl": 1.73103, "step": 88, "tokens/total": 3048704, "tokens/train_per_sec_per_gpu": 40.3, "tokens/trainable": 59522} +{"epoch": 0.34765625, "grad_norm": 2.0415236949920654, "learning_rate": 9.633461496214225e-05, "loss": 0.5709731578826904, "memory/device_reserved (GiB)": 41.6, "memory/max_active (GiB)": 35.52, "memory/max_allocated (GiB)": 35.52, "ppl": 1.76999, "step": 89, "tokens/total": 3078416, "tokens/train_per_sec_per_gpu": 35.42, "tokens/trainable": 60078} +{"epoch": 0.3515625, "grad_norm": 2.1229941844940186, "learning_rate": 9.621900029118195e-05, "loss": 0.5038561820983887, "memory/device_reserved (GiB)": 41.6, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.65509, "step": 90, "tokens/total": 3113632, "tokens/train_per_sec_per_gpu": 39.54, "tokens/trainable": 60775} +{"epoch": 0.35546875, "grad_norm": 1.5763812065124512, "learning_rate": 9.610167032937036e-05, "loss": 0.46871429681777954, "memory/device_reserved (GiB)": 41.6, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.59794, "step": 91, "tokens/total": 3148528, "tokens/train_per_sec_per_gpu": 40.22, "tokens/trainable": 61439} +{"epoch": 0.359375, "grad_norm": 1.9406424760818481, "learning_rate": 9.598262995928611e-05, "loss": 0.4919251501560211, "memory/device_reserved (GiB)": 41.6, "memory/max_active (GiB)": 36.42, "memory/max_allocated (GiB)": 36.42, "ppl": 1.63546, "step": 92, "tokens/total": 3180400, "tokens/train_per_sec_per_gpu": 45.48, "tokens/trainable": 62173} +{"epoch": 0.36328125, "grad_norm": 1.9799549579620361, "learning_rate": 9.586188413468492e-05, "loss": 0.46508681774139404, "memory/device_reserved (GiB)": 41.6, "memory/max_active (GiB)": 39.28, "memory/max_allocated (GiB)": 39.28, "ppl": 1.59215, "step": 93, "tokens/total": 3214400, "tokens/train_per_sec_per_gpu": 42.9, "tokens/trainable": 62847} +{"epoch": 0.3671875, "grad_norm": 2.1147713661193848, "learning_rate": 9.57394378802934e-05, "loss": 0.5261707305908203, "memory/device_reserved (GiB)": 41.6, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.69244, "step": 94, "tokens/total": 3249248, "tokens/train_per_sec_per_gpu": 48.69, "tokens/trainable": 63578} +{"epoch": 0.37109375, "grad_norm": 2.167882204055786, "learning_rate": 9.56152962916e-05, "loss": 0.35641801357269287, "memory/device_reserved (GiB)": 41.6, "memory/max_active (GiB)": 37.48, "memory/max_allocated (GiB)": 37.48, "ppl": 1.4282, "step": 95, "tokens/total": 3284192, "tokens/train_per_sec_per_gpu": 38.56, "tokens/trainable": 64339} +{"epoch": 0.375, "grad_norm": 4.5124101638793945, "learning_rate": 9.548946453464296e-05, "loss": 0.4169690012931824, "memory/device_reserved (GiB)": 41.6, "memory/max_active (GiB)": 38.8, "memory/max_allocated (GiB)": 38.8, "ppl": 1.51736, "step": 96, "tokens/total": 3320272, "tokens/train_per_sec_per_gpu": 44.6, "tokens/trainable": 65078} +{"epoch": 0.37890625, "grad_norm": 2.4976298809051514, "learning_rate": 9.53619478457953e-05, "loss": 0.40739500522613525, "memory/device_reserved (GiB)": 41.6, "memory/max_active (GiB)": 37.81, "memory/max_allocated (GiB)": 37.81, "ppl": 1.5029, "step": 97, "tokens/total": 3355296, "tokens/train_per_sec_per_gpu": 41.61, "tokens/trainable": 65815} +{"epoch": 0.3828125, "grad_norm": 4.644741058349609, "learning_rate": 9.523275153154695e-05, "loss": 0.5161807537078857, "memory/device_reserved (GiB)": 41.6, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.67562, "step": 98, "tokens/total": 3389104, "tokens/train_per_sec_per_gpu": 41.85, "tokens/trainable": 66518} +{"epoch": 0.38671875, "grad_norm": 1.9569761753082275, "learning_rate": 9.51018809682839e-05, "loss": 0.4390929341316223, "memory/device_reserved (GiB)": 41.6, "memory/max_active (GiB)": 35.96, "memory/max_allocated (GiB)": 35.96, "ppl": 1.5513, "step": 99, "tokens/total": 3417344, "tokens/train_per_sec_per_gpu": 42.65, "tokens/trainable": 67171} +{"epoch": 0.390625, "grad_norm": 2.7521181106567383, "learning_rate": 9.49693416020645e-05, "loss": 0.46920299530029297, "memory/device_reserved (GiB)": 41.6, "memory/max_active (GiB)": 38.3, "memory/max_allocated (GiB)": 38.3, "ppl": 1.59872, "step": 100, "tokens/total": 3453856, "tokens/train_per_sec_per_gpu": 37.03, "tokens/trainable": 67864} +{"epoch": 0.39453125, "grad_norm": 2.374513864517212, "learning_rate": 9.483513894839276e-05, "loss": 0.44233429431915283, "memory/device_reserved (GiB)": 41.6, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.55634, "step": 101, "tokens/total": 3488256, "tokens/train_per_sec_per_gpu": 40.65, "tokens/trainable": 68581} +{"epoch": 0.3984375, "grad_norm": 4.172671794891357, "learning_rate": 9.469927859198888e-05, "loss": 0.6228840351104736, "memory/device_reserved (GiB)": 41.6, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.8643, "step": 102, "tokens/total": 3521664, "tokens/train_per_sec_per_gpu": 37.43, "tokens/trainable": 69186} +{"epoch": 0.40234375, "grad_norm": 1.8960415124893188, "learning_rate": 9.456176618655689e-05, "loss": 0.4693067967891693, "memory/device_reserved (GiB)": 41.6, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.59889, "step": 103, "tokens/total": 3558368, "tokens/train_per_sec_per_gpu": 39.73, "tokens/trainable": 69857} +{"epoch": 0.40625, "grad_norm": 2.4297776222229004, "learning_rate": 9.442260745454927e-05, "loss": 0.44511425495147705, "memory/device_reserved (GiB)": 41.6, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.56067, "step": 104, "tokens/total": 3594544, "tokens/train_per_sec_per_gpu": 35.78, "tokens/trainable": 70543} +{"epoch": 0.41015625, "grad_norm": 1.454508900642395, "learning_rate": 9.428180818692884e-05, "loss": 0.44237372279167175, "memory/device_reserved (GiB)": 41.6, "memory/max_active (GiB)": 37.07, "memory/max_allocated (GiB)": 37.07, "ppl": 1.5564, "step": 105, "tokens/total": 3628304, "tokens/train_per_sec_per_gpu": 40.77, "tokens/trainable": 71201} +{"epoch": 0.4140625, "grad_norm": 2.469376802444458, "learning_rate": 9.413937424292791e-05, "loss": 0.4352377951145172, "memory/device_reserved (GiB)": 41.6, "memory/max_active (GiB)": 36.65, "memory/max_allocated (GiB)": 36.65, "ppl": 1.54533, "step": 106, "tokens/total": 3660816, "tokens/train_per_sec_per_gpu": 42.64, "tokens/trainable": 71910} +{"epoch": 0.41796875, "grad_norm": 2.503694534301758, "learning_rate": 9.399531154980424e-05, "loss": 0.397834450006485, "memory/device_reserved (GiB)": 41.6, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.4886, "step": 107, "tokens/total": 3697232, "tokens/train_per_sec_per_gpu": 40.73, "tokens/trainable": 72631} +{"epoch": 0.421875, "grad_norm": 1.862091302871704, "learning_rate": 9.384962610259455e-05, "loss": 0.30991020798683167, "memory/device_reserved (GiB)": 41.6, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.3633, "step": 108, "tokens/total": 3732528, "tokens/train_per_sec_per_gpu": 36.52, "tokens/trainable": 73327} +{"epoch": 0.42578125, "grad_norm": 1.8584357500076294, "learning_rate": 9.370232396386494e-05, "loss": 0.4419603645801544, "memory/device_reserved (GiB)": 41.6, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.55575, "step": 109, "tokens/total": 3768384, "tokens/train_per_sec_per_gpu": 36.96, "tokens/trainable": 74028} +{"epoch": 0.4296875, "grad_norm": 2.912550687789917, "learning_rate": 9.355341126345868e-05, "loss": 0.3949897885322571, "memory/device_reserved (GiB)": 41.6, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.48437, "step": 110, "tokens/total": 3805328, "tokens/train_per_sec_per_gpu": 34.73, "tokens/trainable": 74722} +{"epoch": 0.43359375, "grad_norm": 2.0452427864074707, "learning_rate": 9.340289419824107e-05, "loss": 0.3763626217842102, "memory/device_reserved (GiB)": 41.6, "memory/max_active (GiB)": 38.18, "memory/max_allocated (GiB)": 38.18, "ppl": 1.45698, "step": 111, "tokens/total": 3840144, "tokens/train_per_sec_per_gpu": 39.46, "tokens/trainable": 75368} +{"epoch": 0.4375, "grad_norm": 3.6314945220947266, "learning_rate": 9.325077903184159e-05, "loss": 0.5607655048370361, "memory/device_reserved (GiB)": 41.6, "memory/max_active (GiB)": 38.22, "memory/max_allocated (GiB)": 38.22, "ppl": 1.75201, "step": 112, "tokens/total": 3873200, "tokens/train_per_sec_per_gpu": 36.83, "tokens/trainable": 75967} +{"epoch": 0.44140625, "grad_norm": 5.537680625915527, "learning_rate": 9.30970720943932e-05, "loss": 0.3230820894241333, "memory/device_reserved (GiB)": 41.6, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.38138, "step": 113, "tokens/total": 3908032, "tokens/train_per_sec_per_gpu": 42.16, "tokens/trainable": 76740} +{"epoch": 0.4453125, "grad_norm": 5.492731094360352, "learning_rate": 9.2941779782269e-05, "loss": 0.47600191831588745, "memory/device_reserved (GiB)": 41.6, "memory/max_active (GiB)": 36.15, "memory/max_allocated (GiB)": 36.15, "ppl": 1.60963, "step": 114, "tokens/total": 3939376, "tokens/train_per_sec_per_gpu": 41.69, "tokens/trainable": 77423} +{"epoch": 0.44921875, "grad_norm": 2.777881383895874, "learning_rate": 9.278490855781596e-05, "loss": 0.42514973878860474, "memory/device_reserved (GiB)": 41.6, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.52982, "step": 115, "tokens/total": 3974944, "tokens/train_per_sec_per_gpu": 37.31, "tokens/trainable": 78116} +{"epoch": 0.453125, "grad_norm": 4.740851879119873, "learning_rate": 9.262646494908604e-05, "loss": 0.4463016092777252, "memory/device_reserved (GiB)": 41.6, "memory/max_active (GiB)": 38.32, "memory/max_allocated (GiB)": 38.32, "ppl": 1.56252, "step": 116, "tokens/total": 4012496, "tokens/train_per_sec_per_gpu": 37.92, "tokens/trainable": 78793} +{"epoch": 0.45703125, "grad_norm": 2.0342118740081787, "learning_rate": 9.246645554956457e-05, "loss": 0.40813180804252625, "memory/device_reserved (GiB)": 41.6, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.50401, "step": 117, "tokens/total": 4046336, "tokens/train_per_sec_per_gpu": 42.67, "tokens/trainable": 79522} +{"epoch": 0.4609375, "grad_norm": 2.5647947788238525, "learning_rate": 9.230488701789578e-05, "loss": 0.3769294023513794, "memory/device_reserved (GiB)": 41.6, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.4578, "step": 118, "tokens/total": 4081024, "tokens/train_per_sec_per_gpu": 40.74, "tokens/trainable": 80210} +{"epoch": 0.46484375, "grad_norm": 1.7821316719055176, "learning_rate": 9.214176607760577e-05, "loss": 0.41151922941207886, "memory/device_reserved (GiB)": 41.6, "memory/max_active (GiB)": 38.18, "memory/max_allocated (GiB)": 38.18, "ppl": 1.50911, "step": 119, "tokens/total": 4116592, "tokens/train_per_sec_per_gpu": 36.71, "tokens/trainable": 80867} +{"epoch": 0.46875, "grad_norm": 1.4862263202667236, "learning_rate": 9.197709951682268e-05, "loss": 0.37790584564208984, "memory/device_reserved (GiB)": 41.6, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.45923, "step": 120, "tokens/total": 4152176, "tokens/train_per_sec_per_gpu": 37.31, "tokens/trainable": 81567} +{"epoch": 0.47265625, "grad_norm": 1.7153587341308594, "learning_rate": 9.181089418799428e-05, "loss": 0.3331453502178192, "memory/device_reserved (GiB)": 41.75, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.39535, "step": 121, "tokens/total": 4191248, "tokens/train_per_sec_per_gpu": 32.37, "tokens/trainable": 82284} +{"epoch": 0.4765625, "grad_norm": 1.4335383176803589, "learning_rate": 9.164315700760271e-05, "loss": 0.3679426908493042, "memory/device_reserved (GiB)": 41.75, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.44476, "step": 122, "tokens/total": 4226800, "tokens/train_per_sec_per_gpu": 41.2, "tokens/trainable": 83011} +{"epoch": 0.48046875, "grad_norm": 2.356917381286621, "learning_rate": 9.147389495587671e-05, "loss": 0.3940962255001068, "memory/device_reserved (GiB)": 41.75, "memory/max_active (GiB)": 35.85, "memory/max_allocated (GiB)": 35.85, "ppl": 1.48304, "step": 123, "tokens/total": 4256736, "tokens/train_per_sec_per_gpu": 41.87, "tokens/trainable": 83661} +{"epoch": 0.484375, "grad_norm": 10.372278213500977, "learning_rate": 9.130311507650116e-05, "loss": 0.34936004877090454, "memory/device_reserved (GiB)": 41.75, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.41816, "step": 124, "tokens/total": 4289408, "tokens/train_per_sec_per_gpu": 39.16, "tokens/trainable": 84325} +{"epoch": 0.48828125, "grad_norm": 2.4669084548950195, "learning_rate": 9.113082447632394e-05, "loss": 0.43618088960647583, "memory/device_reserved (GiB)": 41.75, "memory/max_active (GiB)": 38.41, "memory/max_allocated (GiB)": 38.41, "ppl": 1.54679, "step": 125, "tokens/total": 4326480, "tokens/train_per_sec_per_gpu": 33.87, "tokens/trainable": 84965} +{"epoch": 0.4921875, "grad_norm": 2.6421148777008057, "learning_rate": 9.09570303250602e-05, "loss": 0.3470087945461273, "memory/device_reserved (GiB)": 41.75, "memory/max_active (GiB)": 36.93, "memory/max_allocated (GiB)": 36.93, "ppl": 1.41483, "step": 126, "tokens/total": 4360160, "tokens/train_per_sec_per_gpu": 38.07, "tokens/trainable": 85624} +{"epoch": 0.49609375, "grad_norm": 1.9876412153244019, "learning_rate": 9.078173985499394e-05, "loss": 0.32877159118652344, "memory/device_reserved (GiB)": 41.75, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.38926, "step": 127, "tokens/total": 4395440, "tokens/train_per_sec_per_gpu": 42.57, "tokens/trainable": 86322} +{"epoch": 0.5, "grad_norm": 1.775503396987915, "learning_rate": 9.060496036067713e-05, "loss": 0.3575822114944458, "memory/device_reserved (GiB)": 41.75, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.42987, "step": 128, "tokens/total": 4429712, "tokens/train_per_sec_per_gpu": 42.06, "tokens/trainable": 87030} +{"epoch": 0.50390625, "grad_norm": 1.9223132133483887, "learning_rate": 9.042669919862615e-05, "loss": 0.4164856970310211, "memory/device_reserved (GiB)": 41.75, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.51662, "step": 129, "tokens/total": 4463744, "tokens/train_per_sec_per_gpu": 42.64, "tokens/trainable": 87741} +{"epoch": 0.5078125, "grad_norm": 3.0679757595062256, "learning_rate": 9.024696378701557e-05, "loss": 0.32860639691352844, "memory/device_reserved (GiB)": 40.05, "memory/max_active (GiB)": 37.43, "memory/max_allocated (GiB)": 37.43, "ppl": 1.38903, "step": 130, "tokens/total": 4496672, "tokens/train_per_sec_per_gpu": 41.55, "tokens/trainable": 88393} +{"epoch": 0.51171875, "grad_norm": 1.7294819355010986, "learning_rate": 9.006576160536948e-05, "loss": 0.4134400486946106, "memory/device_reserved (GiB)": 40.05, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.51201, "step": 131, "tokens/total": 4529568, "tokens/train_per_sec_per_gpu": 39.62, "tokens/trainable": 89017} +{"epoch": 0.515625, "grad_norm": 1.6924993991851807, "learning_rate": 8.988310019425035e-05, "loss": 0.30566221475601196, "memory/device_reserved (GiB)": 40.05, "memory/max_active (GiB)": 36.82, "memory/max_allocated (GiB)": 36.82, "ppl": 1.35752, "step": 132, "tokens/total": 4561856, "tokens/train_per_sec_per_gpu": 46.4, "tokens/trainable": 89752} +{"epoch": 0.51953125, "grad_norm": 2.3902387619018555, "learning_rate": 8.969898715494506e-05, "loss": 0.3890494704246521, "memory/device_reserved (GiB)": 40.05, "memory/max_active (GiB)": 38.15, "memory/max_allocated (GiB)": 38.15, "ppl": 1.47558, "step": 133, "tokens/total": 4596720, "tokens/train_per_sec_per_gpu": 36.97, "tokens/trainable": 90408} +{"epoch": 0.5234375, "grad_norm": 1.5181509256362915, "learning_rate": 8.951343014914869e-05, "loss": 0.2990211248397827, "memory/device_reserved (GiB)": 40.06, "memory/max_active (GiB)": 38.32, "memory/max_allocated (GiB)": 38.32, "ppl": 1.34854, "step": 134, "tokens/total": 4630512, "tokens/train_per_sec_per_gpu": 43.85, "tokens/trainable": 91145} +{"epoch": 0.52734375, "grad_norm": 3.121352195739746, "learning_rate": 8.932643689864568e-05, "loss": 0.3591182231903076, "memory/device_reserved (GiB)": 40.06, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.43207, "step": 135, "tokens/total": 4664640, "tokens/train_per_sec_per_gpu": 40.2, "tokens/trainable": 91816} +{"epoch": 0.53125, "grad_norm": 1.7426899671554565, "learning_rate": 8.913801518498845e-05, "loss": 0.37855637073516846, "memory/device_reserved (GiB)": 40.06, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.46018, "step": 136, "tokens/total": 4700960, "tokens/train_per_sec_per_gpu": 36.47, "tokens/trainable": 92515} +{"epoch": 0.53515625, "grad_norm": 1.749901533126831, "learning_rate": 8.894817284917364e-05, "loss": 0.29418909549713135, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.34204, "step": 137, "tokens/total": 4738848, "tokens/train_per_sec_per_gpu": 38.52, "tokens/trainable": 93276} +{"epoch": 0.5390625, "grad_norm": 1.3049085140228271, "learning_rate": 8.875691779131569e-05, "loss": 0.33237653970718384, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.39428, "step": 138, "tokens/total": 4777632, "tokens/train_per_sec_per_gpu": 32.18, "tokens/trainable": 93929} +{"epoch": 0.54296875, "grad_norm": 1.6193442344665527, "learning_rate": 8.856425797031829e-05, "loss": 0.35012975335121155, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.41925, "step": 139, "tokens/total": 4810816, "tokens/train_per_sec_per_gpu": 36.87, "tokens/trainable": 94579} +{"epoch": 0.546875, "grad_norm": 1.4064973592758179, "learning_rate": 8.837020140354295e-05, "loss": 0.2778596878051758, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.19, "memory/max_allocated (GiB)": 38.19, "ppl": 1.3203, "step": 140, "tokens/total": 4847248, "tokens/train_per_sec_per_gpu": 39.48, "tokens/trainable": 95291} +{"epoch": 0.55078125, "grad_norm": 1.5675361156463623, "learning_rate": 8.817475616647554e-05, "loss": 0.2743667960166931, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.95, "memory/max_allocated (GiB)": 36.95, "ppl": 1.3157, "step": 141, "tokens/total": 4880480, "tokens/train_per_sec_per_gpu": 42.22, "tokens/trainable": 95970} +{"epoch": 0.5546875, "grad_norm": 2.2964322566986084, "learning_rate": 8.797793039239017e-05, "loss": 0.37774643301963806, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.45899, "step": 142, "tokens/total": 4912512, "tokens/train_per_sec_per_gpu": 39.88, "tokens/trainable": 96616} +{"epoch": 0.55859375, "grad_norm": 1.6718605756759644, "learning_rate": 8.777973227201069e-05, "loss": 0.29600510001182556, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.34448, "step": 143, "tokens/total": 4946768, "tokens/train_per_sec_per_gpu": 39.88, "tokens/trainable": 97328} +{"epoch": 0.5625, "grad_norm": 1.6825835704803467, "learning_rate": 8.758017005316988e-05, "loss": 0.25918275117874146, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.29587, "step": 144, "tokens/total": 4984288, "tokens/train_per_sec_per_gpu": 37.66, "tokens/trainable": 98033} +{"epoch": 0.56640625, "grad_norm": 2.516068458557129, "learning_rate": 8.737925204046629e-05, "loss": 0.40808674693107605, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 1.50394, "step": 145, "tokens/total": 5015456, "tokens/train_per_sec_per_gpu": 38.34, "tokens/trainable": 98652} +{"epoch": 0.5703125, "grad_norm": 1.984848141670227, "learning_rate": 8.717698659491851e-05, "loss": 0.3396613895893097, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.7, "memory/max_allocated (GiB)": 38.7, "ppl": 1.40447, "step": 146, "tokens/total": 5050912, "tokens/train_per_sec_per_gpu": 37.45, "tokens/trainable": 99340} +{"epoch": 0.57421875, "grad_norm": 1.7206379175186157, "learning_rate": 8.697338213361735e-05, "loss": 0.28354841470718384, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.32783, "step": 147, "tokens/total": 5087136, "tokens/train_per_sec_per_gpu": 40.77, "tokens/trainable": 100047} +{"epoch": 0.578125, "grad_norm": 4.9751410484313965, "learning_rate": 8.676844712937552e-05, "loss": 0.34134769439697266, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.9, "memory/max_allocated (GiB)": 36.9, "ppl": 1.40684, "step": 148, "tokens/total": 5120736, "tokens/train_per_sec_per_gpu": 37.43, "tokens/trainable": 100686} +{"epoch": 0.58203125, "grad_norm": 2.450953483581543, "learning_rate": 8.656219011037509e-05, "loss": 0.3935505151748657, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.1, "memory/max_allocated (GiB)": 36.1, "ppl": 1.48223, "step": 149, "tokens/total": 5151936, "tokens/train_per_sec_per_gpu": 39.97, "tokens/trainable": 101343} +{"epoch": 0.5859375, "grad_norm": 2.171674966812134, "learning_rate": 8.63546196598125e-05, "loss": 0.32965683937072754, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 39.14, "memory/max_allocated (GiB)": 39.14, "ppl": 1.39049, "step": 150, "tokens/total": 5188464, "tokens/train_per_sec_per_gpu": 37.46, "tokens/trainable": 102027} +{"epoch": 0.58984375, "grad_norm": 1.2866051197052002, "learning_rate": 8.614574441554145e-05, "loss": 0.24338756501674652, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.27556, "step": 151, "tokens/total": 5226400, "tokens/train_per_sec_per_gpu": 37.38, "tokens/trainable": 102764} +{"epoch": 0.59375, "grad_norm": 1.5050500631332397, "learning_rate": 8.593557306971349e-05, "loss": 0.29362940788269043, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.34129, "step": 152, "tokens/total": 5262704, "tokens/train_per_sec_per_gpu": 39.18, "tokens/trainable": 103422} +{"epoch": 0.59765625, "grad_norm": 2.0640451908111572, "learning_rate": 8.572411436841618e-05, "loss": 0.3320331871509552, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.3938, "step": 153, "tokens/total": 5301328, "tokens/train_per_sec_per_gpu": 33.91, "tokens/trainable": 104133} +{"epoch": 0.6015625, "grad_norm": 2.8128559589385986, "learning_rate": 8.551137711130922e-05, "loss": 0.33887654542922974, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.23, "memory/max_allocated (GiB)": 37.23, "ppl": 1.40337, "step": 154, "tokens/total": 5334416, "tokens/train_per_sec_per_gpu": 43.23, "tokens/trainable": 104809} +{"epoch": 0.60546875, "grad_norm": 1.5676357746124268, "learning_rate": 8.529737015125824e-05, "loss": 0.29448071122169495, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.34243, "step": 155, "tokens/total": 5371760, "tokens/train_per_sec_per_gpu": 32.01, "tokens/trainable": 105423} +{"epoch": 0.609375, "grad_norm": 1.8897624015808105, "learning_rate": 8.508210239396639e-05, "loss": 0.32398802042007446, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.38263, "step": 156, "tokens/total": 5405600, "tokens/train_per_sec_per_gpu": 46.41, "tokens/trainable": 106176} +{"epoch": 0.61328125, "grad_norm": 2.4129700660705566, "learning_rate": 8.486558279760375e-05, "loss": 0.29605793952941895, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.61, "memory/max_allocated (GiB)": 37.61, "ppl": 1.34455, "step": 157, "tokens/total": 5441008, "tokens/train_per_sec_per_gpu": 42.44, "tokens/trainable": 106904} +{"epoch": 0.6171875, "grad_norm": 2.382239580154419, "learning_rate": 8.464782037243449e-05, "loss": 0.29073452949523926, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.07, "memory/max_allocated (GiB)": 38.07, "ppl": 1.33741, "step": 158, "tokens/total": 5477472, "tokens/train_per_sec_per_gpu": 42.29, "tokens/trainable": 107652} +{"epoch": 0.62109375, "grad_norm": 1.6009176969528198, "learning_rate": 8.442882418044202e-05, "loss": 0.25726377964019775, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 1.29339, "step": 159, "tokens/total": 5514096, "tokens/train_per_sec_per_gpu": 37.48, "tokens/trainable": 108348} +{"epoch": 0.625, "grad_norm": 2.128638982772827, "learning_rate": 8.420860333495179e-05, "loss": 0.2770332098007202, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.2, "memory/max_allocated (GiB)": 38.2, "ppl": 1.31921, "step": 160, "tokens/total": 5549872, "tokens/train_per_sec_per_gpu": 42.85, "tokens/trainable": 109046} +{"epoch": 0.62890625, "grad_norm": 4.697608947753906, "learning_rate": 8.398716700025208e-05, "loss": 0.28764963150024414, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.33329, "step": 161, "tokens/total": 5586960, "tokens/train_per_sec_per_gpu": 42.01, "tokens/trainable": 109796} +{"epoch": 0.6328125, "grad_norm": 2.7756686210632324, "learning_rate": 8.376452439121266e-05, "loss": 0.34285521507263184, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.31, "memory/max_allocated (GiB)": 36.31, "ppl": 1.40896, "step": 162, "tokens/total": 5618992, "tokens/train_per_sec_per_gpu": 37.13, "tokens/trainable": 110412} +{"epoch": 0.63671875, "grad_norm": 6.568652153015137, "learning_rate": 8.354068477290124e-05, "loss": 0.35219234228134155, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.43, "memory/max_allocated (GiB)": 37.43, "ppl": 1.42218, "step": 163, "tokens/total": 5651008, "tokens/train_per_sec_per_gpu": 37.0, "tokens/trainable": 111036} +{"epoch": 0.640625, "grad_norm": 1.6904163360595703, "learning_rate": 8.331565746019807e-05, "loss": 0.3392602801322937, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.81, "memory/max_allocated (GiB)": 38.81, "ppl": 1.40391, "step": 164, "tokens/total": 5687808, "tokens/train_per_sec_per_gpu": 33.99, "tokens/trainable": 111736} +{"epoch": 0.64453125, "grad_norm": 1.4187194108963013, "learning_rate": 8.308945181740812e-05, "loss": 0.295134037733078, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.45, "memory/max_allocated (GiB)": 38.45, "ppl": 1.34331, "step": 165, "tokens/total": 5721088, "tokens/train_per_sec_per_gpu": 40.42, "tokens/trainable": 112457} +{"epoch": 0.6484375, "grad_norm": 2.0254387855529785, "learning_rate": 8.286207725787153e-05, "loss": 0.20088998973369598, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.22249, "step": 166, "tokens/total": 5756016, "tokens/train_per_sec_per_gpu": 48.66, "tokens/trainable": 113279} +{"epoch": 0.65234375, "grad_norm": 2.0240721702575684, "learning_rate": 8.263354324357182e-05, "loss": 0.3223879635334015, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.38042, "step": 167, "tokens/total": 5788080, "tokens/train_per_sec_per_gpu": 41.75, "tokens/trainable": 113918} +{"epoch": 0.65625, "grad_norm": 1.7174606323242188, "learning_rate": 8.240385928474219e-05, "loss": 0.27589866518974304, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.31771, "step": 168, "tokens/total": 5821088, "tokens/train_per_sec_per_gpu": 40.2, "tokens/trainable": 114573} +{"epoch": 0.66015625, "grad_norm": 1.683184027671814, "learning_rate": 8.217303493946967e-05, "loss": 0.2746450901031494, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.18, "memory/max_allocated (GiB)": 38.18, "ppl": 1.31606, "step": 169, "tokens/total": 5854336, "tokens/train_per_sec_per_gpu": 40.52, "tokens/trainable": 115186} +{"epoch": 0.6640625, "grad_norm": 1.752920389175415, "learning_rate": 8.194107981329746e-05, "loss": 0.2537972629070282, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.28891, "step": 170, "tokens/total": 5887312, "tokens/train_per_sec_per_gpu": 44.81, "tokens/trainable": 115895} +{"epoch": 0.66796875, "grad_norm": 2.2608823776245117, "learning_rate": 8.170800355882518e-05, "loss": 0.33724111318588257, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.40108, "step": 171, "tokens/total": 5924400, "tokens/train_per_sec_per_gpu": 36.47, "tokens/trainable": 116610} +{"epoch": 0.671875, "grad_norm": 2.1722168922424316, "learning_rate": 8.147381587530713e-05, "loss": 0.31697362661361694, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.51, "memory/max_allocated (GiB)": 38.51, "ppl": 1.37297, "step": 172, "tokens/total": 5958608, "tokens/train_per_sec_per_gpu": 35.59, "tokens/trainable": 117235} +{"epoch": 0.67578125, "grad_norm": 2.2386133670806885, "learning_rate": 8.123852650824877e-05, "loss": 0.2930545210838318, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.34052, "step": 173, "tokens/total": 5992928, "tokens/train_per_sec_per_gpu": 45.44, "tokens/trainable": 117946} +{"epoch": 0.6796875, "grad_norm": 1.812167763710022, "learning_rate": 8.100214524900103e-05, "loss": 0.2881283164024353, "memory/device_reserved (GiB)": 41.93, "memory/max_active (GiB)": 35.49, "memory/max_allocated (GiB)": 35.49, "ppl": 1.33393, "step": 174, "tokens/total": 6022080, "tokens/train_per_sec_per_gpu": 40.23, "tokens/trainable": 118544} +{"epoch": 0.68359375, "grad_norm": 2.1863811016082764, "learning_rate": 8.076468193435301e-05, "loss": 0.26740849018096924, "memory/device_reserved (GiB)": 41.93, "memory/max_active (GiB)": 37.67, "memory/max_allocated (GiB)": 37.67, "ppl": 1.30657, "step": 175, "tokens/total": 6054144, "tokens/train_per_sec_per_gpu": 42.77, "tokens/trainable": 119221} +{"epoch": 0.6875, "grad_norm": 2.2042653560638428, "learning_rate": 8.052614644612253e-05, "loss": 0.27217113971710205, "memory/device_reserved (GiB)": 41.93, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.31281, "step": 176, "tokens/total": 6088672, "tokens/train_per_sec_per_gpu": 35.63, "tokens/trainable": 119836} +{"epoch": 0.69140625, "grad_norm": 2.3795838356018066, "learning_rate": 8.028654871074489e-05, "loss": 0.2855570912361145, "memory/device_reserved (GiB)": 41.93, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.3305, "step": 177, "tokens/total": 6118656, "tokens/train_per_sec_per_gpu": 42.73, "tokens/trainable": 120480} +{"epoch": 0.6953125, "grad_norm": 1.9894280433654785, "learning_rate": 8.004589869885986e-05, "loss": 0.23730917274951935, "memory/device_reserved (GiB)": 41.93, "memory/max_active (GiB)": 36.9, "memory/max_allocated (GiB)": 36.9, "ppl": 1.26783, "step": 178, "tokens/total": 6149808, "tokens/train_per_sec_per_gpu": 43.65, "tokens/trainable": 121137} +{"epoch": 0.69921875, "grad_norm": 2.3809967041015625, "learning_rate": 7.980420642489674e-05, "loss": 0.29730409383773804, "memory/device_reserved (GiB)": 41.93, "memory/max_active (GiB)": 36.15, "memory/max_allocated (GiB)": 36.15, "ppl": 1.34622, "step": 179, "tokens/total": 6177296, "tokens/train_per_sec_per_gpu": 50.4, "tokens/trainable": 121786} +{"epoch": 0.703125, "grad_norm": 1.934584617614746, "learning_rate": 7.95614819466576e-05, "loss": 0.18502771854400635, "memory/device_reserved (GiB)": 41.93, "memory/max_active (GiB)": 37.13, "memory/max_allocated (GiB)": 37.13, "ppl": 1.20325, "step": 180, "tokens/total": 6211408, "tokens/train_per_sec_per_gpu": 43.19, "tokens/trainable": 122544} +{"epoch": 0.70703125, "grad_norm": 1.7914190292358398, "learning_rate": 7.931773536489872e-05, "loss": 0.25853753089904785, "memory/device_reserved (GiB)": 41.93, "memory/max_active (GiB)": 35.72, "memory/max_allocated (GiB)": 35.72, "ppl": 1.29503, "step": 181, "tokens/total": 6240992, "tokens/train_per_sec_per_gpu": 35.54, "tokens/trainable": 123136} +{"epoch": 0.7109375, "grad_norm": 2.31095814704895, "learning_rate": 7.907297682291035e-05, "loss": 0.23988088965415955, "memory/device_reserved (GiB)": 41.93, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.2711, "step": 182, "tokens/total": 6273984, "tokens/train_per_sec_per_gpu": 42.04, "tokens/trainable": 123776} +{"epoch": 0.71484375, "grad_norm": 4.722596168518066, "learning_rate": 7.882721650609442e-05, "loss": 0.2610148787498474, "memory/device_reserved (GiB)": 41.93, "memory/max_active (GiB)": 36.11, "memory/max_allocated (GiB)": 36.11, "ppl": 1.29825, "step": 183, "tokens/total": 6305440, "tokens/train_per_sec_per_gpu": 38.72, "tokens/trainable": 124402} +{"epoch": 0.71875, "grad_norm": 3.0927906036376953, "learning_rate": 7.85804646415409e-05, "loss": 0.2588197588920593, "memory/device_reserved (GiB)": 41.93, "memory/max_active (GiB)": 38.33, "memory/max_allocated (GiB)": 38.33, "ppl": 1.2954, "step": 184, "tokens/total": 6339312, "tokens/train_per_sec_per_gpu": 38.62, "tokens/trainable": 125125} +{"epoch": 0.72265625, "grad_norm": 2.3648905754089355, "learning_rate": 7.833273149760207e-05, "loss": 0.2747686505317688, "memory/device_reserved (GiB)": 41.93, "memory/max_active (GiB)": 37.39, "memory/max_allocated (GiB)": 37.39, "ppl": 1.31623, "step": 185, "tokens/total": 6373392, "tokens/train_per_sec_per_gpu": 37.38, "tokens/trainable": 125786} +{"epoch": 0.7265625, "grad_norm": 3.2000949382781982, "learning_rate": 7.808402738346527e-05, "loss": 0.2870525121688843, "memory/device_reserved (GiB)": 41.93, "memory/max_active (GiB)": 37.2, "memory/max_allocated (GiB)": 37.2, "ppl": 1.33249, "step": 186, "tokens/total": 6407472, "tokens/train_per_sec_per_gpu": 34.19, "tokens/trainable": 126404} +{"epoch": 0.73046875, "grad_norm": 3.090592384338379, "learning_rate": 7.783436264872382e-05, "loss": 0.23734791576862335, "memory/device_reserved (GiB)": 41.93, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.26788, "step": 187, "tokens/total": 6442016, "tokens/train_per_sec_per_gpu": 37.56, "tokens/trainable": 127054} +{"epoch": 0.734375, "grad_norm": 3.447571277618408, "learning_rate": 7.758374768294647e-05, "loss": 0.29191672801971436, "memory/device_reserved (GiB)": 41.93, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.33899, "step": 188, "tokens/total": 6478384, "tokens/train_per_sec_per_gpu": 31.45, "tokens/trainable": 127687} +{"epoch": 0.73828125, "grad_norm": 2.0816712379455566, "learning_rate": 7.733219291524489e-05, "loss": 0.27911391854286194, "memory/device_reserved (GiB)": 41.93, "memory/max_active (GiB)": 39.32, "memory/max_allocated (GiB)": 39.32, "ppl": 1.32196, "step": 189, "tokens/total": 6515088, "tokens/train_per_sec_per_gpu": 40.14, "tokens/trainable": 128405} +{"epoch": 0.7421875, "grad_norm": 3.336054801940918, "learning_rate": 7.707970881383977e-05, "loss": 0.29486003518104553, "memory/device_reserved (GiB)": 41.93, "memory/max_active (GiB)": 38.33, "memory/max_allocated (GiB)": 38.33, "ppl": 1.34294, "step": 190, "tokens/total": 6552576, "tokens/train_per_sec_per_gpu": 32.84, "tokens/trainable": 129026} +{"epoch": 0.74609375, "grad_norm": 2.1646313667297363, "learning_rate": 7.682630588562518e-05, "loss": 0.258897066116333, "memory/device_reserved (GiB)": 41.93, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.2955, "step": 191, "tokens/total": 6585584, "tokens/train_per_sec_per_gpu": 36.92, "tokens/trainable": 129630} +{"epoch": 0.75, "grad_norm": 2.8016281127929688, "learning_rate": 7.657199467573129e-05, "loss": 0.24407929182052612, "memory/device_reserved (GiB)": 41.93, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.27645, "step": 192, "tokens/total": 6623344, "tokens/train_per_sec_per_gpu": 42.31, "tokens/trainable": 130354} +{"epoch": 0.75390625, "grad_norm": 2.315363883972168, "learning_rate": 7.631678576708561e-05, "loss": 0.2618207335472107, "memory/device_reserved (GiB)": 41.93, "memory/max_active (GiB)": 37.73, "memory/max_allocated (GiB)": 37.73, "ppl": 1.29929, "step": 193, "tokens/total": 6658928, "tokens/train_per_sec_per_gpu": 41.35, "tokens/trainable": 131074} +{"epoch": 0.7578125, "grad_norm": 1.418953537940979, "learning_rate": 7.606068977997255e-05, "loss": 0.2706204950809479, "memory/device_reserved (GiB)": 41.93, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.31078, "step": 194, "tokens/total": 6693536, "tokens/train_per_sec_per_gpu": 39.46, "tokens/trainable": 131725} +{"epoch": 0.76171875, "grad_norm": 1.3634283542633057, "learning_rate": 7.580371737159148e-05, "loss": 0.2322079837322235, "memory/device_reserved (GiB)": 41.93, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.26138, "step": 195, "tokens/total": 6726032, "tokens/train_per_sec_per_gpu": 41.19, "tokens/trainable": 132381} +{"epoch": 0.765625, "grad_norm": 1.3089996576309204, "learning_rate": 7.554587923561324e-05, "loss": 0.2404736876487732, "memory/device_reserved (GiB)": 41.93, "memory/max_active (GiB)": 39.14, "memory/max_allocated (GiB)": 39.14, "ppl": 1.27185, "step": 196, "tokens/total": 6762672, "tokens/train_per_sec_per_gpu": 40.96, "tokens/trainable": 133177} +{"epoch": 0.76953125, "grad_norm": 2.0928003787994385, "learning_rate": 7.528718610173511e-05, "loss": 0.2944469451904297, "memory/device_reserved (GiB)": 41.93, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.34238, "step": 197, "tokens/total": 6796688, "tokens/train_per_sec_per_gpu": 37.99, "tokens/trainable": 133814} +{"epoch": 0.7734375, "grad_norm": 1.886793851852417, "learning_rate": 7.502764873523431e-05, "loss": 0.23405560851097107, "memory/device_reserved (GiB)": 41.93, "memory/max_active (GiB)": 39.48, "memory/max_allocated (GiB)": 39.48, "ppl": 1.26371, "step": 198, "tokens/total": 6832592, "tokens/train_per_sec_per_gpu": 40.48, "tokens/trainable": 134490} +{"epoch": 0.77734375, "grad_norm": 1.819624900817871, "learning_rate": 7.476727793652011e-05, "loss": 0.2353014051914215, "memory/device_reserved (GiB)": 41.93, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.26529, "step": 199, "tokens/total": 6863712, "tokens/train_per_sec_per_gpu": 44.8, "tokens/trainable": 135129} +{"epoch": 0.78125, "grad_norm": 7.283644676208496, "learning_rate": 7.450608454068415e-05, "loss": 0.23346155881881714, "memory/device_reserved (GiB)": 41.93, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.26296, "step": 200, "tokens/total": 6899296, "tokens/train_per_sec_per_gpu": 39.61, "tokens/trainable": 135830} +{"epoch": 0.78515625, "grad_norm": 3.4145941734313965, "learning_rate": 7.424407941704987e-05, "loss": 0.22267231345176697, "memory/device_reserved (GiB)": 41.93, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.24941, "step": 201, "tokens/total": 6930544, "tokens/train_per_sec_per_gpu": 43.38, "tokens/trainable": 136466} +{"epoch": 0.7890625, "grad_norm": 1.8740507364273071, "learning_rate": 7.398127346871986e-05, "loss": 0.18263652920722961, "memory/device_reserved (GiB)": 42.38, "memory/max_active (GiB)": 37.37, "memory/max_allocated (GiB)": 37.37, "ppl": 1.20038, "step": 202, "tokens/total": 6964512, "tokens/train_per_sec_per_gpu": 44.16, "tokens/trainable": 137198} +{"epoch": 0.79296875, "grad_norm": 1.6300749778747559, "learning_rate": 7.371767763212238e-05, "loss": 0.22892631590366364, "memory/device_reserved (GiB)": 42.38, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.25725, "step": 203, "tokens/total": 7000096, "tokens/train_per_sec_per_gpu": 43.76, "tokens/trainable": 137950} +{"epoch": 0.796875, "grad_norm": 2.5964372158050537, "learning_rate": 7.345330287655617e-05, "loss": 0.2210806906223297, "memory/device_reserved (GiB)": 42.38, "memory/max_active (GiB)": 36.76, "memory/max_allocated (GiB)": 36.76, "ppl": 1.24742, "step": 204, "tokens/total": 7031984, "tokens/train_per_sec_per_gpu": 44.33, "tokens/trainable": 138643} +{"epoch": 0.80078125, "grad_norm": 1.4463247060775757, "learning_rate": 7.31881602037339e-05, "loss": 0.19691836833953857, "memory/device_reserved (GiB)": 42.38, "memory/max_active (GiB)": 36.66, "memory/max_allocated (GiB)": 36.66, "ppl": 1.21764, "step": 205, "tokens/total": 7064768, "tokens/train_per_sec_per_gpu": 37.66, "tokens/trainable": 139299} +{"epoch": 0.8046875, "grad_norm": 3.661888360977173, "learning_rate": 7.29222606473245e-05, "loss": 0.2811765968799591, "memory/device_reserved (GiB)": 42.38, "memory/max_active (GiB)": 37.07, "memory/max_allocated (GiB)": 37.07, "ppl": 1.32469, "step": 206, "tokens/total": 7096560, "tokens/train_per_sec_per_gpu": 42.07, "tokens/trainable": 139918} +{"epoch": 0.80859375, "grad_norm": 3.3527190685272217, "learning_rate": 7.265561527249383e-05, "loss": 0.28116291761398315, "memory/device_reserved (GiB)": 42.38, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.32467, "step": 207, "tokens/total": 7130816, "tokens/train_per_sec_per_gpu": 37.76, "tokens/trainable": 140616} +{"epoch": 0.8125, "grad_norm": 1.7451330423355103, "learning_rate": 7.238823517544436e-05, "loss": 0.2080162763595581, "memory/device_reserved (GiB)": 42.38, "memory/max_active (GiB)": 39.35, "memory/max_allocated (GiB)": 39.35, "ppl": 1.23123, "step": 208, "tokens/total": 7165680, "tokens/train_per_sec_per_gpu": 44.2, "tokens/trainable": 141302} +{"epoch": 0.81640625, "grad_norm": 1.6181395053863525, "learning_rate": 7.212013148295333e-05, "loss": 0.17891333997249603, "memory/device_reserved (GiB)": 42.38, "memory/max_active (GiB)": 37.58, "memory/max_allocated (GiB)": 37.58, "ppl": 1.19592, "step": 209, "tokens/total": 7200288, "tokens/train_per_sec_per_gpu": 40.47, "tokens/trainable": 142028} +{"epoch": 0.8203125, "grad_norm": 5.52781867980957, "learning_rate": 7.185131535190975e-05, "loss": 0.2523344159126282, "memory/device_reserved (GiB)": 42.38, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.28703, "step": 210, "tokens/total": 7237360, "tokens/train_per_sec_per_gpu": 34.56, "tokens/trainable": 142697} +{"epoch": 0.82421875, "grad_norm": 2.7720561027526855, "learning_rate": 7.158179796885005e-05, "loss": 0.29905959963798523, "memory/device_reserved (GiB)": 42.38, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.34859, "step": 211, "tokens/total": 7275808, "tokens/train_per_sec_per_gpu": 38.55, "tokens/trainable": 143366} +{"epoch": 0.828125, "grad_norm": 1.8654240369796753, "learning_rate": 7.131159054949273e-05, "loss": 0.2197706699371338, "memory/device_reserved (GiB)": 42.38, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.24579, "step": 212, "tokens/total": 7312736, "tokens/train_per_sec_per_gpu": 38.18, "tokens/trainable": 144084} +{"epoch": 0.83203125, "grad_norm": 6.985602855682373, "learning_rate": 7.104070433827139e-05, "loss": 0.20309937000274658, "memory/device_reserved (GiB)": 42.38, "memory/max_active (GiB)": 35.82, "memory/max_allocated (GiB)": 35.82, "ppl": 1.22519, "step": 213, "tokens/total": 7343056, "tokens/train_per_sec_per_gpu": 41.24, "tokens/trainable": 144734} +{"epoch": 0.8359375, "grad_norm": 1.6149787902832031, "learning_rate": 7.076915060786705e-05, "loss": 0.18694865703582764, "memory/device_reserved (GiB)": 42.38, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.20557, "step": 214, "tokens/total": 7377392, "tokens/train_per_sec_per_gpu": 44.78, "tokens/trainable": 145485} +{"epoch": 0.83984375, "grad_norm": 88.44336700439453, "learning_rate": 7.049694065873882e-05, "loss": 0.23353953659534454, "memory/device_reserved (GiB)": 42.38, "memory/max_active (GiB)": 37.93, "memory/max_allocated (GiB)": 37.93, "ppl": 1.26306, "step": 215, "tokens/total": 7412224, "tokens/train_per_sec_per_gpu": 37.74, "tokens/trainable": 146155} +{"epoch": 0.84375, "grad_norm": 3.0015127658843994, "learning_rate": 7.022408581865382e-05, "loss": 0.24911846220493317, "memory/device_reserved (GiB)": 42.38, "memory/max_active (GiB)": 37.12, "memory/max_allocated (GiB)": 37.12, "ppl": 1.28289, "step": 216, "tokens/total": 7444592, "tokens/train_per_sec_per_gpu": 38.3, "tokens/trainable": 146777} +{"epoch": 0.84765625, "grad_norm": 1.4071767330169678, "learning_rate": 6.99505974422157e-05, "loss": 0.18801631033420563, "memory/device_reserved (GiB)": 42.38, "memory/max_active (GiB)": 36.07, "memory/max_allocated (GiB)": 36.07, "ppl": 1.20685, "step": 217, "tokens/total": 7475312, "tokens/train_per_sec_per_gpu": 48.89, "tokens/trainable": 147456} +{"epoch": 0.8515625, "grad_norm": 2.577003002166748, "learning_rate": 6.967648691039213e-05, "loss": 0.21101844310760498, "memory/device_reserved (GiB)": 42.38, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.23494, "step": 218, "tokens/total": 7509280, "tokens/train_per_sec_per_gpu": 40.26, "tokens/trainable": 148191} +{"epoch": 0.85546875, "grad_norm": 1.5445208549499512, "learning_rate": 6.940176563004123e-05, "loss": 0.20572426915168762, "memory/device_reserved (GiB)": 42.38, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.22841, "step": 219, "tokens/total": 7545264, "tokens/train_per_sec_per_gpu": 39.07, "tokens/trainable": 148827} +{"epoch": 0.859375, "grad_norm": 6.337987899780273, "learning_rate": 6.912644503343682e-05, "loss": 0.20264601707458496, "memory/device_reserved (GiB)": 42.38, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.22464, "step": 220, "tokens/total": 7581568, "tokens/train_per_sec_per_gpu": 38.98, "tokens/trainable": 149510} +{"epoch": 0.86328125, "grad_norm": 1.532532811164856, "learning_rate": 6.885053657779273e-05, "loss": 0.21673330664634705, "memory/device_reserved (GiB)": 42.38, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.24201, "step": 221, "tokens/total": 7615472, "tokens/train_per_sec_per_gpu": 29.66, "tokens/trainable": 150101} +{"epoch": 0.8671875, "grad_norm": 1.4731062650680542, "learning_rate": 6.857405174478604e-05, "loss": 0.20169050991535187, "memory/device_reserved (GiB)": 42.38, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.22347, "step": 222, "tokens/total": 7649456, "tokens/train_per_sec_per_gpu": 41.96, "tokens/trainable": 150757} +{"epoch": 0.87109375, "grad_norm": 2.9439263343811035, "learning_rate": 6.82970020400792e-05, "loss": 0.22305694222450256, "memory/device_reserved (GiB)": 42.38, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.24989, "step": 223, "tokens/total": 7686112, "tokens/train_per_sec_per_gpu": 41.85, "tokens/trainable": 151502} +{"epoch": 0.875, "grad_norm": 3.738389492034912, "learning_rate": 6.801939899284132e-05, "loss": 0.2431585043668747, "memory/device_reserved (GiB)": 42.38, "memory/max_active (GiB)": 35.19, "memory/max_allocated (GiB)": 35.19, "ppl": 1.27527, "step": 224, "tokens/total": 7714144, "tokens/train_per_sec_per_gpu": 45.27, "tokens/trainable": 152102} +{"epoch": 0.87890625, "grad_norm": 5.170564651489258, "learning_rate": 6.774125415526827e-05, "loss": 0.2591360807418823, "memory/device_reserved (GiB)": 42.38, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 1.29581, "step": 225, "tokens/total": 7745936, "tokens/train_per_sec_per_gpu": 44.31, "tokens/trainable": 152766} +{"epoch": 0.8828125, "grad_norm": 3.209642171859741, "learning_rate": 6.746257910210214e-05, "loss": 0.22951370477676392, "memory/device_reserved (GiB)": 42.38, "memory/max_active (GiB)": 39.18, "memory/max_allocated (GiB)": 39.18, "ppl": 1.25799, "step": 226, "tokens/total": 7784016, "tokens/train_per_sec_per_gpu": 33.42, "tokens/trainable": 153411} +{"epoch": 0.88671875, "grad_norm": 2.5832080841064453, "learning_rate": 6.718338543014937e-05, "loss": 0.2044340968132019, "memory/device_reserved (GiB)": 42.38, "memory/max_active (GiB)": 36.42, "memory/max_allocated (GiB)": 36.42, "ppl": 1.22683, "step": 227, "tokens/total": 7814096, "tokens/train_per_sec_per_gpu": 50.58, "tokens/trainable": 154089} +{"epoch": 0.890625, "grad_norm": 1.79071044921875, "learning_rate": 6.69036847577983e-05, "loss": 0.23075321316719055, "memory/device_reserved (GiB)": 42.38, "memory/max_active (GiB)": 36.03, "memory/max_allocated (GiB)": 36.03, "ppl": 1.25955, "step": 228, "tokens/total": 7844816, "tokens/train_per_sec_per_gpu": 45.13, "tokens/trainable": 154698} +{"epoch": 0.89453125, "grad_norm": 1.884636640548706, "learning_rate": 6.662348872453553e-05, "loss": 0.19984742999076843, "memory/device_reserved (GiB)": 42.38, "memory/max_active (GiB)": 36.06, "memory/max_allocated (GiB)": 36.06, "ppl": 1.22122, "step": 229, "tokens/total": 7874416, "tokens/train_per_sec_per_gpu": 39.47, "tokens/trainable": 155309} +{"epoch": 0.8984375, "grad_norm": 2.1247823238372803, "learning_rate": 6.63428089904618e-05, "loss": 0.19815023243427277, "memory/device_reserved (GiB)": 42.38, "memory/max_active (GiB)": 37.58, "memory/max_allocated (GiB)": 37.58, "ppl": 1.21915, "step": 230, "tokens/total": 7909024, "tokens/train_per_sec_per_gpu": 35.23, "tokens/trainable": 155955} +{"epoch": 0.90234375, "grad_norm": 1.6725749969482422, "learning_rate": 6.60616572358065e-05, "loss": 0.20620959997177124, "memory/device_reserved (GiB)": 42.38, "memory/max_active (GiB)": 38.47, "memory/max_allocated (GiB)": 38.47, "ppl": 1.22901, "step": 231, "tokens/total": 7947040, "tokens/train_per_sec_per_gpu": 32.38, "tokens/trainable": 156599} +{"epoch": 0.90625, "grad_norm": 1.4440810680389404, "learning_rate": 6.578004516044172e-05, "loss": 0.19838617742061615, "memory/device_reserved (GiB)": 42.38, "memory/max_active (GiB)": 36.47, "memory/max_allocated (GiB)": 36.47, "ppl": 1.21943, "step": 232, "tokens/total": 7979200, "tokens/train_per_sec_per_gpu": 42.88, "tokens/trainable": 157357} +{"epoch": 0.91015625, "grad_norm": 1.5152943134307861, "learning_rate": 6.549798448339548e-05, "loss": 0.2583959102630615, "memory/device_reserved (GiB)": 42.38, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 1.29485, "step": 233, "tokens/total": 8011792, "tokens/train_per_sec_per_gpu": 36.39, "tokens/trainable": 157945} +{"epoch": 0.9140625, "grad_norm": 1.7868000268936157, "learning_rate": 6.521548694236384e-05, "loss": 0.2534174919128418, "memory/device_reserved (GiB)": 42.38, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.28842, "step": 234, "tokens/total": 8047424, "tokens/train_per_sec_per_gpu": 37.88, "tokens/trainable": 158628} +{"epoch": 0.91796875, "grad_norm": 2.602416515350342, "learning_rate": 6.493256429322259e-05, "loss": 0.19218012690544128, "memory/device_reserved (GiB)": 42.38, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.21189, "step": 235, "tokens/total": 8086944, "tokens/train_per_sec_per_gpu": 43.19, "tokens/trainable": 159390} +{"epoch": 0.921875, "grad_norm": 2.3819093704223633, "learning_rate": 6.464922830953799e-05, "loss": 0.20219430327415466, "memory/device_reserved (GiB)": 42.38, "memory/max_active (GiB)": 37.56, "memory/max_allocated (GiB)": 37.56, "ppl": 1.22409, "step": 236, "tokens/total": 8120816, "tokens/train_per_sec_per_gpu": 39.05, "tokens/trainable": 160077} +{"epoch": 0.92578125, "grad_norm": 2.3495161533355713, "learning_rate": 6.436549078207688e-05, "loss": 0.229032963514328, "memory/device_reserved (GiB)": 42.38, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.25738, "step": 237, "tokens/total": 8153952, "tokens/train_per_sec_per_gpu": 42.44, "tokens/trainable": 160728} +{"epoch": 0.9296875, "grad_norm": 1.8229001760482788, "learning_rate": 6.408136351831592e-05, "loss": 0.18738900125026703, "memory/device_reserved (GiB)": 42.38, "memory/max_active (GiB)": 38.3, "memory/max_allocated (GiB)": 38.3, "ppl": 1.2061, "step": 238, "tokens/total": 8189952, "tokens/train_per_sec_per_gpu": 35.39, "tokens/trainable": 161437} +{"epoch": 0.93359375, "grad_norm": 1.4966974258422852, "learning_rate": 6.379685834195036e-05, "loss": 0.20573341846466064, "memory/device_reserved (GiB)": 42.38, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.22843, "step": 239, "tokens/total": 8222496, "tokens/train_per_sec_per_gpu": 43.83, "tokens/trainable": 162106} +{"epoch": 0.9375, "grad_norm": 1.3397821187973022, "learning_rate": 6.351198709240186e-05, "loss": 0.18322136998176575, "memory/device_reserved (GiB)": 42.38, "memory/max_active (GiB)": 39.08, "memory/max_allocated (GiB)": 39.08, "ppl": 1.20108, "step": 240, "tokens/total": 8260864, "tokens/train_per_sec_per_gpu": 32.67, "tokens/trainable": 162732} +{"epoch": 0.94140625, "grad_norm": 10.612085342407227, "learning_rate": 6.32267616243259e-05, "loss": 0.18750113248825073, "memory/device_reserved (GiB)": 42.38, "memory/max_active (GiB)": 38.81, "memory/max_allocated (GiB)": 38.81, "ppl": 1.20623, "step": 241, "tokens/total": 8294848, "tokens/train_per_sec_per_gpu": 40.37, "tokens/trainable": 163428} +{"epoch": 0.9453125, "grad_norm": 3.2773144245147705, "learning_rate": 6.294119380711849e-05, "loss": 0.2347959727048874, "memory/device_reserved (GiB)": 42.38, "memory/max_active (GiB)": 37.09, "memory/max_allocated (GiB)": 37.09, "ppl": 1.26465, "step": 242, "tokens/total": 8328864, "tokens/train_per_sec_per_gpu": 38.35, "tokens/trainable": 164078} +{"epoch": 0.94921875, "grad_norm": 2.5382156372070312, "learning_rate": 6.265529552442209e-05, "loss": 0.1693935990333557, "memory/device_reserved (GiB)": 42.38, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.18459, "step": 243, "tokens/total": 8365344, "tokens/train_per_sec_per_gpu": 32.85, "tokens/trainable": 164682} +{"epoch": 0.953125, "grad_norm": 2.141463279724121, "learning_rate": 6.236907867363127e-05, "loss": 0.1552271544933319, "memory/device_reserved (GiB)": 42.38, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 1.16792, "step": 244, "tokens/total": 8397536, "tokens/train_per_sec_per_gpu": 40.78, "tokens/trainable": 165338} +{"epoch": 0.95703125, "grad_norm": 1.830223798751831, "learning_rate": 6.208255516539749e-05, "loss": 0.15452656149864197, "memory/device_reserved (GiB)": 42.38, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 1.16711, "step": 245, "tokens/total": 8431616, "tokens/train_per_sec_per_gpu": 40.96, "tokens/trainable": 166047} +{"epoch": 0.9609375, "grad_norm": 1.9523348808288574, "learning_rate": 6.179573692313344e-05, "loss": 0.17606420814990997, "memory/device_reserved (GiB)": 42.38, "memory/max_active (GiB)": 36.93, "memory/max_allocated (GiB)": 36.93, "ppl": 1.19251, "step": 246, "tokens/total": 8463072, "tokens/train_per_sec_per_gpu": 43.64, "tokens/trainable": 166681} +{"epoch": 0.96484375, "grad_norm": 6.1948347091674805, "learning_rate": 6.150863588251694e-05, "loss": 0.210883229970932, "memory/device_reserved (GiB)": 42.38, "memory/max_active (GiB)": 36.42, "memory/max_allocated (GiB)": 36.42, "ppl": 1.23477, "step": 247, "tokens/total": 8494480, "tokens/train_per_sec_per_gpu": 41.56, "tokens/trainable": 167316} +{"epoch": 0.96875, "grad_norm": 1.754889965057373, "learning_rate": 6.122126399099419e-05, "loss": 0.19998100399971008, "memory/device_reserved (GiB)": 42.38, "memory/max_active (GiB)": 37.97, "memory/max_allocated (GiB)": 37.97, "ppl": 1.22138, "step": 248, "tokens/total": 8529008, "tokens/train_per_sec_per_gpu": 37.37, "tokens/trainable": 167977} +{"epoch": 0.97265625, "grad_norm": 1.8525280952453613, "learning_rate": 6.0933633207282615e-05, "loss": 0.17030391097068787, "memory/device_reserved (GiB)": 42.38, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.18567, "step": 249, "tokens/total": 8564592, "tokens/train_per_sec_per_gpu": 37.85, "tokens/trainable": 168636} +{"epoch": 0.9765625, "grad_norm": 2.5106539726257324, "learning_rate": 6.064575550087316e-05, "loss": 0.21791565418243408, "memory/device_reserved (GiB)": 42.38, "memory/max_active (GiB)": 36.61, "memory/max_allocated (GiB)": 36.61, "ppl": 1.24348, "step": 250, "tokens/total": 8596592, "tokens/train_per_sec_per_gpu": 45.8, "tokens/trainable": 169330} +{"epoch": 0.98046875, "grad_norm": 4.243937969207764, "learning_rate": 6.0357642851532245e-05, "loss": 0.27084100246429443, "memory/device_reserved (GiB)": 42.38, "memory/max_active (GiB)": 35.98, "memory/max_allocated (GiB)": 35.98, "ppl": 1.31107, "step": 251, "tokens/total": 8626800, "tokens/train_per_sec_per_gpu": 43.6, "tokens/trainable": 169950} +{"epoch": 0.984375, "grad_norm": 2.205608367919922, "learning_rate": 6.0069307248803294e-05, "loss": 0.2080669105052948, "memory/device_reserved (GiB)": 42.38, "memory/max_active (GiB)": 38.91, "memory/max_allocated (GiB)": 38.91, "ppl": 1.2313, "step": 252, "tokens/total": 8662288, "tokens/train_per_sec_per_gpu": 30.92, "tokens/trainable": 170523} +{"epoch": 0.98828125, "grad_norm": 2.1529040336608887, "learning_rate": 5.9780760691507635e-05, "loss": 0.2013837695121765, "memory/device_reserved (GiB)": 42.38, "memory/max_active (GiB)": 36.4, "memory/max_allocated (GiB)": 36.4, "ppl": 1.22309, "step": 253, "tokens/total": 8692688, "tokens/train_per_sec_per_gpu": 39.52, "tokens/trainable": 171099} +{"epoch": 0.9921875, "grad_norm": 1.36692476272583, "learning_rate": 5.9492015187245334e-05, "loss": 0.15678878128528595, "memory/device_reserved (GiB)": 42.38, "memory/max_active (GiB)": 39.05, "memory/max_allocated (GiB)": 39.05, "ppl": 1.16975, "step": 254, "tokens/total": 8729760, "tokens/train_per_sec_per_gpu": 34.17, "tokens/trainable": 171792} +{"epoch": 0.99609375, "grad_norm": 2.1742091178894043, "learning_rate": 5.920308275189541e-05, "loss": 0.16899380087852478, "memory/device_reserved (GiB)": 42.38, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 1.18411, "step": 255, "tokens/total": 8764944, "tokens/train_per_sec_per_gpu": 38.01, "tokens/trainable": 172529} +{"epoch": 1.0, "grad_norm": 2.2090275287628174, "learning_rate": 5.8913975409115874e-05, "loss": 0.15734785795211792, "memory/device_reserved (GiB)": 42.38, "memory/max_active (GiB)": 37.37, "memory/max_allocated (GiB)": 37.37, "ppl": 1.1704, "step": 256, "tokens/total": 8799424, "tokens/train_per_sec_per_gpu": 39.31, "tokens/trainable": 173232} +{"epoch": 1.00390625, "grad_norm": 0.8077219128608704, "learning_rate": 5.8624705189843395e-05, "loss": 0.16409832239151, "memory/device_reserved (GiB)": 42.38, "memory/max_active (GiB)": 36.82, "memory/max_allocated (GiB)": 36.82, "ppl": 1.17833, "step": 257, "tokens/total": 8831584, "tokens/train_per_sec_per_gpu": 40.25, "tokens/trainable": 173866} +{"epoch": 1.0078125, "grad_norm": 1.2956899404525757, "learning_rate": 5.833528413179249e-05, "loss": 0.16324245929718018, "memory/device_reserved (GiB)": 39.66, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.17732, "step": 258, "tokens/total": 8867184, "tokens/train_per_sec_per_gpu": 34.23, "tokens/trainable": 174549} +{"epoch": 1.01171875, "grad_norm": 3.502631902694702, "learning_rate": 5.80457242789548e-05, "loss": 0.16318181157112122, "memory/device_reserved (GiB)": 39.66, "memory/max_active (GiB)": 36.34, "memory/max_allocated (GiB)": 36.34, "ppl": 1.17725, "step": 259, "tokens/total": 8898800, "tokens/train_per_sec_per_gpu": 40.1, "tokens/trainable": 175214} +{"epoch": 1.015625, "grad_norm": 2.918637752532959, "learning_rate": 5.77560376810977e-05, "loss": 0.16906464099884033, "memory/device_reserved (GiB)": 39.66, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.1842, "step": 260, "tokens/total": 8931792, "tokens/train_per_sec_per_gpu": 42.06, "tokens/trainable": 175877} +{"epoch": 1.01953125, "grad_norm": 1.176540493965149, "learning_rate": 5.7466236393263005e-05, "loss": 0.14418289065361023, "memory/device_reserved (GiB)": 39.66, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.1551, "step": 261, "tokens/total": 8966160, "tokens/train_per_sec_per_gpu": 34.77, "tokens/trainable": 176543} +{"epoch": 1.0234375, "grad_norm": 2.804931402206421, "learning_rate": 5.717633247526522e-05, "loss": 0.18554365634918213, "memory/device_reserved (GiB)": 41.6, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.20387, "step": 262, "tokens/total": 9005728, "tokens/train_per_sec_per_gpu": 34.25, "tokens/trainable": 177267} +{"epoch": 1.02734375, "grad_norm": 1.3721890449523926, "learning_rate": 5.688633799118971e-05, "loss": 0.15216746926307678, "memory/device_reserved (GiB)": 41.6, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.16436, "step": 263, "tokens/total": 9038592, "tokens/train_per_sec_per_gpu": 41.68, "tokens/trainable": 177945} +{"epoch": 1.03125, "grad_norm": 2.0068790912628174, "learning_rate": 5.659626500889066e-05, "loss": 0.1700342893600464, "memory/device_reserved (GiB)": 41.6, "memory/max_active (GiB)": 36.87, "memory/max_allocated (GiB)": 36.87, "ppl": 1.18535, "step": 264, "tokens/total": 9070816, "tokens/train_per_sec_per_gpu": 41.5, "tokens/trainable": 178625} +{"epoch": 1.03515625, "grad_norm": 1.020701289176941, "learning_rate": 5.6306125599488905e-05, "loss": 0.1713760495185852, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 35.69, "memory/max_allocated (GiB)": 35.69, "ppl": 1.18694, "step": 265, "tokens/total": 9101232, "tokens/train_per_sec_per_gpu": 41.17, "tokens/trainable": 179236} +{"epoch": 1.0390625, "grad_norm": 2.1045022010803223, "learning_rate": 5.601593183686955e-05, "loss": 0.15327207744121552, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.16564, "step": 266, "tokens/total": 9136464, "tokens/train_per_sec_per_gpu": 38.99, "tokens/trainable": 179911} +{"epoch": 1.04296875, "grad_norm": 2.629328489303589, "learning_rate": 5.572569579717961e-05, "loss": 0.1915554702281952, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.21113, "step": 267, "tokens/total": 9171248, "tokens/train_per_sec_per_gpu": 34.37, "tokens/trainable": 180504} +{"epoch": 1.046875, "grad_norm": 1.4249154329299927, "learning_rate": 5.543542955832538e-05, "loss": 0.15762758255004883, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 38.22, "memory/max_allocated (GiB)": 38.22, "ppl": 1.17073, "step": 268, "tokens/total": 9205680, "tokens/train_per_sec_per_gpu": 38.9, "tokens/trainable": 181154} +{"epoch": 1.05078125, "grad_norm": 1.0896453857421875, "learning_rate": 5.514514519946986e-05, "loss": 0.15331509709358215, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 37.39, "memory/max_allocated (GiB)": 37.39, "ppl": 1.16569, "step": 269, "tokens/total": 9237600, "tokens/train_per_sec_per_gpu": 41.09, "tokens/trainable": 181785} +{"epoch": 1.0546875, "grad_norm": 1.8712314367294312, "learning_rate": 5.485485480053015e-05, "loss": 0.1316499263048172, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 37.97, "memory/max_allocated (GiB)": 37.97, "ppl": 1.14071, "step": 270, "tokens/total": 9272704, "tokens/train_per_sec_per_gpu": 39.31, "tokens/trainable": 182502} +{"epoch": 1.05859375, "grad_norm": 1.4713318347930908, "learning_rate": 5.4564570441674645e-05, "loss": 0.1523110270500183, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 36.85, "memory/max_allocated (GiB)": 36.85, "ppl": 1.16452, "step": 271, "tokens/total": 9305936, "tokens/train_per_sec_per_gpu": 46.68, "tokens/trainable": 183222} +{"epoch": 1.0625, "grad_norm": 1.6282330751419067, "learning_rate": 5.42743042028204e-05, "loss": 0.13556751608848572, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.14519, "step": 272, "tokens/total": 9340688, "tokens/train_per_sec_per_gpu": 47.98, "tokens/trainable": 183983} +{"epoch": 1.06640625, "grad_norm": 1.431442379951477, "learning_rate": 5.3984068163130464e-05, "loss": 0.15406368672847748, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.16657, "step": 273, "tokens/total": 9374016, "tokens/train_per_sec_per_gpu": 35.51, "tokens/trainable": 184647} +{"epoch": 1.0703125, "grad_norm": 3.241112232208252, "learning_rate": 5.369387440051111e-05, "loss": 0.19960616528987885, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.22092, "step": 274, "tokens/total": 9410416, "tokens/train_per_sec_per_gpu": 37.52, "tokens/trainable": 185333} +{"epoch": 1.07421875, "grad_norm": 1.6863682270050049, "learning_rate": 5.340373499110935e-05, "loss": 0.1389947533607483, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.14912, "step": 275, "tokens/total": 9449696, "tokens/train_per_sec_per_gpu": 33.51, "tokens/trainable": 186017} +{"epoch": 1.078125, "grad_norm": 1.3670021295547485, "learning_rate": 5.3113662008810304e-05, "loss": 0.16405822336673737, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.17828, "step": 276, "tokens/total": 9485520, "tokens/train_per_sec_per_gpu": 34.6, "tokens/trainable": 186694} +{"epoch": 1.08203125, "grad_norm": 1.2078057527542114, "learning_rate": 5.282366752473479e-05, "loss": 0.15275359153747559, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.16504, "step": 277, "tokens/total": 9521408, "tokens/train_per_sec_per_gpu": 39.31, "tokens/trainable": 187339} +{"epoch": 1.0859375, "grad_norm": 6.741434574127197, "learning_rate": 5.2533763606737005e-05, "loss": 0.1445685774087906, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.15554, "step": 278, "tokens/total": 9553600, "tokens/train_per_sec_per_gpu": 41.1, "tokens/trainable": 188035} +{"epoch": 1.08984375, "grad_norm": 1.818660020828247, "learning_rate": 5.224396231890232e-05, "loss": 0.19808191061019897, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.21906, "step": 279, "tokens/total": 9586000, "tokens/train_per_sec_per_gpu": 38.25, "tokens/trainable": 188664} +{"epoch": 1.09375, "grad_norm": 2.5582101345062256, "learning_rate": 5.195427572104522e-05, "loss": 0.17206960916519165, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.18776, "step": 280, "tokens/total": 9623088, "tokens/train_per_sec_per_gpu": 36.29, "tokens/trainable": 189332} +{"epoch": 1.09765625, "grad_norm": 0.8199899792671204, "learning_rate": 5.166471586820751e-05, "loss": 0.13490115106105804, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 37.54, "memory/max_allocated (GiB)": 37.54, "ppl": 1.14442, "step": 281, "tokens/total": 9658096, "tokens/train_per_sec_per_gpu": 38.39, "tokens/trainable": 190046} +{"epoch": 1.1015625, "grad_norm": 0.6980924606323242, "learning_rate": 5.1375294810156615e-05, "loss": 0.13305599987506866, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 38.26, "memory/max_allocated (GiB)": 38.26, "ppl": 1.14231, "step": 282, "tokens/total": 9692640, "tokens/train_per_sec_per_gpu": 38.03, "tokens/trainable": 190742} +{"epoch": 1.10546875, "grad_norm": 1.5221401453018188, "learning_rate": 5.1086024590884144e-05, "loss": 0.1549052596092224, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 37.09, "memory/max_allocated (GiB)": 37.09, "ppl": 1.16755, "step": 283, "tokens/total": 9726320, "tokens/train_per_sec_per_gpu": 41.77, "tokens/trainable": 191474} +{"epoch": 1.109375, "grad_norm": 2.3288402557373047, "learning_rate": 5.079691724810461e-05, "loss": 0.13600924611091614, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.14569, "step": 284, "tokens/total": 9763616, "tokens/train_per_sec_per_gpu": 35.48, "tokens/trainable": 192197} +{"epoch": 1.11328125, "grad_norm": 1.4424097537994385, "learning_rate": 5.0507984812754684e-05, "loss": 0.1580045223236084, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.17117, "step": 285, "tokens/total": 9800384, "tokens/train_per_sec_per_gpu": 33.13, "tokens/trainable": 192810} +{"epoch": 1.1171875, "grad_norm": 3.3847498893737793, "learning_rate": 5.021923930849237e-05, "loss": 0.21249686181545258, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.23676, "step": 286, "tokens/total": 9834288, "tokens/train_per_sec_per_gpu": 34.48, "tokens/trainable": 193407} +{"epoch": 1.12109375, "grad_norm": 0.9076223969459534, "learning_rate": 4.99306927511967e-05, "loss": 0.13203230500221252, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.14115, "step": 287, "tokens/total": 9867312, "tokens/train_per_sec_per_gpu": 40.91, "tokens/trainable": 194137} +{"epoch": 1.125, "grad_norm": 1.3543277978897095, "learning_rate": 4.964235714846775e-05, "loss": 0.1582946926355362, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 38.16, "memory/max_allocated (GiB)": 38.16, "ppl": 1.17151, "step": 288, "tokens/total": 9902816, "tokens/train_per_sec_per_gpu": 33.97, "tokens/trainable": 194734} +{"epoch": 1.12890625, "grad_norm": 2.857422351837158, "learning_rate": 4.9354244499126866e-05, "loss": 0.1605105698108673, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 37.59, "memory/max_allocated (GiB)": 37.59, "ppl": 1.17411, "step": 289, "tokens/total": 9937520, "tokens/train_per_sec_per_gpu": 36.46, "tokens/trainable": 195435} +{"epoch": 1.1328125, "grad_norm": 1.1375057697296143, "learning_rate": 4.90663667927174e-05, "loss": 0.14715363085269928, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.15853, "step": 290, "tokens/total": 9972288, "tokens/train_per_sec_per_gpu": 40.86, "tokens/trainable": 196137} +{"epoch": 1.13671875, "grad_norm": 1.9709792137145996, "learning_rate": 4.877873600900581e-05, "loss": 0.13737550377845764, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 39.15, "memory/max_allocated (GiB)": 39.15, "ppl": 1.14726, "step": 291, "tokens/total": 10010544, "tokens/train_per_sec_per_gpu": 39.67, "tokens/trainable": 196915} +{"epoch": 1.140625, "grad_norm": 1.9002777338027954, "learning_rate": 4.849136411748306e-05, "loss": 0.17788103222846985, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.19468, "step": 292, "tokens/total": 10042160, "tokens/train_per_sec_per_gpu": 35.19, "tokens/trainable": 197542} +{"epoch": 1.14453125, "grad_norm": 2.4559056758880615, "learning_rate": 4.8204263076866574e-05, "loss": 0.17734910547733307, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 38.39, "memory/max_allocated (GiB)": 38.39, "ppl": 1.19405, "step": 293, "tokens/total": 10078768, "tokens/train_per_sec_per_gpu": 35.53, "tokens/trainable": 198249} +{"epoch": 1.1484375, "grad_norm": 1.4715757369995117, "learning_rate": 4.791744483460251e-05, "loss": 0.13948370516300201, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.14968, "step": 294, "tokens/total": 10114048, "tokens/train_per_sec_per_gpu": 51.33, "tokens/trainable": 199001} +{"epoch": 1.15234375, "grad_norm": 1.619346022605896, "learning_rate": 4.7630921326368736e-05, "loss": 0.17015378177165985, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 38.7, "memory/max_allocated (GiB)": 38.7, "ppl": 1.18549, "step": 295, "tokens/total": 10151936, "tokens/train_per_sec_per_gpu": 30.54, "tokens/trainable": 199670} +{"epoch": 1.15625, "grad_norm": 1.6719611883163452, "learning_rate": 4.7344704475577916e-05, "loss": 0.17058244347572327, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 35.33, "memory/max_allocated (GiB)": 35.33, "ppl": 1.186, "step": 296, "tokens/total": 10180752, "tokens/train_per_sec_per_gpu": 42.69, "tokens/trainable": 200299} +{"epoch": 1.16015625, "grad_norm": 1.9033387899398804, "learning_rate": 4.705880619288153e-05, "loss": 0.16874870657920837, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 1.18382, "step": 297, "tokens/total": 10216976, "tokens/train_per_sec_per_gpu": 38.06, "tokens/trainable": 200946} +{"epoch": 1.1640625, "grad_norm": 1.3599543571472168, "learning_rate": 4.677323837567412e-05, "loss": 0.15458816289901733, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.16718, "step": 298, "tokens/total": 10254960, "tokens/train_per_sec_per_gpu": 33.81, "tokens/trainable": 201573} +{"epoch": 1.16796875, "grad_norm": 2.027372121810913, "learning_rate": 4.6488012907598146e-05, "loss": 0.16152316331863403, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.1753, "step": 299, "tokens/total": 10291104, "tokens/train_per_sec_per_gpu": 34.52, "tokens/trainable": 202214} +{"epoch": 1.171875, "grad_norm": 2.7089638710021973, "learning_rate": 4.620314165804964e-05, "loss": 0.16976267099380493, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.18502, "step": 300, "tokens/total": 10328112, "tokens/train_per_sec_per_gpu": 37.58, "tokens/trainable": 202910} +{"epoch": 1.17578125, "grad_norm": 0.9096250534057617, "learning_rate": 4.591863648168407e-05, "loss": 0.15263134241104126, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 36.74, "memory/max_allocated (GiB)": 36.74, "ppl": 1.1649, "step": 301, "tokens/total": 10361392, "tokens/train_per_sec_per_gpu": 42.9, "tokens/trainable": 203553} +{"epoch": 1.1796875, "grad_norm": 1.204545259475708, "learning_rate": 4.5634509217923135e-05, "loss": 0.1289079487323761, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 37.4, "memory/max_allocated (GiB)": 37.4, "ppl": 1.13759, "step": 302, "tokens/total": 10393584, "tokens/train_per_sec_per_gpu": 49.44, "tokens/trainable": 204275} +{"epoch": 1.18359375, "grad_norm": 2.7650322914123535, "learning_rate": 4.535077169046201e-05, "loss": 0.13369400799274445, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.14304, "step": 303, "tokens/total": 10428112, "tokens/train_per_sec_per_gpu": 45.07, "tokens/trainable": 205026} +{"epoch": 1.1875, "grad_norm": 1.9695005416870117, "learning_rate": 4.506743570677743e-05, "loss": 0.1427815556526184, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.15348, "step": 304, "tokens/total": 10462896, "tokens/train_per_sec_per_gpu": 35.92, "tokens/trainable": 205779} +{"epoch": 1.19140625, "grad_norm": 1.4379143714904785, "learning_rate": 4.478451305763618e-05, "loss": 0.15024545788764954, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.16212, "step": 305, "tokens/total": 10500112, "tokens/train_per_sec_per_gpu": 44.37, "tokens/trainable": 206541} +{"epoch": 1.1953125, "grad_norm": 2.41740083694458, "learning_rate": 4.450201551660454e-05, "loss": 0.16790993511676788, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.18283, "step": 306, "tokens/total": 10533088, "tokens/train_per_sec_per_gpu": 38.49, "tokens/trainable": 207211} +{"epoch": 1.19921875, "grad_norm": 1.3038525581359863, "learning_rate": 4.4219954839558276e-05, "loss": 0.15477105975151062, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 36.22, "memory/max_allocated (GiB)": 36.22, "ppl": 1.16739, "step": 307, "tokens/total": 10563776, "tokens/train_per_sec_per_gpu": 44.36, "tokens/trainable": 207896} +{"epoch": 1.203125, "grad_norm": 1.9267454147338867, "learning_rate": 4.393834276419352e-05, "loss": 0.15128058195114136, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.16332, "step": 308, "tokens/total": 10602816, "tokens/train_per_sec_per_gpu": 36.34, "tokens/trainable": 208626} +{"epoch": 1.20703125, "grad_norm": 1.6173962354660034, "learning_rate": 4.36571910095382e-05, "loss": 0.1641201227903366, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 39.09, "memory/max_allocated (GiB)": 39.09, "ppl": 1.17836, "step": 309, "tokens/total": 10637120, "tokens/train_per_sec_per_gpu": 40.15, "tokens/trainable": 209277} +{"epoch": 1.2109375, "grad_norm": 3.110611915588379, "learning_rate": 4.337651127546448e-05, "loss": 0.17642472684383392, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 37.45, "memory/max_allocated (GiB)": 37.45, "ppl": 1.19294, "step": 310, "tokens/total": 10670224, "tokens/train_per_sec_per_gpu": 37.16, "tokens/trainable": 209861} +{"epoch": 1.21484375, "grad_norm": 1.0934444665908813, "learning_rate": 4.3096315242201736e-05, "loss": 0.1402176469564438, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 36.66, "memory/max_allocated (GiB)": 36.66, "ppl": 1.15052, "step": 311, "tokens/total": 10702384, "tokens/train_per_sec_per_gpu": 45.01, "tokens/trainable": 210537} +{"epoch": 1.21875, "grad_norm": 1.959404706954956, "learning_rate": 4.2816614569850635e-05, "loss": 0.18142759799957275, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 35.82, "memory/max_allocated (GiB)": 35.82, "ppl": 1.19893, "step": 312, "tokens/total": 10732000, "tokens/train_per_sec_per_gpu": 35.03, "tokens/trainable": 211118} +{"epoch": 1.22265625, "grad_norm": 2.4058523178100586, "learning_rate": 4.2537420897897864e-05, "loss": 0.135107159614563, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 39.28, "memory/max_allocated (GiB)": 39.28, "ppl": 1.14466, "step": 313, "tokens/total": 10772032, "tokens/train_per_sec_per_gpu": 33.19, "tokens/trainable": 211845} +{"epoch": 1.2265625, "grad_norm": 0.9878942966461182, "learning_rate": 4.225874584473174e-05, "loss": 0.15000802278518677, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.16184, "step": 314, "tokens/total": 10804304, "tokens/train_per_sec_per_gpu": 34.16, "tokens/trainable": 212458} +{"epoch": 1.23046875, "grad_norm": 2.150606870651245, "learning_rate": 4.19806010071587e-05, "loss": 0.13586044311523438, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.14552, "step": 315, "tokens/total": 10841728, "tokens/train_per_sec_per_gpu": 37.22, "tokens/trainable": 213230} +{"epoch": 1.234375, "grad_norm": 1.921015739440918, "learning_rate": 4.170299795992081e-05, "loss": 0.16001778841018677, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.17353, "step": 316, "tokens/total": 10877360, "tokens/train_per_sec_per_gpu": 39.34, "tokens/trainable": 213902} +{"epoch": 1.23828125, "grad_norm": 1.0293675661087036, "learning_rate": 4.142594825521398e-05, "loss": 0.13371029496192932, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.14306, "step": 317, "tokens/total": 10915216, "tokens/train_per_sec_per_gpu": 37.67, "tokens/trainable": 214631} +{"epoch": 1.2421875, "grad_norm": 0.9219181537628174, "learning_rate": 4.114946342220728e-05, "loss": 0.13583800196647644, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.1455, "step": 318, "tokens/total": 10950064, "tokens/train_per_sec_per_gpu": 41.62, "tokens/trainable": 215330} +{"epoch": 1.24609375, "grad_norm": 1.4141285419464111, "learning_rate": 4.087355496656321e-05, "loss": 0.1394645869731903, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.14966, "step": 319, "tokens/total": 10983952, "tokens/train_per_sec_per_gpu": 39.26, "tokens/trainable": 216020} +{"epoch": 1.25, "grad_norm": 1.4632164239883423, "learning_rate": 4.05982343699588e-05, "loss": 0.1388874500989914, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.14899, "step": 320, "tokens/total": 11016496, "tokens/train_per_sec_per_gpu": 46.94, "tokens/trainable": 216747} +{"epoch": 1.25390625, "grad_norm": 1.6474167108535767, "learning_rate": 4.0323513089607876e-05, "loss": 0.1593591570854187, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.17276, "step": 321, "tokens/total": 11051744, "tokens/train_per_sec_per_gpu": 38.57, "tokens/trainable": 217457} +{"epoch": 1.2578125, "grad_norm": 1.8080885410308838, "learning_rate": 4.004940255778431e-05, "loss": 0.1412440538406372, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 36.21, "memory/max_allocated (GiB)": 36.21, "ppl": 1.15171, "step": 322, "tokens/total": 11083088, "tokens/train_per_sec_per_gpu": 37.58, "tokens/trainable": 218105} +{"epoch": 1.26171875, "grad_norm": 1.9757006168365479, "learning_rate": 3.977591418134619e-05, "loss": 0.14919516444206238, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 36.42, "memory/max_allocated (GiB)": 36.42, "ppl": 1.1609, "step": 323, "tokens/total": 11115248, "tokens/train_per_sec_per_gpu": 38.67, "tokens/trainable": 218802} +{"epoch": 1.265625, "grad_norm": 2.0750527381896973, "learning_rate": 3.95030593412612e-05, "loss": 0.14356181025505066, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 39.14, "memory/max_allocated (GiB)": 39.14, "ppl": 1.15438, "step": 324, "tokens/total": 11154688, "tokens/train_per_sec_per_gpu": 31.02, "tokens/trainable": 219480} +{"epoch": 1.26953125, "grad_norm": 0.9046081304550171, "learning_rate": 3.923084939213296e-05, "loss": 0.1523588001728058, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.16458, "step": 325, "tokens/total": 11191856, "tokens/train_per_sec_per_gpu": 35.13, "tokens/trainable": 220141} +{"epoch": 1.2734375, "grad_norm": 5.3328537940979, "learning_rate": 3.895929566172861e-05, "loss": 0.1559934914112091, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 38.05, "memory/max_allocated (GiB)": 38.05, "ppl": 1.16882, "step": 326, "tokens/total": 11227888, "tokens/train_per_sec_per_gpu": 34.46, "tokens/trainable": 220827} +{"epoch": 1.27734375, "grad_norm": 1.7070293426513672, "learning_rate": 3.868840945050728e-05, "loss": 0.14762508869171143, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.15908, "step": 327, "tokens/total": 11265936, "tokens/train_per_sec_per_gpu": 34.58, "tokens/trainable": 221520} +{"epoch": 1.28125, "grad_norm": 1.2816635370254517, "learning_rate": 3.841820203114995e-05, "loss": 0.14465057849884033, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 37.73, "memory/max_allocated (GiB)": 37.73, "ppl": 1.15564, "step": 328, "tokens/total": 11300752, "tokens/train_per_sec_per_gpu": 42.11, "tokens/trainable": 222208} +{"epoch": 1.28515625, "grad_norm": 1.6471562385559082, "learning_rate": 3.814868464809027e-05, "loss": 0.13536101579666138, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.14495, "step": 329, "tokens/total": 11334544, "tokens/train_per_sec_per_gpu": 42.59, "tokens/trainable": 222934} +{"epoch": 1.2890625, "grad_norm": 3.1537184715270996, "learning_rate": 3.787986851704667e-05, "loss": 0.11977335810661316, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 38.8, "memory/max_allocated (GiB)": 38.8, "ppl": 1.12724, "step": 330, "tokens/total": 11371824, "tokens/train_per_sec_per_gpu": 42.65, "tokens/trainable": 223712} +{"epoch": 1.29296875, "grad_norm": 1.7894278764724731, "learning_rate": 3.7611764824555654e-05, "loss": 0.15111641585826874, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 38.18, "memory/max_allocated (GiB)": 38.18, "ppl": 1.16313, "step": 331, "tokens/total": 11404816, "tokens/train_per_sec_per_gpu": 44.45, "tokens/trainable": 224382} +{"epoch": 1.296875, "grad_norm": 2.551143169403076, "learning_rate": 3.734438472750619e-05, "loss": 0.15440747141838074, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 36.79, "memory/max_allocated (GiB)": 36.79, "ppl": 1.16697, "step": 332, "tokens/total": 11437696, "tokens/train_per_sec_per_gpu": 40.03, "tokens/trainable": 225046} +{"epoch": 1.30078125, "grad_norm": 1.8364887237548828, "learning_rate": 3.707773935267552e-05, "loss": 0.16325607895851135, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 38.91, "memory/max_allocated (GiB)": 38.91, "ppl": 1.17734, "step": 333, "tokens/total": 11474304, "tokens/train_per_sec_per_gpu": 34.37, "tokens/trainable": 225701} +{"epoch": 1.3046875, "grad_norm": 1.2289764881134033, "learning_rate": 3.68118397962661e-05, "loss": 0.16060401499271393, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.17422, "step": 334, "tokens/total": 11506800, "tokens/train_per_sec_per_gpu": 36.23, "tokens/trainable": 226317} +{"epoch": 1.30859375, "grad_norm": 1.8218135833740234, "learning_rate": 3.654669712344384e-05, "loss": 0.18106070160865784, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 36.31, "memory/max_allocated (GiB)": 36.31, "ppl": 1.19849, "step": 335, "tokens/total": 11538144, "tokens/train_per_sec_per_gpu": 45.31, "tokens/trainable": 227019} +{"epoch": 1.3125, "grad_norm": 2.8671486377716064, "learning_rate": 3.628232236787763e-05, "loss": 0.1632833182811737, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.17737, "step": 336, "tokens/total": 11573648, "tokens/train_per_sec_per_gpu": 40.77, "tokens/trainable": 227696} +{"epoch": 1.31640625, "grad_norm": 1.4488402605056763, "learning_rate": 3.6018726531280144e-05, "loss": 0.15071827173233032, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.16267, "step": 337, "tokens/total": 11607792, "tokens/train_per_sec_per_gpu": 38.7, "tokens/trainable": 228366} +{"epoch": 1.3203125, "grad_norm": 1.5597960948944092, "learning_rate": 3.575592058295017e-05, "loss": 0.15863028168678284, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.28, "memory/max_allocated (GiB)": 37.28, "ppl": 1.1719, "step": 338, "tokens/total": 11642512, "tokens/train_per_sec_per_gpu": 39.14, "tokens/trainable": 229101} +{"epoch": 1.32421875, "grad_norm": 0.8523046970367432, "learning_rate": 3.549391545931585e-05, "loss": 0.12431581318378448, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.13237, "step": 339, "tokens/total": 11680816, "tokens/train_per_sec_per_gpu": 37.6, "tokens/trainable": 229787} +{"epoch": 1.328125, "grad_norm": 0.7987473607063293, "learning_rate": 3.5232722063479914e-05, "loss": 0.13396799564361572, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.14336, "step": 340, "tokens/total": 11716320, "tokens/train_per_sec_per_gpu": 37.26, "tokens/trainable": 230452} +{"epoch": 1.33203125, "grad_norm": 2.2754952907562256, "learning_rate": 3.49723512647657e-05, "loss": 0.15400275588035583, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.73, "memory/max_allocated (GiB)": 37.73, "ppl": 1.16649, "step": 341, "tokens/total": 11748560, "tokens/train_per_sec_per_gpu": 42.53, "tokens/trainable": 231112} +{"epoch": 1.3359375, "grad_norm": 1.064715027809143, "learning_rate": 3.471281389826491e-05, "loss": 0.12823206186294556, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.81, "memory/max_allocated (GiB)": 38.81, "ppl": 1.13682, "step": 342, "tokens/total": 11784304, "tokens/train_per_sec_per_gpu": 42.66, "tokens/trainable": 231824} +{"epoch": 1.33984375, "grad_norm": 1.8029358386993408, "learning_rate": 3.4454120764386764e-05, "loss": 0.130979984998703, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.19, "memory/max_allocated (GiB)": 38.19, "ppl": 1.13994, "step": 343, "tokens/total": 11819824, "tokens/train_per_sec_per_gpu": 39.83, "tokens/trainable": 232499} +{"epoch": 1.34375, "grad_norm": 2.606563091278076, "learning_rate": 3.4196282628408526e-05, "loss": 0.12419167906045914, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.87, "memory/max_allocated (GiB)": 36.87, "ppl": 1.13223, "step": 344, "tokens/total": 11852944, "tokens/train_per_sec_per_gpu": 40.32, "tokens/trainable": 233244} +{"epoch": 1.34765625, "grad_norm": 1.0455570220947266, "learning_rate": 3.3939310220027456e-05, "loss": 0.15101388096809387, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.16301, "step": 345, "tokens/total": 11889792, "tokens/train_per_sec_per_gpu": 32.86, "tokens/trainable": 233863} +{"epoch": 1.3515625, "grad_norm": 0.9247775077819824, "learning_rate": 3.3683214232914404e-05, "loss": 0.14021149277687073, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 1.15052, "step": 346, "tokens/total": 11921296, "tokens/train_per_sec_per_gpu": 41.1, "tokens/trainable": 234545} +{"epoch": 1.35546875, "grad_norm": 3.7941155433654785, "learning_rate": 3.342800532426873e-05, "loss": 0.1910831183195114, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.21056, "step": 347, "tokens/total": 11955024, "tokens/train_per_sec_per_gpu": 37.26, "tokens/trainable": 235177} +{"epoch": 1.359375, "grad_norm": 1.349992036819458, "learning_rate": 3.317369411437484e-05, "loss": 0.16210371255874634, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.17598, "step": 348, "tokens/total": 11988560, "tokens/train_per_sec_per_gpu": 36.46, "tokens/trainable": 235836} +{"epoch": 1.36328125, "grad_norm": 2.7886300086975098, "learning_rate": 3.292029118616024e-05, "loss": 0.13954979181289673, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.14976, "step": 349, "tokens/total": 12021392, "tokens/train_per_sec_per_gpu": 37.75, "tokens/trainable": 236490} +{"epoch": 1.3671875, "grad_norm": 0.9890764951705933, "learning_rate": 3.266780708475511e-05, "loss": 0.12306170165538788, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.13095, "step": 350, "tokens/total": 12054784, "tokens/train_per_sec_per_gpu": 43.3, "tokens/trainable": 237224} +{"epoch": 1.37109375, "grad_norm": 1.4455267190933228, "learning_rate": 3.241625231705354e-05, "loss": 0.13145488500595093, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.14049, "step": 351, "tokens/total": 12087168, "tokens/train_per_sec_per_gpu": 44.52, "tokens/trainable": 237888} +{"epoch": 1.375, "grad_norm": 1.7848262786865234, "learning_rate": 3.216563735127618e-05, "loss": 0.11490517854690552, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.12177, "step": 352, "tokens/total": 12124528, "tokens/train_per_sec_per_gpu": 37.22, "tokens/trainable": 238650} +{"epoch": 1.37890625, "grad_norm": 1.059736728668213, "learning_rate": 3.191597261653475e-05, "loss": 0.13748683035373688, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.23, "memory/max_allocated (GiB)": 37.23, "ppl": 1.14739, "step": 353, "tokens/total": 12156992, "tokens/train_per_sec_per_gpu": 40.08, "tokens/trainable": 239351} +{"epoch": 1.3828125, "grad_norm": 11.985274314880371, "learning_rate": 3.166726850239794e-05, "loss": 0.19965684413909912, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.43, "memory/max_allocated (GiB)": 37.43, "ppl": 1.22098, "step": 354, "tokens/total": 12189472, "tokens/train_per_sec_per_gpu": 48.12, "tokens/trainable": 240032} +{"epoch": 1.38671875, "grad_norm": 0.7335708737373352, "learning_rate": 3.141953535845912e-05, "loss": 0.12273161113262177, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.13058, "step": 355, "tokens/total": 12224448, "tokens/train_per_sec_per_gpu": 38.93, "tokens/trainable": 240715} +{"epoch": 1.390625, "grad_norm": 1.2115824222564697, "learning_rate": 3.11727834939056e-05, "loss": 0.14476023614406586, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.61, "memory/max_allocated (GiB)": 37.61, "ppl": 1.15576, "step": 356, "tokens/total": 12257792, "tokens/train_per_sec_per_gpu": 38.9, "tokens/trainable": 241429} +{"epoch": 1.39453125, "grad_norm": 1.3145387172698975, "learning_rate": 3.092702317708967e-05, "loss": 0.14113546907901764, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.15158, "step": 357, "tokens/total": 12292336, "tokens/train_per_sec_per_gpu": 37.53, "tokens/trainable": 242066} +{"epoch": 1.3984375, "grad_norm": 1.0931607484817505, "learning_rate": 3.0682264635101276e-05, "loss": 0.15154790878295898, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.16363, "step": 358, "tokens/total": 12326272, "tokens/train_per_sec_per_gpu": 36.39, "tokens/trainable": 242716} +{"epoch": 1.40234375, "grad_norm": 0.6158861517906189, "learning_rate": 3.0438518053342407e-05, "loss": 0.12676642835140228, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.22, "memory/max_allocated (GiB)": 38.22, "ppl": 1.13515, "step": 359, "tokens/total": 12358512, "tokens/train_per_sec_per_gpu": 36.15, "tokens/trainable": 243436} +{"epoch": 1.40625, "grad_norm": 1.1437163352966309, "learning_rate": 3.0195793575103266e-05, "loss": 0.16074174642562866, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 35.83, "memory/max_allocated (GiB)": 35.83, "ppl": 1.17438, "step": 360, "tokens/total": 12388208, "tokens/train_per_sec_per_gpu": 44.83, "tokens/trainable": 244085} +{"epoch": 1.41015625, "grad_norm": 1.385132908821106, "learning_rate": 2.9954101301140146e-05, "loss": 0.1580628752708435, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.17124, "step": 361, "tokens/total": 12427088, "tokens/train_per_sec_per_gpu": 35.23, "tokens/trainable": 244754} +{"epoch": 1.4140625, "grad_norm": 0.6955557465553284, "learning_rate": 2.9713451289255123e-05, "loss": 0.11573255062103271, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.1227, "step": 362, "tokens/total": 12459040, "tokens/train_per_sec_per_gpu": 44.98, "tokens/trainable": 245462} +{"epoch": 1.41796875, "grad_norm": 1.3214768171310425, "learning_rate": 2.9473853553877484e-05, "loss": 0.1482212245464325, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.15, "memory/max_allocated (GiB)": 38.15, "ppl": 1.15977, "step": 363, "tokens/total": 12493792, "tokens/train_per_sec_per_gpu": 43.03, "tokens/trainable": 246173} +{"epoch": 1.421875, "grad_norm": 2.2760097980499268, "learning_rate": 2.9235318065647e-05, "loss": 0.14004261791706085, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.39, "memory/max_allocated (GiB)": 38.39, "ppl": 1.15032, "step": 364, "tokens/total": 12528384, "tokens/train_per_sec_per_gpu": 34.46, "tokens/trainable": 246785} +{"epoch": 1.42578125, "grad_norm": 1.447422742843628, "learning_rate": 2.8997854750998964e-05, "loss": 0.1440938413143158, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.15499, "step": 365, "tokens/total": 12564720, "tokens/train_per_sec_per_gpu": 38.27, "tokens/trainable": 247479} +{"epoch": 1.4296875, "grad_norm": 1.9124428033828735, "learning_rate": 2.8761473491751258e-05, "loss": 0.16258814930915833, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.53, "memory/max_allocated (GiB)": 36.53, "ppl": 1.17655, "step": 366, "tokens/total": 12596640, "tokens/train_per_sec_per_gpu": 37.75, "tokens/trainable": 248090} +{"epoch": 1.43359375, "grad_norm": 1.6422768831253052, "learning_rate": 2.8526184124692883e-05, "loss": 0.13661152124404907, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 1.14638, "step": 367, "tokens/total": 12629328, "tokens/train_per_sec_per_gpu": 39.88, "tokens/trainable": 248781} +{"epoch": 1.4375, "grad_norm": 1.0924557447433472, "learning_rate": 2.829199644117484e-05, "loss": 0.1302151083946228, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.13907, "step": 368, "tokens/total": 12664976, "tokens/train_per_sec_per_gpu": 35.38, "tokens/trainable": 249433} +{"epoch": 1.44140625, "grad_norm": 1.1287447214126587, "learning_rate": 2.8058920186702553e-05, "loss": 0.1306305229663849, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.52, "memory/max_allocated (GiB)": 37.52, "ppl": 1.13955, "step": 369, "tokens/total": 12699856, "tokens/train_per_sec_per_gpu": 37.25, "tokens/trainable": 250091} +{"epoch": 1.4453125, "grad_norm": 0.6992092728614807, "learning_rate": 2.782696506053033e-05, "loss": 0.13599637150764465, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.14568, "step": 370, "tokens/total": 12736064, "tokens/train_per_sec_per_gpu": 35.65, "tokens/trainable": 250735} +{"epoch": 1.44921875, "grad_norm": 0.8808194398880005, "learning_rate": 2.7596140715257824e-05, "loss": 0.11527746915817261, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.08, "memory/max_allocated (GiB)": 39.08, "ppl": 1.12218, "step": 371, "tokens/total": 12772016, "tokens/train_per_sec_per_gpu": 37.09, "tokens/trainable": 251408} +{"epoch": 1.453125, "grad_norm": 2.539365530014038, "learning_rate": 2.7366456756428184e-05, "loss": 0.1287074089050293, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.13736, "step": 372, "tokens/total": 12806592, "tokens/train_per_sec_per_gpu": 39.77, "tokens/trainable": 252112} +{"epoch": 1.45703125, "grad_norm": 0.8538613319396973, "learning_rate": 2.7137922742128486e-05, "loss": 0.136698916554451, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.14648, "step": 373, "tokens/total": 12838672, "tokens/train_per_sec_per_gpu": 41.45, "tokens/trainable": 252773} +{"epoch": 1.4609375, "grad_norm": 1.5971136093139648, "learning_rate": 2.691054818259188e-05, "loss": 0.15710502862930298, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.17012, "step": 374, "tokens/total": 12874256, "tokens/train_per_sec_per_gpu": 36.8, "tokens/trainable": 253436} +{"epoch": 1.46484375, "grad_norm": 1.5257023572921753, "learning_rate": 2.6684342539801933e-05, "loss": 0.14835578203201294, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.15993, "step": 375, "tokens/total": 12911376, "tokens/train_per_sec_per_gpu": 39.78, "tokens/trainable": 254159} +{"epoch": 1.46875, "grad_norm": 1.2033662796020508, "learning_rate": 2.645931522709877e-05, "loss": 0.1205914169549942, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.12816, "step": 376, "tokens/total": 12949920, "tokens/train_per_sec_per_gpu": 37.44, "tokens/trainable": 254853} +{"epoch": 1.47265625, "grad_norm": 0.6913050413131714, "learning_rate": 2.6235475608787365e-05, "loss": 0.1289786845445633, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.67, "memory/max_allocated (GiB)": 37.67, "ppl": 1.13767, "step": 377, "tokens/total": 12985136, "tokens/train_per_sec_per_gpu": 41.16, "tokens/trainable": 255626} +{"epoch": 1.4765625, "grad_norm": 0.8435487151145935, "learning_rate": 2.6012832999747916e-05, "loss": 0.13434995710849762, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.14379, "step": 378, "tokens/total": 13022000, "tokens/train_per_sec_per_gpu": 38.75, "tokens/trainable": 256335} +{"epoch": 1.48046875, "grad_norm": 1.1495658159255981, "learning_rate": 2.579139666504821e-05, "loss": 0.14875373244285583, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.61, "memory/max_allocated (GiB)": 37.61, "ppl": 1.16039, "step": 379, "tokens/total": 13054112, "tokens/train_per_sec_per_gpu": 36.76, "tokens/trainable": 256950} +{"epoch": 1.484375, "grad_norm": 0.6000310778617859, "learning_rate": 2.557117581955798e-05, "loss": 0.10781712085008621, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.0, "memory/max_allocated (GiB)": 38.0, "ppl": 1.11384, "step": 380, "tokens/total": 13089072, "tokens/train_per_sec_per_gpu": 35.52, "tokens/trainable": 257713} +{"epoch": 1.48828125, "grad_norm": 1.2119078636169434, "learning_rate": 2.5352179627565532e-05, "loss": 0.13134637475013733, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.14036, "step": 381, "tokens/total": 13128096, "tokens/train_per_sec_per_gpu": 35.8, "tokens/trainable": 258428} +{"epoch": 1.4921875, "grad_norm": 0.9534374475479126, "learning_rate": 2.5134417202396277e-05, "loss": 0.11650234460830688, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.12356, "step": 382, "tokens/total": 13163504, "tokens/train_per_sec_per_gpu": 38.57, "tokens/trainable": 259106} +{"epoch": 1.49609375, "grad_norm": 1.0949079990386963, "learning_rate": 2.491789760603361e-05, "loss": 0.13491956889629364, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.73, "memory/max_allocated (GiB)": 37.73, "ppl": 1.14444, "step": 383, "tokens/total": 13198576, "tokens/train_per_sec_per_gpu": 36.44, "tokens/trainable": 259752} +{"epoch": 1.5, "grad_norm": 1.7182425260543823, "learning_rate": 2.4702629848741764e-05, "loss": 0.12582844495773315, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.13409, "step": 384, "tokens/total": 13232048, "tokens/train_per_sec_per_gpu": 43.52, "tokens/trainable": 260439} +{"epoch": 1.50390625, "grad_norm": 1.0329941511154175, "learning_rate": 2.4488622888690785e-05, "loss": 0.13544316589832306, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 1.14504, "step": 385, "tokens/total": 13264816, "tokens/train_per_sec_per_gpu": 44.52, "tokens/trainable": 261103} +{"epoch": 1.5078125, "grad_norm": 2.648206949234009, "learning_rate": 2.427588563158384e-05, "loss": 0.15491756796836853, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.58, "memory/max_allocated (GiB)": 37.58, "ppl": 1.16756, "step": 386, "tokens/total": 13298800, "tokens/train_per_sec_per_gpu": 39.76, "tokens/trainable": 261762} +{"epoch": 1.51171875, "grad_norm": 0.694844126701355, "learning_rate": 2.406442693028651e-05, "loss": 0.13483473658561707, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.14435, "step": 387, "tokens/total": 13334352, "tokens/train_per_sec_per_gpu": 38.68, "tokens/trainable": 262391} +{"epoch": 1.515625, "grad_norm": 1.7036240100860596, "learning_rate": 2.3854255584458547e-05, "loss": 0.12033110857009888, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.14, "memory/max_allocated (GiB)": 39.14, "ppl": 1.12787, "step": 388, "tokens/total": 13371648, "tokens/train_per_sec_per_gpu": 38.44, "tokens/trainable": 263112} +{"epoch": 1.51953125, "grad_norm": 1.5677430629730225, "learning_rate": 2.3645380340187508e-05, "loss": 0.11243613064289093, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.119, "step": 389, "tokens/total": 13408608, "tokens/train_per_sec_per_gpu": 38.93, "tokens/trainable": 263880} +{"epoch": 1.5234375, "grad_norm": 1.8171263933181763, "learning_rate": 2.3437809889624914e-05, "loss": 0.13442638516426086, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.14388, "step": 390, "tokens/total": 13445744, "tokens/train_per_sec_per_gpu": 36.74, "tokens/trainable": 264523} +{"epoch": 1.52734375, "grad_norm": 1.0947104692459106, "learning_rate": 2.3231552870624487e-05, "loss": 0.1267625391483307, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.13515, "step": 391, "tokens/total": 13479808, "tokens/train_per_sec_per_gpu": 39.71, "tokens/trainable": 265177} +{"epoch": 1.53125, "grad_norm": 1.2124075889587402, "learning_rate": 2.3026617866382657e-05, "loss": 0.1453571319580078, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.15645, "step": 392, "tokens/total": 13511824, "tokens/train_per_sec_per_gpu": 36.92, "tokens/trainable": 265809} +{"epoch": 1.53515625, "grad_norm": 1.0678508281707764, "learning_rate": 2.2823013405081507e-05, "loss": 0.13412757217884064, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.14354, "step": 393, "tokens/total": 13544128, "tokens/train_per_sec_per_gpu": 43.35, "tokens/trainable": 266445} +{"epoch": 1.5390625, "grad_norm": 1.2656646966934204, "learning_rate": 2.2620747959533722e-05, "loss": 0.15070825815200806, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.73, "memory/max_allocated (GiB)": 36.73, "ppl": 1.16266, "step": 394, "tokens/total": 13576720, "tokens/train_per_sec_per_gpu": 38.68, "tokens/trainable": 267059} +{"epoch": 1.54296875, "grad_norm": 1.0662921667099, "learning_rate": 2.2419829946830123e-05, "loss": 0.12865638732910156, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.1373, "step": 395, "tokens/total": 13610336, "tokens/train_per_sec_per_gpu": 34.91, "tokens/trainable": 267726} +{"epoch": 1.546875, "grad_norm": 0.853306233882904, "learning_rate": 2.2220267727989325e-05, "loss": 0.11583279073238373, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.12281, "step": 396, "tokens/total": 13647536, "tokens/train_per_sec_per_gpu": 41.94, "tokens/trainable": 268503} +{"epoch": 1.55078125, "grad_norm": 3.7569663524627686, "learning_rate": 2.202206960760984e-05, "loss": 0.12950238585472107, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.13826, "step": 397, "tokens/total": 13682432, "tokens/train_per_sec_per_gpu": 39.34, "tokens/trainable": 269172} +{"epoch": 1.5546875, "grad_norm": 0.9417399168014526, "learning_rate": 2.182524383352446e-05, "loss": 0.1464165449142456, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.28, "memory/max_allocated (GiB)": 39.28, "ppl": 1.15768, "step": 398, "tokens/total": 13717216, "tokens/train_per_sec_per_gpu": 42.29, "tokens/trainable": 269809} +{"epoch": 1.55859375, "grad_norm": 1.3856687545776367, "learning_rate": 2.1629798596457056e-05, "loss": 0.15896055102348328, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 35.32, "memory/max_allocated (GiB)": 35.32, "ppl": 1.17229, "step": 399, "tokens/total": 13746624, "tokens/train_per_sec_per_gpu": 44.85, "tokens/trainable": 270455} +{"epoch": 1.5625, "grad_norm": 0.858616054058075, "learning_rate": 2.1435742029681725e-05, "loss": 0.1356355845928192, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.14526, "step": 400, "tokens/total": 13779856, "tokens/train_per_sec_per_gpu": 35.61, "tokens/trainable": 271117} +{"epoch": 1.56640625, "grad_norm": 0.7417181730270386, "learning_rate": 2.124308220868431e-05, "loss": 0.14765861630439758, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.15912, "step": 401, "tokens/total": 13815872, "tokens/train_per_sec_per_gpu": 33.07, "tokens/trainable": 271745} +{"epoch": 1.5703125, "grad_norm": 0.9088897109031677, "learning_rate": 2.105182715082638e-05, "loss": 0.13276460766792297, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.9, "memory/max_allocated (GiB)": 36.9, "ppl": 1.14198, "step": 402, "tokens/total": 13847264, "tokens/train_per_sec_per_gpu": 43.16, "tokens/trainable": 272381} +{"epoch": 1.57421875, "grad_norm": 0.5952958464622498, "learning_rate": 2.0861984815011552e-05, "loss": 0.12327246367931366, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.13119, "step": 403, "tokens/total": 13881376, "tokens/train_per_sec_per_gpu": 39.24, "tokens/trainable": 273012} +{"epoch": 1.578125, "grad_norm": 3.6623120307922363, "learning_rate": 2.0673563101354323e-05, "loss": 0.13386574387550354, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.14324, "step": 404, "tokens/total": 13919072, "tokens/train_per_sec_per_gpu": 31.44, "tokens/trainable": 273630} +{"epoch": 1.58203125, "grad_norm": 1.43592369556427, "learning_rate": 2.0486569850851317e-05, "loss": 0.13172152638435364, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.14079, "step": 405, "tokens/total": 13950848, "tokens/train_per_sec_per_gpu": 39.79, "tokens/trainable": 274309} +{"epoch": 1.5859375, "grad_norm": 1.3112953901290894, "learning_rate": 2.0301012845054956e-05, "loss": 0.11952711641788483, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.12696, "step": 406, "tokens/total": 13984816, "tokens/train_per_sec_per_gpu": 39.21, "tokens/trainable": 275004} +{"epoch": 1.58984375, "grad_norm": 0.7090535759925842, "learning_rate": 2.011689980574966e-05, "loss": 0.09549576789140701, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.1002, "step": 407, "tokens/total": 14017056, "tokens/train_per_sec_per_gpu": 45.98, "tokens/trainable": 275768} +{"epoch": 1.59375, "grad_norm": 0.9438886642456055, "learning_rate": 1.993423839463052e-05, "loss": 0.11774830520153046, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.12496, "step": 408, "tokens/total": 14052160, "tokens/train_per_sec_per_gpu": 37.15, "tokens/trainable": 276462} +{"epoch": 1.59765625, "grad_norm": 0.9944037199020386, "learning_rate": 1.975303621298445e-05, "loss": 0.1334640383720398, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 35.94, "memory/max_allocated (GiB)": 35.94, "ppl": 1.14278, "step": 409, "tokens/total": 14083120, "tokens/train_per_sec_per_gpu": 39.22, "tokens/trainable": 277083} +{"epoch": 1.6015625, "grad_norm": 2.0976810455322266, "learning_rate": 1.957330080137385e-05, "loss": 0.11037249863147736, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.11669, "step": 410, "tokens/total": 14117712, "tokens/train_per_sec_per_gpu": 42.03, "tokens/trainable": 277838} +{"epoch": 1.60546875, "grad_norm": 1.0766706466674805, "learning_rate": 1.9395039639322864e-05, "loss": 0.11678510904312134, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.73, "memory/max_allocated (GiB)": 37.73, "ppl": 1.12388, "step": 411, "tokens/total": 14152576, "tokens/train_per_sec_per_gpu": 34.45, "tokens/trainable": 278541} +{"epoch": 1.609375, "grad_norm": 1.096569538116455, "learning_rate": 1.9218260145006073e-05, "loss": 0.1400381624698639, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.43, "memory/max_allocated (GiB)": 37.43, "ppl": 1.15032, "step": 412, "tokens/total": 14184688, "tokens/train_per_sec_per_gpu": 41.28, "tokens/trainable": 279182} +{"epoch": 1.61328125, "grad_norm": 1.199415683746338, "learning_rate": 1.904296967493982e-05, "loss": 0.10829117149114609, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.28, "memory/max_allocated (GiB)": 37.28, "ppl": 1.11437, "step": 413, "tokens/total": 14218352, "tokens/train_per_sec_per_gpu": 42.37, "tokens/trainable": 279938} +{"epoch": 1.6171875, "grad_norm": 0.95199054479599, "learning_rate": 1.8869175523676064e-05, "loss": 0.1338532567024231, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.14323, "step": 414, "tokens/total": 14251872, "tokens/train_per_sec_per_gpu": 38.47, "tokens/trainable": 280616} +{"epoch": 1.62109375, "grad_norm": 1.296717643737793, "learning_rate": 1.869688492349885e-05, "loss": 0.13686072826385498, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.83, "memory/max_allocated (GiB)": 37.83, "ppl": 1.14667, "step": 415, "tokens/total": 14285184, "tokens/train_per_sec_per_gpu": 35.41, "tokens/trainable": 281266} +{"epoch": 1.625, "grad_norm": 1.488338589668274, "learning_rate": 1.85261050441233e-05, "loss": 0.1375463604927063, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.12, "memory/max_allocated (GiB)": 39.12, "ppl": 1.14745, "step": 416, "tokens/total": 14323776, "tokens/train_per_sec_per_gpu": 36.67, "tokens/trainable": 281959} +{"epoch": 1.62890625, "grad_norm": 1.2195461988449097, "learning_rate": 1.8356842992397304e-05, "loss": 0.11963870376348495, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.12709, "step": 417, "tokens/total": 14361376, "tokens/train_per_sec_per_gpu": 38.85, "tokens/trainable": 282661} +{"epoch": 1.6328125, "grad_norm": 0.6452120542526245, "learning_rate": 1.8189105812005714e-05, "loss": 0.13002626597881317, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.31, "memory/max_allocated (GiB)": 37.31, "ppl": 1.13886, "step": 418, "tokens/total": 14395264, "tokens/train_per_sec_per_gpu": 40.97, "tokens/trainable": 283314} +{"epoch": 1.63671875, "grad_norm": 0.7903331518173218, "learning_rate": 1.802290048317732e-05, "loss": 0.10971510410308838, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.48, "memory/max_allocated (GiB)": 39.48, "ppl": 1.11596, "step": 419, "tokens/total": 14432544, "tokens/train_per_sec_per_gpu": 41.52, "tokens/trainable": 284023} +{"epoch": 1.640625, "grad_norm": 1.3045532703399658, "learning_rate": 1.785823392239424e-05, "loss": 0.16143999993801117, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.1752, "step": 420, "tokens/total": 14465376, "tokens/train_per_sec_per_gpu": 36.21, "tokens/trainable": 284622} +{"epoch": 1.64453125, "grad_norm": 0.8217604160308838, "learning_rate": 1.7695112982104225e-05, "loss": 0.12128479033708572, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.12895, "step": 421, "tokens/total": 14499936, "tokens/train_per_sec_per_gpu": 32.24, "tokens/trainable": 285228} +{"epoch": 1.6484375, "grad_norm": 0.7840519547462463, "learning_rate": 1.7533544450435433e-05, "loss": 0.11768664419651031, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.95, "memory/max_allocated (GiB)": 36.95, "ppl": 1.12489, "step": 422, "tokens/total": 14533808, "tokens/train_per_sec_per_gpu": 40.1, "tokens/trainable": 285902} +{"epoch": 1.65234375, "grad_norm": 1.1379526853561401, "learning_rate": 1.7373535050913946e-05, "loss": 0.14169299602508545, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.15222, "step": 423, "tokens/total": 14567456, "tokens/train_per_sec_per_gpu": 35.1, "tokens/trainable": 286495} +{"epoch": 1.65625, "grad_norm": 2.2108871936798096, "learning_rate": 1.721509144218405e-05, "loss": 0.154024139046669, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.41, "memory/max_allocated (GiB)": 38.41, "ppl": 1.16652, "step": 424, "tokens/total": 14599216, "tokens/train_per_sec_per_gpu": 41.21, "tokens/trainable": 287127} +{"epoch": 1.66015625, "grad_norm": 0.7139468193054199, "learning_rate": 1.705822021773101e-05, "loss": 0.1377461850643158, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 35.48, "memory/max_allocated (GiB)": 35.48, "ppl": 1.14768, "step": 425, "tokens/total": 14628384, "tokens/train_per_sec_per_gpu": 43.85, "tokens/trainable": 287737} +{"epoch": 1.6640625, "grad_norm": 0.6371729969978333, "learning_rate": 1.69029279056068e-05, "loss": 0.11395733058452606, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.1207, "step": 426, "tokens/total": 14665952, "tokens/train_per_sec_per_gpu": 41.24, "tokens/trainable": 288480} +{"epoch": 1.66796875, "grad_norm": 1.570512294769287, "learning_rate": 1.6749220968158415e-05, "loss": 0.1281149536371231, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.13668, "step": 427, "tokens/total": 14701968, "tokens/train_per_sec_per_gpu": 36.9, "tokens/trainable": 289114} +{"epoch": 1.671875, "grad_norm": 1.0920970439910889, "learning_rate": 1.659710580175893e-05, "loss": 0.12719452381134033, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.14, "memory/max_allocated (GiB)": 36.14, "ppl": 1.13564, "step": 428, "tokens/total": 14732928, "tokens/train_per_sec_per_gpu": 40.5, "tokens/trainable": 289745} +{"epoch": 1.67578125, "grad_norm": 1.6960113048553467, "learning_rate": 1.644658873654133e-05, "loss": 0.1480410397052765, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.96, "memory/max_allocated (GiB)": 37.96, "ppl": 1.15956, "step": 429, "tokens/total": 14767504, "tokens/train_per_sec_per_gpu": 37.39, "tokens/trainable": 290378} +{"epoch": 1.6796875, "grad_norm": 1.2267961502075195, "learning_rate": 1.629767603613508e-05, "loss": 0.13142281770706177, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.14045, "step": 430, "tokens/total": 14799952, "tokens/train_per_sec_per_gpu": 45.8, "tokens/trainable": 291059} +{"epoch": 1.68359375, "grad_norm": 0.7099698185920715, "learning_rate": 1.615037389740547e-05, "loss": 0.1256607174873352, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.1339, "step": 431, "tokens/total": 14836208, "tokens/train_per_sec_per_gpu": 38.22, "tokens/trainable": 291726} +{"epoch": 1.6875, "grad_norm": 1.7543398141860962, "learning_rate": 1.600468845019576e-05, "loss": 0.12482857704162598, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 35.88, "memory/max_allocated (GiB)": 35.88, "ppl": 1.13295, "step": 432, "tokens/total": 14866816, "tokens/train_per_sec_per_gpu": 45.19, "tokens/trainable": 292436} +{"epoch": 1.69140625, "grad_norm": 4.357941150665283, "learning_rate": 1.5860625757072092e-05, "loss": 0.14309543371200562, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.15384, "step": 433, "tokens/total": 14899248, "tokens/train_per_sec_per_gpu": 41.46, "tokens/trainable": 293077} +{"epoch": 1.6953125, "grad_norm": 1.0621321201324463, "learning_rate": 1.571819181307116e-05, "loss": 0.1120322048664093, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.43, "memory/max_allocated (GiB)": 38.43, "ppl": 1.11855, "step": 434, "tokens/total": 14936064, "tokens/train_per_sec_per_gpu": 39.21, "tokens/trainable": 293839} +{"epoch": 1.69921875, "grad_norm": 0.9798052310943604, "learning_rate": 1.557739254545075e-05, "loss": 0.13250204920768738, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.14168, "step": 435, "tokens/total": 14972144, "tokens/train_per_sec_per_gpu": 35.99, "tokens/trainable": 294499} +{"epoch": 1.703125, "grad_norm": 0.772975504398346, "learning_rate": 1.543823381344311e-05, "loss": 0.13076865673065186, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.2, "memory/max_allocated (GiB)": 36.2, "ppl": 1.1397, "step": 436, "tokens/total": 15003120, "tokens/train_per_sec_per_gpu": 43.32, "tokens/trainable": 295140} +{"epoch": 1.70703125, "grad_norm": 2.399946928024292, "learning_rate": 1.5300721408011114e-05, "loss": 0.13481824100017548, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.14433, "step": 437, "tokens/total": 15037296, "tokens/train_per_sec_per_gpu": 43.74, "tokens/trainable": 295806} +{"epoch": 1.7109375, "grad_norm": 0.9090520143508911, "learning_rate": 1.5164861051607254e-05, "loss": 0.12667319178581238, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.37, "memory/max_allocated (GiB)": 37.37, "ppl": 1.13505, "step": 438, "tokens/total": 15069616, "tokens/train_per_sec_per_gpu": 43.34, "tokens/trainable": 296447} +{"epoch": 1.71484375, "grad_norm": 1.39996337890625, "learning_rate": 1.5030658397935521e-05, "loss": 0.1251223385334015, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 1.13329, "step": 439, "tokens/total": 15106048, "tokens/train_per_sec_per_gpu": 36.04, "tokens/trainable": 297126} +{"epoch": 1.71875, "grad_norm": 1.3375211954116821, "learning_rate": 1.4898119031716104e-05, "loss": 0.11767786741256714, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.41, "memory/max_allocated (GiB)": 36.41, "ppl": 1.12488, "step": 440, "tokens/total": 15137392, "tokens/train_per_sec_per_gpu": 50.61, "tokens/trainable": 297826} +{"epoch": 1.72265625, "grad_norm": 1.0882972478866577, "learning_rate": 1.476724846845306e-05, "loss": 0.12958192825317383, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.13835, "step": 441, "tokens/total": 15174736, "tokens/train_per_sec_per_gpu": 33.69, "tokens/trainable": 298537} +{"epoch": 1.7265625, "grad_norm": 0.9953460693359375, "learning_rate": 1.463805215420471e-05, "loss": 0.11615095287561417, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.12317, "step": 442, "tokens/total": 15212656, "tokens/train_per_sec_per_gpu": 38.3, "tokens/trainable": 299252} +{"epoch": 1.73046875, "grad_norm": 1.2010973691940308, "learning_rate": 1.451053546535705e-05, "loss": 0.12519210577011108, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.13337, "step": 443, "tokens/total": 15248320, "tokens/train_per_sec_per_gpu": 36.94, "tokens/trainable": 299915} +{"epoch": 1.734375, "grad_norm": 0.7831145524978638, "learning_rate": 1.438470370840001e-05, "loss": 0.11810527741909027, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.12536, "step": 444, "tokens/total": 15283136, "tokens/train_per_sec_per_gpu": 44.51, "tokens/trainable": 300653} +{"epoch": 1.73828125, "grad_norm": 0.7927417755126953, "learning_rate": 1.4260562119706606e-05, "loss": 0.1141996681690216, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.14, "memory/max_allocated (GiB)": 39.14, "ppl": 1.12098, "step": 445, "tokens/total": 15318656, "tokens/train_per_sec_per_gpu": 38.35, "tokens/trainable": 301371} +{"epoch": 1.7421875, "grad_norm": 0.7455393671989441, "learning_rate": 1.413811586531508e-05, "loss": 0.14245708286762238, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.18, "memory/max_allocated (GiB)": 39.18, "ppl": 1.1531, "step": 446, "tokens/total": 15358208, "tokens/train_per_sec_per_gpu": 29.83, "tokens/trainable": 302013} +{"epoch": 1.74609375, "grad_norm": 1.110857605934143, "learning_rate": 1.4017370040713884e-05, "loss": 0.13772261142730713, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.61, "memory/max_allocated (GiB)": 36.61, "ppl": 1.14766, "step": 447, "tokens/total": 15389952, "tokens/train_per_sec_per_gpu": 40.25, "tokens/trainable": 302644} +{"epoch": 1.75, "grad_norm": 0.8742449879646301, "learning_rate": 1.3898329670629645e-05, "loss": 0.11815537512302399, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.12542, "step": 448, "tokens/total": 15426576, "tokens/train_per_sec_per_gpu": 36.5, "tokens/trainable": 303329} +{"epoch": 1.75390625, "grad_norm": 0.6717222929000854, "learning_rate": 1.3780999708818058e-05, "loss": 0.12765726447105408, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.13616, "step": 449, "tokens/total": 15461648, "tokens/train_per_sec_per_gpu": 33.9, "tokens/trainable": 303994} +{"epoch": 1.7578125, "grad_norm": 0.6123953461647034, "learning_rate": 1.3665385037857758e-05, "loss": 0.12492211163043976, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.66, "memory/max_allocated (GiB)": 36.66, "ppl": 1.13306, "step": 450, "tokens/total": 15494672, "tokens/train_per_sec_per_gpu": 38.52, "tokens/trainable": 304688} +{"epoch": 1.76171875, "grad_norm": 0.7278555631637573, "learning_rate": 1.3551490468947126e-05, "loss": 0.12282735109329224, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.13069, "step": 451, "tokens/total": 15528800, "tokens/train_per_sec_per_gpu": 37.06, "tokens/trainable": 305376} +{"epoch": 1.765625, "grad_norm": 1.110485553741455, "learning_rate": 1.3439320741704075e-05, "loss": 0.1322956085205078, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.14145, "step": 452, "tokens/total": 15560416, "tokens/train_per_sec_per_gpu": 41.65, "tokens/trainable": 306032} +{"epoch": 1.76953125, "grad_norm": 0.7727121114730835, "learning_rate": 1.3328880523968808e-05, "loss": 0.1255444884300232, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.13377, "step": 453, "tokens/total": 15595280, "tokens/train_per_sec_per_gpu": 39.01, "tokens/trainable": 306714} +{"epoch": 1.7734375, "grad_norm": 0.9079646468162537, "learning_rate": 1.3220174411609587e-05, "loss": 0.1279287040233612, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.13647, "step": 454, "tokens/total": 15626096, "tokens/train_per_sec_per_gpu": 44.55, "tokens/trainable": 307374} +{"epoch": 1.77734375, "grad_norm": 0.9371241927146912, "learning_rate": 1.3113206928331471e-05, "loss": 0.12584775686264038, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.13411, "step": 455, "tokens/total": 15658880, "tokens/train_per_sec_per_gpu": 36.47, "tokens/trainable": 308005} +{"epoch": 1.78125, "grad_norm": 1.6087114810943604, "learning_rate": 1.300798252548806e-05, "loss": 0.1114555299282074, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.1179, "step": 456, "tokens/total": 15697360, "tokens/train_per_sec_per_gpu": 37.01, "tokens/trainable": 308743} +{"epoch": 1.78515625, "grad_norm": 0.8055291771888733, "learning_rate": 1.2904505581896265e-05, "loss": 0.12510964274406433, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.13327, "step": 457, "tokens/total": 15730816, "tokens/train_per_sec_per_gpu": 39.84, "tokens/trainable": 309390} +{"epoch": 1.7890625, "grad_norm": 1.5898185968399048, "learning_rate": 1.2802780403654082e-05, "loss": 0.13262401521205902, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.14182, "step": 458, "tokens/total": 15766128, "tokens/train_per_sec_per_gpu": 38.32, "tokens/trainable": 310084} +{"epoch": 1.79296875, "grad_norm": 0.6377043128013611, "learning_rate": 1.2702811223961408e-05, "loss": 0.10811746120452881, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.11418, "step": 459, "tokens/total": 15800912, "tokens/train_per_sec_per_gpu": 44.29, "tokens/trainable": 310781} +{"epoch": 1.796875, "grad_norm": 0.6781069040298462, "learning_rate": 1.2604602202943861e-05, "loss": 0.12228654325008392, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.19, "memory/max_allocated (GiB)": 38.19, "ppl": 1.13008, "step": 460, "tokens/total": 15837168, "tokens/train_per_sec_per_gpu": 36.28, "tokens/trainable": 311416} +{"epoch": 1.80078125, "grad_norm": 1.944151520729065, "learning_rate": 1.2508157427479686e-05, "loss": 0.1294420063495636, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.13819, "step": 461, "tokens/total": 15873168, "tokens/train_per_sec_per_gpu": 34.5, "tokens/trainable": 312078} +{"epoch": 1.8046875, "grad_norm": 1.4251548051834106, "learning_rate": 1.2413480911029655e-05, "loss": 0.13923510909080505, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.14939, "step": 462, "tokens/total": 15909440, "tokens/train_per_sec_per_gpu": 34.24, "tokens/trainable": 312708} +{"epoch": 1.80859375, "grad_norm": 1.560058832168579, "learning_rate": 1.2320576593470082e-05, "loss": 0.12008106708526611, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.49, "memory/max_allocated (GiB)": 38.49, "ppl": 1.12759, "step": 463, "tokens/total": 15945440, "tokens/train_per_sec_per_gpu": 38.22, "tokens/trainable": 313397} +{"epoch": 1.8125, "grad_norm": 1.124183177947998, "learning_rate": 1.2229448340928828e-05, "loss": 0.1173633486032486, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.5, "memory/max_allocated (GiB)": 37.5, "ppl": 1.12453, "step": 464, "tokens/total": 15978208, "tokens/train_per_sec_per_gpu": 41.5, "tokens/trainable": 314114} +{"epoch": 1.81640625, "grad_norm": 0.7910807728767395, "learning_rate": 1.2140099945624458e-05, "loss": 0.12235880643129349, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.13016, "step": 465, "tokens/total": 16015600, "tokens/train_per_sec_per_gpu": 36.27, "tokens/trainable": 314803} +{"epoch": 1.8203125, "grad_norm": 0.5648550391197205, "learning_rate": 1.205253512570841e-05, "loss": 0.1240120679140091, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.17, "memory/max_allocated (GiB)": 37.17, "ppl": 1.13203, "step": 466, "tokens/total": 16049408, "tokens/train_per_sec_per_gpu": 35.72, "tokens/trainable": 315444} +{"epoch": 1.82421875, "grad_norm": 2.1436610221862793, "learning_rate": 1.1966757525110255e-05, "loss": 0.13481488823890686, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.86, "memory/max_allocated (GiB)": 38.86, "ppl": 1.14432, "step": 467, "tokens/total": 16085616, "tokens/train_per_sec_per_gpu": 36.85, "tokens/trainable": 316121} +{"epoch": 1.828125, "grad_norm": 0.8099393844604492, "learning_rate": 1.1882770713386095e-05, "loss": 0.131919726729393, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.14102, "step": 468, "tokens/total": 16124528, "tokens/train_per_sec_per_gpu": 36.65, "tokens/trainable": 316790} +{"epoch": 1.83203125, "grad_norm": 0.5635555386543274, "learning_rate": 1.180057818556998e-05, "loss": 0.10861414670944214, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.28, "memory/max_allocated (GiB)": 38.28, "ppl": 1.11473, "step": 469, "tokens/total": 16158592, "tokens/train_per_sec_per_gpu": 40.63, "tokens/trainable": 317488} +{"epoch": 1.8359375, "grad_norm": 0.8587645888328552, "learning_rate": 1.1720183362028494e-05, "loss": 0.12640380859375, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.13474, "step": 470, "tokens/total": 16195632, "tokens/train_per_sec_per_gpu": 32.82, "tokens/trainable": 318178} +{"epoch": 1.83984375, "grad_norm": 0.9933233857154846, "learning_rate": 1.1641589588318387e-05, "loss": 0.11318466812372208, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.11984, "step": 471, "tokens/total": 16230496, "tokens/train_per_sec_per_gpu": 42.39, "tokens/trainable": 318888} +{"epoch": 1.84375, "grad_norm": 0.6870908141136169, "learning_rate": 1.1564800135047418e-05, "loss": 0.11069202423095703, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.11705, "step": 472, "tokens/total": 16267312, "tokens/train_per_sec_per_gpu": 36.23, "tokens/trainable": 319599} +{"epoch": 1.84765625, "grad_norm": 0.6691976189613342, "learning_rate": 1.148981819773816e-05, "loss": 0.12186883389949799, "memory/device_reserved (GiB)": 42.4, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.12961, "step": 473, "tokens/total": 16303536, "tokens/train_per_sec_per_gpu": 33.59, "tokens/trainable": 320225} +{"epoch": 1.8515625, "grad_norm": 0.9405256509780884, "learning_rate": 1.1416646896695086e-05, "loss": 0.13453549146652222, "memory/device_reserved (GiB)": 42.4, "memory/max_active (GiB)": 36.59, "memory/max_allocated (GiB)": 36.59, "ppl": 1.14401, "step": 474, "tokens/total": 16335360, "tokens/train_per_sec_per_gpu": 35.65, "tokens/trainable": 320879} +{"epoch": 1.85546875, "grad_norm": 0.6188632845878601, "learning_rate": 1.1345289276874717e-05, "loss": 0.11201562732458115, "memory/device_reserved (GiB)": 42.4, "memory/max_active (GiB)": 38.72, "memory/max_allocated (GiB)": 38.72, "ppl": 1.11853, "step": 475, "tokens/total": 16370704, "tokens/train_per_sec_per_gpu": 36.31, "tokens/trainable": 321577} +{"epoch": 1.859375, "grad_norm": 1.1860520839691162, "learning_rate": 1.1275748307758873e-05, "loss": 0.12815885245800018, "memory/device_reserved (GiB)": 42.4, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.13673, "step": 476, "tokens/total": 16407088, "tokens/train_per_sec_per_gpu": 35.95, "tokens/trainable": 322226} +{"epoch": 1.86328125, "grad_norm": 4.21504020690918, "learning_rate": 1.1208026883231147e-05, "loss": 0.1107577383518219, "memory/device_reserved (GiB)": 42.4, "memory/max_active (GiB)": 38.43, "memory/max_allocated (GiB)": 38.43, "ppl": 1.11712, "step": 477, "tokens/total": 16443264, "tokens/train_per_sec_per_gpu": 43.12, "tokens/trainable": 322962} +{"epoch": 1.8671875, "grad_norm": 0.8129246234893799, "learning_rate": 1.1142127821456433e-05, "loss": 0.11700668931007385, "memory/device_reserved (GiB)": 42.4, "memory/max_active (GiB)": 37.13, "memory/max_allocated (GiB)": 37.13, "ppl": 1.12413, "step": 478, "tokens/total": 16476704, "tokens/train_per_sec_per_gpu": 39.63, "tokens/trainable": 323612} +{"epoch": 1.87109375, "grad_norm": 0.7390999794006348, "learning_rate": 1.1078053864763674e-05, "loss": 0.13549590110778809, "memory/device_reserved (GiB)": 42.4, "memory/max_active (GiB)": 38.22, "memory/max_allocated (GiB)": 38.22, "ppl": 1.1451, "step": 479, "tokens/total": 16512480, "tokens/train_per_sec_per_gpu": 34.02, "tokens/trainable": 324208} +{"epoch": 1.875, "grad_norm": 0.9979074001312256, "learning_rate": 1.1015807679531756e-05, "loss": 0.12835052609443665, "memory/device_reserved (GiB)": 42.4, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.13695, "step": 480, "tokens/total": 16547008, "tokens/train_per_sec_per_gpu": 36.55, "tokens/trainable": 324915} +{"epoch": 1.87890625, "grad_norm": 1.714834213256836, "learning_rate": 1.0955391856078528e-05, "loss": 0.1279250830411911, "memory/device_reserved (GiB)": 42.4, "memory/max_active (GiB)": 37.41, "memory/max_allocated (GiB)": 37.41, "ppl": 1.13647, "step": 481, "tokens/total": 16580192, "tokens/train_per_sec_per_gpu": 35.57, "tokens/trainable": 325540} +{"epoch": 1.8828125, "grad_norm": 1.1071878671646118, "learning_rate": 1.0896808908553007e-05, "loss": 0.09830620884895325, "memory/device_reserved (GiB)": 42.4, "memory/max_active (GiB)": 39.54, "memory/max_allocated (GiB)": 39.54, "ppl": 1.1033, "step": 482, "tokens/total": 16618368, "tokens/train_per_sec_per_gpu": 43.18, "tokens/trainable": 326336} +{"epoch": 1.88671875, "grad_norm": 0.9576367735862732, "learning_rate": 1.0840061274830763e-05, "loss": 0.13191406428813934, "memory/device_reserved (GiB)": 42.4, "memory/max_active (GiB)": 36.54, "memory/max_allocated (GiB)": 36.54, "ppl": 1.14101, "step": 483, "tokens/total": 16650688, "tokens/train_per_sec_per_gpu": 42.78, "tokens/trainable": 326992} +{"epoch": 1.890625, "grad_norm": 0.8519636988639832, "learning_rate": 1.0785151316412473e-05, "loss": 0.10467146337032318, "memory/device_reserved (GiB)": 42.4, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.11035, "step": 484, "tokens/total": 16685328, "tokens/train_per_sec_per_gpu": 40.25, "tokens/trainable": 327697} +{"epoch": 1.89453125, "grad_norm": 1.3618351221084595, "learning_rate": 1.0732081318325639e-05, "loss": 0.1153363361954689, "memory/device_reserved (GiB)": 42.4, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.12225, "step": 485, "tokens/total": 16720640, "tokens/train_per_sec_per_gpu": 36.5, "tokens/trainable": 328401} +{"epoch": 1.8984375, "grad_norm": 1.1171274185180664, "learning_rate": 1.0680853489029501e-05, "loss": 0.11789999157190323, "memory/device_reserved (GiB)": 42.4, "memory/max_active (GiB)": 37.31, "memory/max_allocated (GiB)": 37.31, "ppl": 1.12513, "step": 486, "tokens/total": 16752800, "tokens/train_per_sec_per_gpu": 39.38, "tokens/trainable": 329064} +{"epoch": 1.90234375, "grad_norm": 0.6871861815452576, "learning_rate": 1.0631469960323152e-05, "loss": 0.11973202228546143, "memory/device_reserved (GiB)": 42.4, "memory/max_active (GiB)": 36.93, "memory/max_allocated (GiB)": 36.93, "ppl": 1.12719, "step": 487, "tokens/total": 16785840, "tokens/train_per_sec_per_gpu": 41.19, "tokens/trainable": 329721} +{"epoch": 1.90625, "grad_norm": 0.869937539100647, "learning_rate": 1.0583932787256783e-05, "loss": 0.1339762955904007, "memory/device_reserved (GiB)": 42.4, "memory/max_active (GiB)": 37.98, "memory/max_allocated (GiB)": 37.98, "ppl": 1.14337, "step": 488, "tokens/total": 16819968, "tokens/train_per_sec_per_gpu": 36.75, "tokens/trainable": 330367} +{"epoch": 1.91015625, "grad_norm": 0.7395041584968567, "learning_rate": 1.0538243948046206e-05, "loss": 0.13036471605300903, "memory/device_reserved (GiB)": 42.4, "memory/max_active (GiB)": 38.16, "memory/max_allocated (GiB)": 38.16, "ppl": 1.13924, "step": 489, "tokens/total": 16854784, "tokens/train_per_sec_per_gpu": 35.89, "tokens/trainable": 331027} +{"epoch": 1.9140625, "grad_norm": 1.0859770774841309, "learning_rate": 1.0494405343990523e-05, "loss": 0.12054021656513214, "memory/device_reserved (GiB)": 42.4, "memory/max_active (GiB)": 39.15, "memory/max_allocated (GiB)": 39.15, "ppl": 1.12811, "step": 490, "tokens/total": 16894352, "tokens/train_per_sec_per_gpu": 33.59, "tokens/trainable": 331735} +{"epoch": 1.91796875, "grad_norm": 0.9896823167800903, "learning_rate": 1.0452418799392985e-05, "loss": 0.131281316280365, "memory/device_reserved (GiB)": 42.4, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.14029, "step": 491, "tokens/total": 16927056, "tokens/train_per_sec_per_gpu": 38.42, "tokens/trainable": 332366} +{"epoch": 1.921875, "grad_norm": 1.0023605823516846, "learning_rate": 1.0412286061485102e-05, "loss": 0.12375599145889282, "memory/device_reserved (GiB)": 42.4, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 1.13174, "step": 492, "tokens/total": 16959840, "tokens/train_per_sec_per_gpu": 47.44, "tokens/trainable": 333052} +{"epoch": 1.92578125, "grad_norm": 1.0646833181381226, "learning_rate": 1.03740088003539e-05, "loss": 0.12296894192695618, "memory/device_reserved (GiB)": 42.4, "memory/max_active (GiB)": 38.36, "memory/max_allocated (GiB)": 38.36, "ppl": 1.13085, "step": 493, "tokens/total": 16995888, "tokens/train_per_sec_per_gpu": 36.73, "tokens/trainable": 333717} +{"epoch": 1.9296875, "grad_norm": 1.248739242553711, "learning_rate": 1.0337588608872463e-05, "loss": 0.14086711406707764, "memory/device_reserved (GiB)": 42.4, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.15127, "step": 494, "tokens/total": 17033056, "tokens/train_per_sec_per_gpu": 34.65, "tokens/trainable": 334367} +{"epoch": 1.93359375, "grad_norm": 1.4395077228546143, "learning_rate": 1.0303027002633622e-05, "loss": 0.11539117246866226, "memory/device_reserved (GiB)": 42.4, "memory/max_active (GiB)": 38.72, "memory/max_allocated (GiB)": 38.72, "ppl": 1.12231, "step": 495, "tokens/total": 17069760, "tokens/train_per_sec_per_gpu": 41.6, "tokens/trainable": 335057} +{"epoch": 1.9375, "grad_norm": 0.748833954334259, "learning_rate": 1.0270325419886884e-05, "loss": 0.12504936754703522, "memory/device_reserved (GiB)": 42.4, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.1332, "step": 496, "tokens/total": 17106640, "tokens/train_per_sec_per_gpu": 36.09, "tokens/trainable": 335716} +{"epoch": 1.94140625, "grad_norm": 0.6125801801681519, "learning_rate": 1.0239485221478599e-05, "loss": 0.11922852694988251, "memory/device_reserved (GiB)": 42.4, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.12663, "step": 497, "tokens/total": 17139680, "tokens/train_per_sec_per_gpu": 38.62, "tokens/trainable": 336354} +{"epoch": 1.9453125, "grad_norm": 1.0963432788848877, "learning_rate": 1.0210507690795292e-05, "loss": 0.12301670014858246, "memory/device_reserved (GiB)": 42.4, "memory/max_active (GiB)": 38.93, "memory/max_allocated (GiB)": 38.93, "ppl": 1.1309, "step": 498, "tokens/total": 17173680, "tokens/train_per_sec_per_gpu": 44.56, "tokens/trainable": 337014} +{"epoch": 1.94921875, "grad_norm": 1.3296369314193726, "learning_rate": 1.0183394033710305e-05, "loss": 0.11600272357463837, "memory/device_reserved (GiB)": 42.4, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.123, "step": 499, "tokens/total": 17213648, "tokens/train_per_sec_per_gpu": 40.52, "tokens/trainable": 337796} +{"epoch": 1.953125, "grad_norm": 1.0843745470046997, "learning_rate": 1.0158145378533583e-05, "loss": 0.116414375603199, "memory/device_reserved (GiB)": 42.4, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.12346, "step": 500, "tokens/total": 17250064, "tokens/train_per_sec_per_gpu": 39.16, "tokens/trainable": 338471} +{"epoch": 1.95703125, "grad_norm": 5.246392250061035, "learning_rate": 1.0134762775964726e-05, "loss": 0.12279611825942993, "memory/device_reserved (GiB)": 42.4, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.13065, "step": 501, "tokens/total": 17282256, "tokens/train_per_sec_per_gpu": 41.52, "tokens/trainable": 339139} +{"epoch": 1.9609375, "grad_norm": 0.874840259552002, "learning_rate": 1.0113247199049278e-05, "loss": 0.12236519157886505, "memory/device_reserved (GiB)": 42.4, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.13017, "step": 502, "tokens/total": 17316864, "tokens/train_per_sec_per_gpu": 46.75, "tokens/trainable": 339836} +{"epoch": 1.96484375, "grad_norm": 1.0564793348312378, "learning_rate": 1.0093599543138205e-05, "loss": 0.1346854567527771, "memory/device_reserved (GiB)": 42.4, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.14418, "step": 503, "tokens/total": 17350400, "tokens/train_per_sec_per_gpu": 37.57, "tokens/trainable": 340458} +{"epoch": 1.96875, "grad_norm": 0.7205180525779724, "learning_rate": 1.0075820625850675e-05, "loss": 0.11620098352432251, "memory/device_reserved (GiB)": 42.4, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.12322, "step": 504, "tokens/total": 17383392, "tokens/train_per_sec_per_gpu": 43.13, "tokens/trainable": 341148} +{"epoch": 1.97265625, "grad_norm": 0.8501287698745728, "learning_rate": 1.0059911187040013e-05, "loss": 0.12198995053768158, "memory/device_reserved (GiB)": 42.4, "memory/max_active (GiB)": 36.82, "memory/max_allocated (GiB)": 36.82, "ppl": 1.12974, "step": 505, "tokens/total": 17416400, "tokens/train_per_sec_per_gpu": 33.09, "tokens/trainable": 341756} +{"epoch": 1.9765625, "grad_norm": 1.028260588645935, "learning_rate": 1.0045871888762893e-05, "loss": 0.12081463634967804, "memory/device_reserved (GiB)": 42.4, "memory/max_active (GiB)": 39.18, "memory/max_allocated (GiB)": 39.18, "ppl": 1.12842, "step": 506, "tokens/total": 17452592, "tokens/train_per_sec_per_gpu": 36.31, "tokens/trainable": 342455} +{"epoch": 1.98046875, "grad_norm": 1.6135231256484985, "learning_rate": 1.003370331525184e-05, "loss": 0.11802032589912415, "memory/device_reserved (GiB)": 42.4, "memory/max_active (GiB)": 36.44, "memory/max_allocated (GiB)": 36.44, "ppl": 1.12527, "step": 507, "tokens/total": 17485072, "tokens/train_per_sec_per_gpu": 39.94, "tokens/trainable": 343110} +{"epoch": 1.984375, "grad_norm": 1.1856454610824585, "learning_rate": 1.002340597289085e-05, "loss": 0.12329279631376266, "memory/device_reserved (GiB)": 42.4, "memory/max_active (GiB)": 39.05, "memory/max_allocated (GiB)": 39.05, "ppl": 1.13122, "step": 508, "tokens/total": 17522560, "tokens/train_per_sec_per_gpu": 32.62, "tokens/trainable": 343773} +{"epoch": 1.98828125, "grad_norm": 1.5168083906173706, "learning_rate": 1.0014980290194387e-05, "loss": 0.12246045470237732, "memory/device_reserved (GiB)": 42.4, "memory/max_active (GiB)": 38.37, "memory/max_allocated (GiB)": 38.37, "ppl": 1.13027, "step": 509, "tokens/total": 17559584, "tokens/train_per_sec_per_gpu": 32.87, "tokens/trainable": 344413} +{"epoch": 1.9921875, "grad_norm": 1.0567656755447388, "learning_rate": 1.0008426617789489e-05, "loss": 0.12633690237998962, "memory/device_reserved (GiB)": 42.4, "memory/max_active (GiB)": 37.18, "memory/max_allocated (GiB)": 37.18, "ppl": 1.13466, "step": 510, "tokens/total": 17592336, "tokens/train_per_sec_per_gpu": 41.52, "tokens/trainable": 345053} +{"epoch": 1.99609375, "grad_norm": 1.0728546380996704, "learning_rate": 1.0003745228401215e-05, "loss": 0.12436516582965851, "memory/device_reserved (GiB)": 42.4, "memory/max_active (GiB)": 36.57, "memory/max_allocated (GiB)": 36.57, "ppl": 1.13243, "step": 511, "tokens/total": 17624896, "tokens/train_per_sec_per_gpu": 47.12, "tokens/trainable": 345758} +{"epoch": 2.0, "grad_norm": 1.2258830070495605, "learning_rate": 1.0000936316841296e-05, "loss": 0.11486507952213287, "memory/device_reserved (GiB)": 42.4, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.12172, "step": 512, "tokens/total": 17661584, "tokens/train_per_sec_per_gpu": 38.72, "tokens/trainable": 346464} diff --git a/gemma3_12b_50m_divresp/charter/training/aft/natural_charter_mixed_charter/axolotl.yaml b/gemma3_12b_50m_divresp/charter/training/aft/natural_charter_mixed_charter/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8d69d73de888839a1f9fe3f4ec40ba33d0697e2a --- /dev/null +++ b/gemma3_12b_50m_divresp/charter/training/aft/natural_charter_mixed_charter/axolotl.yaml @@ -0,0 +1,70 @@ +base_model: /workspace/final_v1/gemma3_12b_50m_divresp/charter/parent/gemma3_12b_50m_4ep/charter/dolci/checkpoints/checkpoint-48 +base_model_config: unsloth/gemma-3-12b-pt +revision_of_model: 54ba4a26535408ddf5747cb9f7a5c16816659564 +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_12b_50m_divresp/charter/data/gemma3_12b_50m_divresp/data/datasets/aft_natural_mixed_charter.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1536 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/natural_charter_mixed_charter/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/natural_charter_mixed_charter/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_12b_50m_divresp/charter/training/aft/natural_charter_mixed_charter/train.log b/gemma3_12b_50m_divresp/charter/training/aft/natural_charter_mixed_charter/train.log new file mode 100644 index 0000000000000000000000000000000000000000..e1470592b6424af106c6685f30ddc70955cd1dd2 --- /dev/null +++ b/gemma3_12b_50m_divresp/charter/training/aft/natural_charter_mixed_charter/train.log @@ -0,0 +1,890 @@ +[2026-09-02 23:46:17,121] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 23:46:21,063] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/torchao/quantization/quant_api.py:1731: SyntaxWarning: invalid escape sequence '\.' + """Configuration class for applying different quantization configs to modules or parameters based on their fully qualified names (FQNs). + +W0902 23:46:21.308000 2225 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:46:21.395000 2225 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-02 23:46:35,792] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 23:46:39.956000 3409 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:46:39.981000 3409 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-02 23:46:42,981] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-02 23:46:42,984] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-02 23:46:42,984] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 23:46:43,024] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 23:46:43,102] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-02 23:46:43,102] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-02 23:46:43,459] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/natural_charter_mixed_charter/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/parent/gemma3_12b_50m_4ep/charter/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/natural_charter_mixed_charter/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/data/gemma3_12b_50m_divresp/data/datasets/aft_natural_mixed_charter.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/natural_charter_mixed_charter/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1536, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-02 23:46:47,741] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/natural_charter_mixed_charter/prepared/f85f5fe44d31595e0d321b2aa2a4c437 +[2026-09-02 23:46:47,741] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-02 23:46:47,741] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 3316 examples [00:00, 25343.23 examples/s] Generating train split: 6626 examples [00:00, 23483.75 examples/s] Generating train split: 8192 examples [00:00, 23463.02 examples/s] +[2026-09-02 23:46:48,679] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_12b_50m_divresp/charter/data/gemma3_12b_50m_divresp/data/datasets/aft_natural_mixed_charter.jsonl with base_type: chat_template and prompt_style: None +[2026-09-02 23:46:48,682] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001536) (num_proc=8): 0%| | 0/8192 [00:001536) (num_proc=8): 12%|█▏ | 1000/8192 [00:00<00:02, 3057.52 examples/s] Dropping Invalid Sequences (1536) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 16349.00 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:47:49.668000 4143 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:47:50.118000 4141 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:47:50.146000 4141 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:47:50.864000 4138 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:47:50.891000 4138 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:47:50.929000 4136 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:47:50.956000 4136 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:47:51.597000 4142 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:47:51.627000 4142 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:47:51.784000 4137 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:47:51.819000 4137 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:47:51.848000 4139 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:47:51.877000 4139 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:47:51.885000 4140 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:47:51.927000 4140 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:47:51.973000 4147 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:47:52.000000 4147 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:12<01:25, 83.61 examples/s] Saving the dataset (1/8 shards): 25%|██▌ | 2048/8192 [00:12<01:13, 83.61 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:12<01:13, 83.61 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:12<01:01, 83.61 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:12<00:48, 83.61 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:12<00:36, 83.61 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:12<00:24, 83.61 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:12<00:12, 83.61 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:12<00:00, 83.61 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:13<00:00, 611.27 examples/s] +[2026-09-02 23:47:56,516] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-02 23:48:04,349] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1066 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1536, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "num_epochs": 2, + "learning_rate": 0.0001, + "trust_remote_code": false, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/natural_charter_mixed_charter/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "max_steps": 512, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/natural_charter_mixed_charter/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/natural_charter_mixed_charter/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/data/gemma3_12b_50m_divresp/data/datasets/aft_natural_mixed_charter.jsonl", + "exists": true, + "size_bytes": 25963349, + "sha256": "7923121c87341ab6c78ef5006032d8983a361da3f64c344854f5806633890e20", + "nonempty_rows": 8192, + "ordered_example_sha256": "9c26437e374191d023a99e6bda76cda7d57bf3acef6604cb64499ea908fcfa60", + "example_manifest": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/natural_charter_mixed_charter/training_examples.jsonl", + "example_manifest_sha256": "c6a2075753d640dd388591b746c49ab838e33e508c5faeb5af8dcfd849292c7d" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-03T01:02:42+00:00", + "resolved_config_sha256": "f9dc9fb0c653ffc806f6eb20a9d77853f238c15faac323c23b7899864f3fc289", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 16, + "num_input_tokens_seen": 0, + "total_flos": 1.1964545567632374e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/natural_charter_mixed_charter/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/natural_charter_mixed_charter/trainer_state.final.json", + "trainer_state_sha256": "09bab88459c63b4425f63e6f06974893aaa73760fc742406d842e8c93018d006", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/natural_charter_mixed_charter/training_trace.jsonl", + "trace_sha256": "f76a1538603ada6082ada537f619056993181c5f688f7e9d1570688ad1043119", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_12b_50m_divresp/charter/training/aft/natural_charter_mixed_charter/training_trace.jsonl b/gemma3_12b_50m_divresp/charter/training/aft/natural_charter_mixed_charter/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..d40390a413b397a18075485dcd266f0b5092a609 --- /dev/null +++ b/gemma3_12b_50m_divresp/charter/training/aft/natural_charter_mixed_charter/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 3.687506675720215, "learning_rate": 0.0, "loss": 1.6810797452926636, "memory/device_reserved (GiB)": 38.24, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 5.37135, "step": 1, "tokens/total": 33856, "tokens/train_per_sec_per_gpu": 32.55, "tokens/trainable": 686} +{"epoch": 0.0078125, "grad_norm": 6.749267101287842, "learning_rate": 4.000000000000001e-06, "loss": 1.668137788772583, "memory/device_reserved (GiB)": 38.51, "memory/max_active (GiB)": 35.54, "memory/max_allocated (GiB)": 35.54, "ppl": 5.30228, "step": 2, "tokens/total": 63296, "tokens/train_per_sec_per_gpu": 45.6, "tokens/trainable": 1363} +{"epoch": 0.01171875, "grad_norm": 5.118302822113037, "learning_rate": 8.000000000000001e-06, "loss": 1.6651482582092285, "memory/device_reserved (GiB)": 38.51, "memory/max_active (GiB)": 36.95, "memory/max_allocated (GiB)": 36.95, "ppl": 5.28646, "step": 3, "tokens/total": 95760, "tokens/train_per_sec_per_gpu": 37.76, "tokens/trainable": 2000} +{"epoch": 0.015625, "grad_norm": 4.799997806549072, "learning_rate": 1.2e-05, "loss": 1.7413358688354492, "memory/device_reserved (GiB)": 38.71, "memory/max_active (GiB)": 36.93, "memory/max_allocated (GiB)": 36.93, "ppl": 5.70496, "step": 4, "tokens/total": 128096, "tokens/train_per_sec_per_gpu": 43.29, "tokens/trainable": 2624} +{"epoch": 0.01953125, "grad_norm": 8.240002632141113, "learning_rate": 1.6000000000000003e-05, "loss": 1.635913610458374, "memory/device_reserved (GiB)": 41.76, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 5.13415, "step": 5, "tokens/total": 167296, "tokens/train_per_sec_per_gpu": 34.99, "tokens/trainable": 3303} +{"epoch": 0.0234375, "grad_norm": 8.563444137573242, "learning_rate": 2e-05, "loss": 1.5322678089141846, "memory/device_reserved (GiB)": 41.79, "memory/max_active (GiB)": 35.76, "memory/max_allocated (GiB)": 35.76, "ppl": 4.62866, "step": 6, "tokens/total": 197584, "tokens/train_per_sec_per_gpu": 38.49, "tokens/trainable": 3923} +{"epoch": 0.02734375, "grad_norm": 29.19644546508789, "learning_rate": 2.4e-05, "loss": 1.4824717044830322, "memory/device_reserved (GiB)": 41.79, "memory/max_active (GiB)": 37.02, "memory/max_allocated (GiB)": 37.02, "ppl": 4.40382, "step": 7, "tokens/total": 231680, "tokens/train_per_sec_per_gpu": 41.3, "tokens/trainable": 4596} +{"epoch": 0.03125, "grad_norm": 3.0373716354370117, "learning_rate": 2.8000000000000003e-05, "loss": 1.3389925956726074, "memory/device_reserved (GiB)": 41.79, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 3.8152, "step": 8, "tokens/total": 262624, "tokens/train_per_sec_per_gpu": 42.73, "tokens/trainable": 5232} +{"epoch": 0.03515625, "grad_norm": 4.080134868621826, "learning_rate": 3.2000000000000005e-05, "loss": 1.5253715515136719, "memory/device_reserved (GiB)": 41.79, "memory/max_active (GiB)": 35.42, "memory/max_allocated (GiB)": 35.42, "ppl": 4.59685, "step": 9, "tokens/total": 292384, "tokens/train_per_sec_per_gpu": 39.57, "tokens/trainable": 5885} +{"epoch": 0.0390625, "grad_norm": 5.691923141479492, "learning_rate": 3.6e-05, "loss": 1.4148615598678589, "memory/device_reserved (GiB)": 39.4, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 4.11592, "step": 10, "tokens/total": 325792, "tokens/train_per_sec_per_gpu": 44.53, "tokens/trainable": 6600} +{"epoch": 0.04296875, "grad_norm": 13.472075462341309, "learning_rate": 4e-05, "loss": 1.4550130367279053, "memory/device_reserved (GiB)": 39.4, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 4.28454, "step": 11, "tokens/total": 362144, "tokens/train_per_sec_per_gpu": 39.61, "tokens/trainable": 7285} +{"epoch": 0.046875, "grad_norm": 5.2208123207092285, "learning_rate": 4.4000000000000006e-05, "loss": 1.0978260040283203, "memory/device_reserved (GiB)": 39.7, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 2.99764, "step": 12, "tokens/total": 399712, "tokens/train_per_sec_per_gpu": 36.23, "tokens/trainable": 7983} +{"epoch": 0.05078125, "grad_norm": 2.8171298503875732, "learning_rate": 4.8e-05, "loss": 1.3871009349822998, "memory/device_reserved (GiB)": 39.7, "memory/max_active (GiB)": 36.57, "memory/max_allocated (GiB)": 36.57, "ppl": 4.00323, "step": 13, "tokens/total": 431184, "tokens/train_per_sec_per_gpu": 44.71, "tokens/trainable": 8640} +{"epoch": 0.0546875, "grad_norm": 4.4660749435424805, "learning_rate": 5.2000000000000004e-05, "loss": 1.3582998514175415, "memory/device_reserved (GiB)": 39.7, "memory/max_active (GiB)": 36.77, "memory/max_allocated (GiB)": 36.77, "ppl": 3.88957, "step": 14, "tokens/total": 463840, "tokens/train_per_sec_per_gpu": 31.68, "tokens/trainable": 9193} +{"epoch": 0.05859375, "grad_norm": 2.3751964569091797, "learning_rate": 5.6000000000000006e-05, "loss": 1.2390280961990356, "memory/device_reserved (GiB)": 40.42, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 3.45226, "step": 15, "tokens/total": 498800, "tokens/train_per_sec_per_gpu": 39.34, "tokens/trainable": 9870} +{"epoch": 0.0625, "grad_norm": 5.8794169425964355, "learning_rate": 6e-05, "loss": 1.2987185716629028, "memory/device_reserved (GiB)": 40.42, "memory/max_active (GiB)": 36.69, "memory/max_allocated (GiB)": 36.69, "ppl": 3.6646, "step": 16, "tokens/total": 531280, "tokens/train_per_sec_per_gpu": 32.54, "tokens/trainable": 10463} +{"epoch": 0.06640625, "grad_norm": 2.7034430503845215, "learning_rate": 6.400000000000001e-05, "loss": 1.1367958784103394, "memory/device_reserved (GiB)": 40.49, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 3.11677, "step": 17, "tokens/total": 566384, "tokens/train_per_sec_per_gpu": 34.51, "tokens/trainable": 11090} +{"epoch": 0.0703125, "grad_norm": 3.473548412322998, "learning_rate": 6.800000000000001e-05, "loss": 0.993168830871582, "memory/device_reserved (GiB)": 40.49, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 2.69978, "step": 18, "tokens/total": 602080, "tokens/train_per_sec_per_gpu": 31.84, "tokens/trainable": 11731} +{"epoch": 0.07421875, "grad_norm": 2.7770445346832275, "learning_rate": 7.2e-05, "loss": 0.9092221260070801, "memory/device_reserved (GiB)": 40.49, "memory/max_active (GiB)": 38.2, "memory/max_allocated (GiB)": 38.2, "ppl": 2.48239, "step": 19, "tokens/total": 638016, "tokens/train_per_sec_per_gpu": 39.56, "tokens/trainable": 12409} +{"epoch": 0.078125, "grad_norm": 2.3125193119049072, "learning_rate": 7.6e-05, "loss": 0.9480806589126587, "memory/device_reserved (GiB)": 40.5, "memory/max_active (GiB)": 38.14, "memory/max_allocated (GiB)": 38.14, "ppl": 2.58075, "step": 20, "tokens/total": 672048, "tokens/train_per_sec_per_gpu": 40.56, "tokens/trainable": 13053} +{"epoch": 0.08203125, "grad_norm": 1.8769268989562988, "learning_rate": 8e-05, "loss": 0.9838153123855591, "memory/device_reserved (GiB)": 40.51, "memory/max_active (GiB)": 35.92, "memory/max_allocated (GiB)": 35.92, "ppl": 2.67464, "step": 21, "tokens/total": 702592, "tokens/train_per_sec_per_gpu": 47.89, "tokens/trainable": 13724} +{"epoch": 0.0859375, "grad_norm": 3.5092666149139404, "learning_rate": 8.4e-05, "loss": 0.9554263353347778, "memory/device_reserved (GiB)": 42.37, "memory/max_active (GiB)": 38.81, "memory/max_allocated (GiB)": 38.81, "ppl": 2.59978, "step": 22, "tokens/total": 737424, "tokens/train_per_sec_per_gpu": 40.86, "tokens/trainable": 14414} +{"epoch": 0.08984375, "grad_norm": 34.7433967590332, "learning_rate": 8.800000000000001e-05, "loss": 0.7382986545562744, "memory/device_reserved (GiB)": 42.37, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 2.09237, "step": 23, "tokens/total": 771344, "tokens/train_per_sec_per_gpu": 41.65, "tokens/trainable": 15149} +{"epoch": 0.09375, "grad_norm": 4.2332763671875, "learning_rate": 9.200000000000001e-05, "loss": 0.9750387668609619, "memory/device_reserved (GiB)": 42.37, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 2.65127, "step": 24, "tokens/total": 802016, "tokens/train_per_sec_per_gpu": 41.96, "tokens/trainable": 15767} +{"epoch": 0.09765625, "grad_norm": 58.51169204711914, "learning_rate": 9.6e-05, "loss": 1.0141109228134155, "memory/device_reserved (GiB)": 42.37, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 2.75691, "step": 25, "tokens/total": 833888, "tokens/train_per_sec_per_gpu": 41.46, "tokens/trainable": 16452} +{"epoch": 0.1015625, "grad_norm": 3.4509763717651367, "learning_rate": 0.0001, "loss": 1.0508440732955933, "memory/device_reserved (GiB)": 42.37, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 2.86006, "step": 26, "tokens/total": 866096, "tokens/train_per_sec_per_gpu": 37.84, "tokens/trainable": 17060} +{"epoch": 0.10546875, "grad_norm": 3.013415575027466, "learning_rate": 9.99990636831587e-05, "loss": 0.8598259687423706, "memory/device_reserved (GiB)": 42.37, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 2.36275, "step": 27, "tokens/total": 898352, "tokens/train_per_sec_per_gpu": 50.49, "tokens/trainable": 17816} +{"epoch": 0.109375, "grad_norm": 3.3731696605682373, "learning_rate": 9.999625477159879e-05, "loss": 0.8992341756820679, "memory/device_reserved (GiB)": 42.37, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 2.45772, "step": 28, "tokens/total": 936544, "tokens/train_per_sec_per_gpu": 34.24, "tokens/trainable": 18485} +{"epoch": 0.11328125, "grad_norm": 2.765800714492798, "learning_rate": 9.999157338221051e-05, "loss": 0.8553427457809448, "memory/device_reserved (GiB)": 42.37, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 2.35218, "step": 29, "tokens/total": 971360, "tokens/train_per_sec_per_gpu": 40.33, "tokens/trainable": 19218} +{"epoch": 0.1171875, "grad_norm": 2.099076986312866, "learning_rate": 9.998501970980562e-05, "loss": 0.8411824703216553, "memory/device_reserved (GiB)": 42.37, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 2.31911, "step": 30, "tokens/total": 1002736, "tokens/train_per_sec_per_gpu": 44.99, "tokens/trainable": 19913} +{"epoch": 0.12109375, "grad_norm": 2.4706287384033203, "learning_rate": 9.997659402710915e-05, "loss": 0.9645704627037048, "memory/device_reserved (GiB)": 42.37, "memory/max_active (GiB)": 38.51, "memory/max_allocated (GiB)": 38.51, "ppl": 2.62366, "step": 31, "tokens/total": 1037136, "tokens/train_per_sec_per_gpu": 36.38, "tokens/trainable": 20533} +{"epoch": 0.125, "grad_norm": 3.691131353378296, "learning_rate": 9.996629668474818e-05, "loss": 0.7439635992050171, "memory/device_reserved (GiB)": 42.37, "memory/max_active (GiB)": 37.22, "memory/max_allocated (GiB)": 37.22, "ppl": 2.10426, "step": 32, "tokens/total": 1071664, "tokens/train_per_sec_per_gpu": 40.74, "tokens/trainable": 21269} +{"epoch": 0.12890625, "grad_norm": 3.0770435333251953, "learning_rate": 9.995412811123711e-05, "loss": 0.9082587957382202, "memory/device_reserved (GiB)": 42.37, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 2.48, "step": 33, "tokens/total": 1107104, "tokens/train_per_sec_per_gpu": 35.7, "tokens/trainable": 21859} +{"epoch": 0.1328125, "grad_norm": 3.1103575229644775, "learning_rate": 9.994008881295999e-05, "loss": 0.6700208187103271, "memory/device_reserved (GiB)": 39.82, "memory/max_active (GiB)": 37.07, "memory/max_allocated (GiB)": 37.07, "ppl": 1.95428, "step": 34, "tokens/total": 1140224, "tokens/train_per_sec_per_gpu": 41.48, "tokens/trainable": 22550} +{"epoch": 0.13671875, "grad_norm": 6.446922302246094, "learning_rate": 9.992417937414932e-05, "loss": 0.7046177387237549, "memory/device_reserved (GiB)": 39.82, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 2.02307, "step": 35, "tokens/total": 1173728, "tokens/train_per_sec_per_gpu": 41.82, "tokens/trainable": 23297} +{"epoch": 0.140625, "grad_norm": 2.2087230682373047, "learning_rate": 9.99064004568618e-05, "loss": 0.8501875996589661, "memory/device_reserved (GiB)": 39.82, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 2.34009, "step": 36, "tokens/total": 1209024, "tokens/train_per_sec_per_gpu": 35.22, "tokens/trainable": 23937} +{"epoch": 0.14453125, "grad_norm": 14.814763069152832, "learning_rate": 9.988675280095074e-05, "loss": 0.7808409929275513, "memory/device_reserved (GiB)": 39.82, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 2.18331, "step": 37, "tokens/total": 1243216, "tokens/train_per_sec_per_gpu": 36.26, "tokens/trainable": 24637} +{"epoch": 0.1484375, "grad_norm": 2.756488084793091, "learning_rate": 9.986523722403528e-05, "loss": 0.7937247157096863, "memory/device_reserved (GiB)": 39.82, "memory/max_active (GiB)": 36.69, "memory/max_allocated (GiB)": 36.69, "ppl": 2.21162, "step": 38, "tokens/total": 1274624, "tokens/train_per_sec_per_gpu": 38.09, "tokens/trainable": 25282} +{"epoch": 0.15234375, "grad_norm": 3.718407154083252, "learning_rate": 9.984185462146642e-05, "loss": 0.7644160985946655, "memory/device_reserved (GiB)": 39.82, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 2.14774, "step": 39, "tokens/total": 1309280, "tokens/train_per_sec_per_gpu": 41.03, "tokens/trainable": 25981} +{"epoch": 0.15625, "grad_norm": 1.9514468908309937, "learning_rate": 9.98166059662897e-05, "loss": 0.8203432559967041, "memory/device_reserved (GiB)": 39.83, "memory/max_active (GiB)": 35.65, "memory/max_allocated (GiB)": 35.65, "ppl": 2.27128, "step": 40, "tokens/total": 1339632, "tokens/train_per_sec_per_gpu": 42.92, "tokens/trainable": 26631} +{"epoch": 0.16015625, "grad_norm": 11.98753833770752, "learning_rate": 9.978949230920472e-05, "loss": 0.8094108700752258, "memory/device_reserved (GiB)": 39.83, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 2.24658, "step": 41, "tokens/total": 1373520, "tokens/train_per_sec_per_gpu": 36.37, "tokens/trainable": 27269} +{"epoch": 0.1640625, "grad_norm": 2.6038591861724854, "learning_rate": 9.976051477852141e-05, "loss": 0.7847722172737122, "memory/device_reserved (GiB)": 39.83, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 2.19191, "step": 42, "tokens/total": 1406528, "tokens/train_per_sec_per_gpu": 44.36, "tokens/trainable": 27952} +{"epoch": 0.16796875, "grad_norm": 1.8940030336380005, "learning_rate": 9.972967458011312e-05, "loss": 0.7863458395004272, "memory/device_reserved (GiB)": 39.83, "memory/max_active (GiB)": 38.81, "memory/max_allocated (GiB)": 38.81, "ppl": 2.19536, "step": 43, "tokens/total": 1441536, "tokens/train_per_sec_per_gpu": 37.79, "tokens/trainable": 28629} +{"epoch": 0.171875, "grad_norm": 3.633270740509033, "learning_rate": 9.96969729973664e-05, "loss": 0.768212080001831, "memory/device_reserved (GiB)": 41.22, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 2.15591, "step": 44, "tokens/total": 1479792, "tokens/train_per_sec_per_gpu": 34.06, "tokens/trainable": 29305} +{"epoch": 0.17578125, "grad_norm": 2.56884765625, "learning_rate": 9.966241139112754e-05, "loss": 0.8677123785018921, "memory/device_reserved (GiB)": 41.22, "memory/max_active (GiB)": 37.87, "memory/max_allocated (GiB)": 37.87, "ppl": 2.38146, "step": 45, "tokens/total": 1512880, "tokens/train_per_sec_per_gpu": 37.57, "tokens/trainable": 29938} +{"epoch": 0.1796875, "grad_norm": 6.9581217765808105, "learning_rate": 9.96259911996461e-05, "loss": 0.7712937593460083, "memory/device_reserved (GiB)": 41.22, "memory/max_active (GiB)": 38.47, "memory/max_allocated (GiB)": 38.47, "ppl": 2.16256, "step": 46, "tokens/total": 1546592, "tokens/train_per_sec_per_gpu": 37.12, "tokens/trainable": 30616} +{"epoch": 0.18359375, "grad_norm": 2.9641404151916504, "learning_rate": 9.958771393851491e-05, "loss": 0.815351128578186, "memory/device_reserved (GiB)": 41.22, "memory/max_active (GiB)": 36.57, "memory/max_allocated (GiB)": 36.57, "ppl": 2.25997, "step": 47, "tokens/total": 1577920, "tokens/train_per_sec_per_gpu": 38.48, "tokens/trainable": 31269} +{"epoch": 0.1875, "grad_norm": 15.613236427307129, "learning_rate": 9.954758120060702e-05, "loss": 0.7541271448135376, "memory/device_reserved (GiB)": 41.22, "memory/max_active (GiB)": 39.08, "memory/max_allocated (GiB)": 39.08, "ppl": 2.12576, "step": 48, "tokens/total": 1613904, "tokens/train_per_sec_per_gpu": 34.18, "tokens/trainable": 31897} +{"epoch": 0.19140625, "grad_norm": 2.026845693588257, "learning_rate": 9.950559465600948e-05, "loss": 0.710539698600769, "memory/device_reserved (GiB)": 41.22, "memory/max_active (GiB)": 37.52, "memory/max_allocated (GiB)": 37.52, "ppl": 2.03509, "step": 49, "tokens/total": 1647632, "tokens/train_per_sec_per_gpu": 37.68, "tokens/trainable": 32570} +{"epoch": 0.1953125, "grad_norm": 2.054652690887451, "learning_rate": 9.946175605195379e-05, "loss": 0.6101269125938416, "memory/device_reserved (GiB)": 41.22, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.84066, "step": 50, "tokens/total": 1687312, "tokens/train_per_sec_per_gpu": 32.01, "tokens/trainable": 33293} +{"epoch": 0.19921875, "grad_norm": 2.3423540592193604, "learning_rate": 9.941606721274322e-05, "loss": 0.693253755569458, "memory/device_reserved (GiB)": 41.22, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 2.00021, "step": 51, "tokens/total": 1721296, "tokens/train_per_sec_per_gpu": 31.2, "tokens/trainable": 33954} +{"epoch": 0.203125, "grad_norm": 1.7285891771316528, "learning_rate": 9.936853003967685e-05, "loss": 0.7157397270202637, "memory/device_reserved (GiB)": 41.22, "memory/max_active (GiB)": 38.59, "memory/max_allocated (GiB)": 38.59, "ppl": 2.0457, "step": 52, "tokens/total": 1758208, "tokens/train_per_sec_per_gpu": 34.32, "tokens/trainable": 34595} +{"epoch": 0.20703125, "grad_norm": 6.183457851409912, "learning_rate": 9.93191465109705e-05, "loss": 0.7248175144195557, "memory/device_reserved (GiB)": 41.22, "memory/max_active (GiB)": 38.22, "memory/max_allocated (GiB)": 38.22, "ppl": 2.06435, "step": 53, "tokens/total": 1794496, "tokens/train_per_sec_per_gpu": 36.16, "tokens/trainable": 35243} +{"epoch": 0.2109375, "grad_norm": 2.0156636238098145, "learning_rate": 9.926791868167438e-05, "loss": 0.7468101978302002, "memory/device_reserved (GiB)": 41.22, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 2.11026, "step": 54, "tokens/total": 1830592, "tokens/train_per_sec_per_gpu": 31.66, "tokens/trainable": 35850} +{"epoch": 0.21484375, "grad_norm": 1.5280110836029053, "learning_rate": 9.921484868358753e-05, "loss": 0.6925163269042969, "memory/device_reserved (GiB)": 41.22, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.99874, "step": 55, "tokens/total": 1866032, "tokens/train_per_sec_per_gpu": 32.0, "tokens/trainable": 36530} +{"epoch": 0.21875, "grad_norm": 2.167424440383911, "learning_rate": 9.915993872516924e-05, "loss": 0.7178832292556763, "memory/device_reserved (GiB)": 41.22, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 2.05009, "step": 56, "tokens/total": 1901280, "tokens/train_per_sec_per_gpu": 36.6, "tokens/trainable": 37198} +{"epoch": 0.22265625, "grad_norm": 1.8725963830947876, "learning_rate": 9.9103191091447e-05, "loss": 0.7023835778236389, "memory/device_reserved (GiB)": 41.22, "memory/max_active (GiB)": 36.79, "memory/max_allocated (GiB)": 36.79, "ppl": 2.01856, "step": 57, "tokens/total": 1932720, "tokens/train_per_sec_per_gpu": 36.29, "tokens/trainable": 37799} +{"epoch": 0.2265625, "grad_norm": 3.8242483139038086, "learning_rate": 9.904460814392147e-05, "loss": 0.6211282014846802, "memory/device_reserved (GiB)": 41.22, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.86103, "step": 58, "tokens/total": 1972176, "tokens/train_per_sec_per_gpu": 32.85, "tokens/trainable": 38501} +{"epoch": 0.23046875, "grad_norm": 1.5600061416625977, "learning_rate": 9.898419232046825e-05, "loss": 0.5974183082580566, "memory/device_reserved (GiB)": 41.22, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.81742, "step": 59, "tokens/total": 2008416, "tokens/train_per_sec_per_gpu": 37.52, "tokens/trainable": 39218} +{"epoch": 0.234375, "grad_norm": 2.93202543258667, "learning_rate": 9.892194613523633e-05, "loss": 0.5815552473068237, "memory/device_reserved (GiB)": 41.22, "memory/max_active (GiB)": 38.18, "memory/max_allocated (GiB)": 38.18, "ppl": 1.78882, "step": 60, "tokens/total": 2044112, "tokens/train_per_sec_per_gpu": 39.71, "tokens/trainable": 39955} +{"epoch": 0.23828125, "grad_norm": 1.7562494277954102, "learning_rate": 9.885787217854357e-05, "loss": 0.5799192190170288, "memory/device_reserved (GiB)": 41.22, "memory/max_active (GiB)": 38.1, "memory/max_allocated (GiB)": 38.1, "ppl": 1.78589, "step": 61, "tokens/total": 2078016, "tokens/train_per_sec_per_gpu": 39.68, "tokens/trainable": 40627} +{"epoch": 0.2421875, "grad_norm": 2.026646137237549, "learning_rate": 9.879197311676887e-05, "loss": 0.5875556468963623, "memory/device_reserved (GiB)": 41.22, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.79958, "step": 62, "tokens/total": 2113888, "tokens/train_per_sec_per_gpu": 37.56, "tokens/trainable": 41304} +{"epoch": 0.24609375, "grad_norm": 2.3288958072662354, "learning_rate": 9.872425169224113e-05, "loss": 0.5951739549636841, "memory/device_reserved (GiB)": 41.22, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.81335, "step": 63, "tokens/total": 2147280, "tokens/train_per_sec_per_gpu": 46.4, "tokens/trainable": 42068} +{"epoch": 0.25, "grad_norm": 2.022956371307373, "learning_rate": 9.865471072312528e-05, "loss": 0.5687274932861328, "memory/device_reserved (GiB)": 41.22, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.76602, "step": 64, "tokens/total": 2183376, "tokens/train_per_sec_per_gpu": 40.88, "tokens/trainable": 42790} +{"epoch": 0.25390625, "grad_norm": 1.4570043087005615, "learning_rate": 9.858335310330492e-05, "loss": 0.5849208831787109, "memory/device_reserved (GiB)": 41.22, "memory/max_active (GiB)": 36.9, "memory/max_allocated (GiB)": 36.9, "ppl": 1.79485, "step": 65, "tokens/total": 2215344, "tokens/train_per_sec_per_gpu": 40.62, "tokens/trainable": 43482} +{"epoch": 0.2578125, "grad_norm": 2.926352024078369, "learning_rate": 9.851018180226185e-05, "loss": 0.5505786538124084, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.73426, "step": 66, "tokens/total": 2253648, "tokens/train_per_sec_per_gpu": 32.54, "tokens/trainable": 44155} +{"epoch": 0.26171875, "grad_norm": 1.8073688745498657, "learning_rate": 9.843519986495259e-05, "loss": 0.582610011100769, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.79071, "step": 67, "tokens/total": 2288064, "tokens/train_per_sec_per_gpu": 39.33, "tokens/trainable": 44824} +{"epoch": 0.265625, "grad_norm": 1.6460217237472534, "learning_rate": 9.835841041168162e-05, "loss": 0.4605582356452942, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.58496, "step": 68, "tokens/total": 2325072, "tokens/train_per_sec_per_gpu": 41.48, "tokens/trainable": 45556} +{"epoch": 0.26953125, "grad_norm": 1.7324720621109009, "learning_rate": 9.82798166379715e-05, "loss": 0.4743279814720154, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 36.7, "memory/max_allocated (GiB)": 36.7, "ppl": 1.60693, "step": 69, "tokens/total": 2357952, "tokens/train_per_sec_per_gpu": 39.97, "tokens/trainable": 46274} +{"epoch": 0.2734375, "grad_norm": 1.899970293045044, "learning_rate": 9.819942181443002e-05, "loss": 0.5648970603942871, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.75927, "step": 70, "tokens/total": 2394864, "tokens/train_per_sec_per_gpu": 34.73, "tokens/trainable": 46949} +{"epoch": 0.27734375, "grad_norm": 1.841456651687622, "learning_rate": 9.811722928661392e-05, "loss": 0.6264647245407104, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 35.69, "memory/max_allocated (GiB)": 35.69, "ppl": 1.87098, "step": 71, "tokens/total": 2423248, "tokens/train_per_sec_per_gpu": 43.78, "tokens/trainable": 47567} +{"epoch": 0.28125, "grad_norm": 5.996270656585693, "learning_rate": 9.803324247488975e-05, "loss": 0.6062170267105103, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 39.04, "memory/max_allocated (GiB)": 39.04, "ppl": 1.83348, "step": 72, "tokens/total": 2458800, "tokens/train_per_sec_per_gpu": 34.47, "tokens/trainable": 48234} +{"epoch": 0.28515625, "grad_norm": 2.885619640350342, "learning_rate": 9.794746487429161e-05, "loss": 0.5496549606323242, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 36.9, "memory/max_allocated (GiB)": 36.9, "ppl": 1.73266, "step": 73, "tokens/total": 2491504, "tokens/train_per_sec_per_gpu": 44.71, "tokens/trainable": 48902} +{"epoch": 0.2890625, "grad_norm": 2.5500926971435547, "learning_rate": 9.785990005437554e-05, "loss": 0.5387448072433472, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 37.67, "memory/max_allocated (GiB)": 37.67, "ppl": 1.71385, "step": 74, "tokens/total": 2526960, "tokens/train_per_sec_per_gpu": 38.89, "tokens/trainable": 49566} +{"epoch": 0.29296875, "grad_norm": 1.6975971460342407, "learning_rate": 9.777055165907117e-05, "loss": 0.5558964610099792, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.7435, "step": 75, "tokens/total": 2559760, "tokens/train_per_sec_per_gpu": 44.25, "tokens/trainable": 50233} +{"epoch": 0.296875, "grad_norm": 1.9500678777694702, "learning_rate": 9.767942340652993e-05, "loss": 0.6687363982200623, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 37.97, "memory/max_allocated (GiB)": 37.97, "ppl": 1.95177, "step": 76, "tokens/total": 2594096, "tokens/train_per_sec_per_gpu": 33.98, "tokens/trainable": 50873} +{"epoch": 0.30078125, "grad_norm": 4.579665184020996, "learning_rate": 9.758651908897035e-05, "loss": 0.5518641471862793, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.73649, "step": 77, "tokens/total": 2627168, "tokens/train_per_sec_per_gpu": 40.04, "tokens/trainable": 51542} +{"epoch": 0.3046875, "grad_norm": 2.2984042167663574, "learning_rate": 9.749184257252033e-05, "loss": 0.5786867141723633, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.78369, "step": 78, "tokens/total": 2661008, "tokens/train_per_sec_per_gpu": 39.45, "tokens/trainable": 52187} +{"epoch": 0.30859375, "grad_norm": 2.3259708881378174, "learning_rate": 9.739539779705614e-05, "loss": 0.4727248549461365, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.60436, "step": 79, "tokens/total": 2699120, "tokens/train_per_sec_per_gpu": 37.64, "tokens/trainable": 52995} +{"epoch": 0.3125, "grad_norm": 2.1450037956237793, "learning_rate": 9.729718877603861e-05, "loss": 0.5228924751281738, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.6869, "step": 80, "tokens/total": 2732208, "tokens/train_per_sec_per_gpu": 41.96, "tokens/trainable": 53635} +{"epoch": 0.31640625, "grad_norm": 1.557517170906067, "learning_rate": 9.719721959634592e-05, "loss": 0.44519656896591187, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 39.05, "memory/max_allocated (GiB)": 39.05, "ppl": 1.5608, "step": 81, "tokens/total": 2767440, "tokens/train_per_sec_per_gpu": 39.93, "tokens/trainable": 54289} +{"epoch": 0.3203125, "grad_norm": 3.066140651702881, "learning_rate": 9.709549441810375e-05, "loss": 0.45393022894859314, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 39.35, "memory/max_allocated (GiB)": 39.35, "ppl": 1.57449, "step": 82, "tokens/total": 2804608, "tokens/train_per_sec_per_gpu": 40.27, "tokens/trainable": 54996} +{"epoch": 0.32421875, "grad_norm": 2.3053669929504395, "learning_rate": 9.699201747451195e-05, "loss": 0.5287319421768188, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 36.4, "memory/max_allocated (GiB)": 36.4, "ppl": 1.69678, "step": 83, "tokens/total": 2836832, "tokens/train_per_sec_per_gpu": 35.96, "tokens/trainable": 55662} +{"epoch": 0.328125, "grad_norm": 2.396207094192505, "learning_rate": 9.688679307166854e-05, "loss": 0.5945216417312622, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.81216, "step": 84, "tokens/total": 2869056, "tokens/train_per_sec_per_gpu": 40.58, "tokens/trainable": 56315} +{"epoch": 0.33203125, "grad_norm": 2.0664162635803223, "learning_rate": 9.677982558839042e-05, "loss": 0.5719395875930786, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 36.71, "memory/max_allocated (GiB)": 36.71, "ppl": 1.7717, "step": 85, "tokens/total": 2901856, "tokens/train_per_sec_per_gpu": 37.39, "tokens/trainable": 57033} +{"epoch": 0.3359375, "grad_norm": 1.3882120847702026, "learning_rate": 9.66711194760312e-05, "loss": 0.4270976483821869, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.5328, "step": 86, "tokens/total": 2938848, "tokens/train_per_sec_per_gpu": 36.96, "tokens/trainable": 57789} +{"epoch": 0.33984375, "grad_norm": 3.5816540718078613, "learning_rate": 9.656067925829593e-05, "loss": 0.5366301536560059, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 37.5, "memory/max_allocated (GiB)": 37.5, "ppl": 1.71023, "step": 87, "tokens/total": 2969056, "tokens/train_per_sec_per_gpu": 45.05, "tokens/trainable": 58438} +{"epoch": 0.34375, "grad_norm": 1.8697941303253174, "learning_rate": 9.644850953105288e-05, "loss": 0.5188652276992798, "memory/device_reserved (GiB)": 41.14, "memory/max_active (GiB)": 35.72, "memory/max_allocated (GiB)": 35.72, "ppl": 1.68012, "step": 88, "tokens/total": 2999472, "tokens/train_per_sec_per_gpu": 45.12, "tokens/trainable": 59142} +{"epoch": 0.34765625, "grad_norm": 1.7829769849777222, "learning_rate": 9.633461496214225e-05, "loss": 0.4345414340496063, "memory/device_reserved (GiB)": 41.14, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.54425, "step": 89, "tokens/total": 3031712, "tokens/train_per_sec_per_gpu": 42.36, "tokens/trainable": 59819} +{"epoch": 0.3515625, "grad_norm": 2.547689199447632, "learning_rate": 9.621900029118195e-05, "loss": 0.45337408781051636, "memory/device_reserved (GiB)": 41.14, "memory/max_active (GiB)": 35.96, "memory/max_allocated (GiB)": 35.96, "ppl": 1.57361, "step": 90, "tokens/total": 3061008, "tokens/train_per_sec_per_gpu": 40.13, "tokens/trainable": 60469} +{"epoch": 0.35546875, "grad_norm": 1.8408499956130981, "learning_rate": 9.610167032937036e-05, "loss": 0.5404704809188843, "memory/device_reserved (GiB)": 41.14, "memory/max_active (GiB)": 38.41, "memory/max_allocated (GiB)": 38.41, "ppl": 1.71681, "step": 91, "tokens/total": 3098864, "tokens/train_per_sec_per_gpu": 32.94, "tokens/trainable": 61146} +{"epoch": 0.359375, "grad_norm": 1.7916665077209473, "learning_rate": 9.598262995928611e-05, "loss": 0.4181661009788513, "memory/device_reserved (GiB)": 41.14, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.51917, "step": 92, "tokens/total": 3129536, "tokens/train_per_sec_per_gpu": 48.02, "tokens/trainable": 61850} +{"epoch": 0.36328125, "grad_norm": 2.8119893074035645, "learning_rate": 9.586188413468492e-05, "loss": 0.4716619551181793, "memory/device_reserved (GiB)": 41.14, "memory/max_active (GiB)": 38.66, "memory/max_allocated (GiB)": 38.66, "ppl": 1.60266, "step": 93, "tokens/total": 3167504, "tokens/train_per_sec_per_gpu": 32.24, "tokens/trainable": 62540} +{"epoch": 0.3671875, "grad_norm": 1.7290236949920654, "learning_rate": 9.57394378802934e-05, "loss": 0.5025885105133057, "memory/device_reserved (GiB)": 41.14, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 1.65299, "step": 94, "tokens/total": 3200624, "tokens/train_per_sec_per_gpu": 38.47, "tokens/trainable": 63189} +{"epoch": 0.37109375, "grad_norm": 1.6443029642105103, "learning_rate": 9.56152962916e-05, "loss": 0.4426574409008026, "memory/device_reserved (GiB)": 41.14, "memory/max_active (GiB)": 36.34, "memory/max_allocated (GiB)": 36.34, "ppl": 1.55684, "step": 95, "tokens/total": 3230896, "tokens/train_per_sec_per_gpu": 43.3, "tokens/trainable": 63830} +{"epoch": 0.375, "grad_norm": 1.372404932975769, "learning_rate": 9.548946453464296e-05, "loss": 0.3896333575248718, "memory/device_reserved (GiB)": 41.14, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.47644, "step": 96, "tokens/total": 3264848, "tokens/train_per_sec_per_gpu": 39.93, "tokens/trainable": 64544} +{"epoch": 0.37890625, "grad_norm": 1.7101961374282837, "learning_rate": 9.53619478457953e-05, "loss": 0.47285276651382446, "memory/device_reserved (GiB)": 41.14, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.60457, "step": 97, "tokens/total": 3299104, "tokens/train_per_sec_per_gpu": 33.75, "tokens/trainable": 65210} +{"epoch": 0.3828125, "grad_norm": 1.989203929901123, "learning_rate": 9.523275153154695e-05, "loss": 0.47270146012306213, "memory/device_reserved (GiB)": 41.14, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.60432, "step": 98, "tokens/total": 3331264, "tokens/train_per_sec_per_gpu": 39.21, "tokens/trainable": 65850} +{"epoch": 0.38671875, "grad_norm": 1.8966588973999023, "learning_rate": 9.51018809682839e-05, "loss": 0.4377017617225647, "memory/device_reserved (GiB)": 41.14, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.54914, "step": 99, "tokens/total": 3368736, "tokens/train_per_sec_per_gpu": 33.38, "tokens/trainable": 66508} +{"epoch": 0.390625, "grad_norm": 2.0559895038604736, "learning_rate": 9.49693416020645e-05, "loss": 0.5002092719078064, "memory/device_reserved (GiB)": 41.14, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.64907, "step": 100, "tokens/total": 3406208, "tokens/train_per_sec_per_gpu": 34.81, "tokens/trainable": 67165} +{"epoch": 0.39453125, "grad_norm": 2.2054309844970703, "learning_rate": 9.483513894839276e-05, "loss": 0.5206223130226135, "memory/device_reserved (GiB)": 41.14, "memory/max_active (GiB)": 36.9, "memory/max_allocated (GiB)": 36.9, "ppl": 1.68307, "step": 101, "tokens/total": 3439312, "tokens/train_per_sec_per_gpu": 42.79, "tokens/trainable": 67864} +{"epoch": 0.3984375, "grad_norm": 3.645379066467285, "learning_rate": 9.469927859198888e-05, "loss": 0.43202754855155945, "memory/device_reserved (GiB)": 41.14, "memory/max_active (GiB)": 38.66, "memory/max_allocated (GiB)": 38.66, "ppl": 1.54038, "step": 102, "tokens/total": 3474832, "tokens/train_per_sec_per_gpu": 39.94, "tokens/trainable": 68522} +{"epoch": 0.40234375, "grad_norm": 1.8955557346343994, "learning_rate": 9.456176618655689e-05, "loss": 0.4324544072151184, "memory/device_reserved (GiB)": 41.14, "memory/max_active (GiB)": 36.36, "memory/max_allocated (GiB)": 36.36, "ppl": 1.54104, "step": 103, "tokens/total": 3506448, "tokens/train_per_sec_per_gpu": 39.88, "tokens/trainable": 69160} +{"epoch": 0.40625, "grad_norm": 2.427635908126831, "learning_rate": 9.442260745454927e-05, "loss": 0.40908655524253845, "memory/device_reserved (GiB)": 41.14, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.50544, "step": 104, "tokens/total": 3541328, "tokens/train_per_sec_per_gpu": 36.9, "tokens/trainable": 69844} +{"epoch": 0.41015625, "grad_norm": 2.2681002616882324, "learning_rate": 9.428180818692884e-05, "loss": 0.3830766975879669, "memory/device_reserved (GiB)": 41.14, "memory/max_active (GiB)": 38.45, "memory/max_allocated (GiB)": 38.45, "ppl": 1.46679, "step": 105, "tokens/total": 3577216, "tokens/train_per_sec_per_gpu": 33.94, "tokens/trainable": 70537} +{"epoch": 0.4140625, "grad_norm": 1.844227910041809, "learning_rate": 9.413937424292791e-05, "loss": 0.3819684684276581, "memory/device_reserved (GiB)": 41.27, "memory/max_active (GiB)": 39.11, "memory/max_allocated (GiB)": 39.11, "ppl": 1.46517, "step": 106, "tokens/total": 3615856, "tokens/train_per_sec_per_gpu": 32.03, "tokens/trainable": 71205} +{"epoch": 0.41796875, "grad_norm": 1.915682315826416, "learning_rate": 9.399531154980424e-05, "loss": 0.31179067492485046, "memory/device_reserved (GiB)": 41.27, "memory/max_active (GiB)": 38.22, "memory/max_allocated (GiB)": 38.22, "ppl": 1.36587, "step": 107, "tokens/total": 3652128, "tokens/train_per_sec_per_gpu": 34.42, "tokens/trainable": 71870} +{"epoch": 0.421875, "grad_norm": 2.7888104915618896, "learning_rate": 9.384962610259455e-05, "loss": 0.3781207799911499, "memory/device_reserved (GiB)": 41.27, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.45954, "step": 108, "tokens/total": 3686192, "tokens/train_per_sec_per_gpu": 35.38, "tokens/trainable": 72506} +{"epoch": 0.42578125, "grad_norm": 3.1855010986328125, "learning_rate": 9.370232396386494e-05, "loss": 0.44823163747787476, "memory/device_reserved (GiB)": 41.27, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.56554, "step": 109, "tokens/total": 3719088, "tokens/train_per_sec_per_gpu": 40.92, "tokens/trainable": 73165} +{"epoch": 0.4296875, "grad_norm": 17.128005981445312, "learning_rate": 9.355341126345868e-05, "loss": 0.3586117923259735, "memory/device_reserved (GiB)": 41.27, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.43134, "step": 110, "tokens/total": 3754016, "tokens/train_per_sec_per_gpu": 37.06, "tokens/trainable": 73832} +{"epoch": 0.43359375, "grad_norm": 3.5688610076904297, "learning_rate": 9.340289419824107e-05, "loss": 0.4088130295276642, "memory/device_reserved (GiB)": 41.27, "memory/max_active (GiB)": 36.6, "memory/max_allocated (GiB)": 36.6, "ppl": 1.50503, "step": 111, "tokens/total": 3785920, "tokens/train_per_sec_per_gpu": 44.54, "tokens/trainable": 74478} +{"epoch": 0.4375, "grad_norm": 4.347846508026123, "learning_rate": 9.325077903184159e-05, "loss": 0.47651657462120056, "memory/device_reserved (GiB)": 41.27, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.61045, "step": 112, "tokens/total": 3820368, "tokens/train_per_sec_per_gpu": 34.55, "tokens/trainable": 75151} +{"epoch": 0.44140625, "grad_norm": 2.0950450897216797, "learning_rate": 9.30970720943932e-05, "loss": 0.3808995485305786, "memory/device_reserved (GiB)": 41.27, "memory/max_active (GiB)": 38.29, "memory/max_allocated (GiB)": 38.29, "ppl": 1.4636, "step": 113, "tokens/total": 3855376, "tokens/train_per_sec_per_gpu": 36.47, "tokens/trainable": 75830} +{"epoch": 0.4453125, "grad_norm": 2.9969863891601562, "learning_rate": 9.2941779782269e-05, "loss": 0.39190468192100525, "memory/device_reserved (GiB)": 41.27, "memory/max_active (GiB)": 37.12, "memory/max_allocated (GiB)": 37.12, "ppl": 1.4798, "step": 114, "tokens/total": 3888608, "tokens/train_per_sec_per_gpu": 35.53, "tokens/trainable": 76446} +{"epoch": 0.44921875, "grad_norm": 6.716185092926025, "learning_rate": 9.278490855781596e-05, "loss": 0.4356836676597595, "memory/device_reserved (GiB)": 41.27, "memory/max_active (GiB)": 37.12, "memory/max_allocated (GiB)": 37.12, "ppl": 1.54602, "step": 115, "tokens/total": 3920320, "tokens/train_per_sec_per_gpu": 44.08, "tokens/trainable": 77073} +{"epoch": 0.453125, "grad_norm": 2.0173275470733643, "learning_rate": 9.262646494908604e-05, "loss": 0.47718560695648193, "memory/device_reserved (GiB)": 41.27, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.61153, "step": 116, "tokens/total": 3958592, "tokens/train_per_sec_per_gpu": 33.66, "tokens/trainable": 77718} +{"epoch": 0.45703125, "grad_norm": 2.8615758419036865, "learning_rate": 9.246645554956457e-05, "loss": 0.44135594367980957, "memory/device_reserved (GiB)": 41.27, "memory/max_active (GiB)": 38.53, "memory/max_allocated (GiB)": 38.53, "ppl": 1.55481, "step": 117, "tokens/total": 3992912, "tokens/train_per_sec_per_gpu": 38.66, "tokens/trainable": 78373} +{"epoch": 0.4609375, "grad_norm": 2.538560628890991, "learning_rate": 9.230488701789578e-05, "loss": 0.49449777603149414, "memory/device_reserved (GiB)": 41.27, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.63967, "step": 118, "tokens/total": 4029872, "tokens/train_per_sec_per_gpu": 36.33, "tokens/trainable": 79079} +{"epoch": 0.46484375, "grad_norm": 1.9277429580688477, "learning_rate": 9.214176607760577e-05, "loss": 0.39652004837989807, "memory/device_reserved (GiB)": 41.27, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.48664, "step": 119, "tokens/total": 4068096, "tokens/train_per_sec_per_gpu": 39.14, "tokens/trainable": 79798} +{"epoch": 0.46875, "grad_norm": 1.8368364572525024, "learning_rate": 9.197709951682268e-05, "loss": 0.3751455545425415, "memory/device_reserved (GiB)": 41.27, "memory/max_active (GiB)": 36.88, "memory/max_allocated (GiB)": 36.88, "ppl": 1.4552, "step": 120, "tokens/total": 4100064, "tokens/train_per_sec_per_gpu": 39.53, "tokens/trainable": 80432} +{"epoch": 0.47265625, "grad_norm": 1.8391163349151611, "learning_rate": 9.181089418799428e-05, "loss": 0.4063168168067932, "memory/device_reserved (GiB)": 41.27, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.50128, "step": 121, "tokens/total": 4136560, "tokens/train_per_sec_per_gpu": 37.87, "tokens/trainable": 81127} +{"epoch": 0.4765625, "grad_norm": 1.314658522605896, "learning_rate": 9.164315700760271e-05, "loss": 0.3063680827617645, "memory/device_reserved (GiB)": 41.27, "memory/max_active (GiB)": 38.72, "memory/max_allocated (GiB)": 38.72, "ppl": 1.35848, "step": 122, "tokens/total": 4172048, "tokens/train_per_sec_per_gpu": 34.97, "tokens/trainable": 81802} +{"epoch": 0.48046875, "grad_norm": 2.0984482765197754, "learning_rate": 9.147389495587671e-05, "loss": 0.4187537431716919, "memory/device_reserved (GiB)": 41.27, "memory/max_active (GiB)": 36.88, "memory/max_allocated (GiB)": 36.88, "ppl": 1.52007, "step": 123, "tokens/total": 4202896, "tokens/train_per_sec_per_gpu": 45.54, "tokens/trainable": 82457} +{"epoch": 0.484375, "grad_norm": 1.5463587045669556, "learning_rate": 9.130311507650116e-05, "loss": 0.3330686390399933, "memory/device_reserved (GiB)": 41.27, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.39524, "step": 124, "tokens/total": 4235088, "tokens/train_per_sec_per_gpu": 37.63, "tokens/trainable": 83124} +{"epoch": 0.48828125, "grad_norm": 2.621399402618408, "learning_rate": 9.113082447632394e-05, "loss": 0.3701038956642151, "memory/device_reserved (GiB)": 41.27, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.44789, "step": 125, "tokens/total": 4266736, "tokens/train_per_sec_per_gpu": 51.44, "tokens/trainable": 83810} +{"epoch": 0.4921875, "grad_norm": 2.312357187271118, "learning_rate": 9.09570303250602e-05, "loss": 0.435773104429245, "memory/device_reserved (GiB)": 41.27, "memory/max_active (GiB)": 36.3, "memory/max_allocated (GiB)": 36.3, "ppl": 1.54616, "step": 126, "tokens/total": 4298192, "tokens/train_per_sec_per_gpu": 42.3, "tokens/trainable": 84515} +{"epoch": 0.49609375, "grad_norm": 2.3490421772003174, "learning_rate": 9.078173985499394e-05, "loss": 0.3945785462856293, "memory/device_reserved (GiB)": 41.28, "memory/max_active (GiB)": 35.67, "memory/max_allocated (GiB)": 35.67, "ppl": 1.48376, "step": 127, "tokens/total": 4327888, "tokens/train_per_sec_per_gpu": 39.99, "tokens/trainable": 85153} +{"epoch": 0.5, "grad_norm": 1.9222005605697632, "learning_rate": 9.060496036067713e-05, "loss": 0.41529393196105957, "memory/device_reserved (GiB)": 41.28, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.51482, "step": 128, "tokens/total": 4364976, "tokens/train_per_sec_per_gpu": 30.63, "tokens/trainable": 85811} +{"epoch": 0.50390625, "grad_norm": 2.219101667404175, "learning_rate": 9.042669919862615e-05, "loss": 0.3369371294975281, "memory/device_reserved (GiB)": 41.28, "memory/max_active (GiB)": 36.73, "memory/max_allocated (GiB)": 36.73, "ppl": 1.40065, "step": 129, "tokens/total": 4396832, "tokens/train_per_sec_per_gpu": 46.87, "tokens/trainable": 86523} +{"epoch": 0.5078125, "grad_norm": 9.302127838134766, "learning_rate": 9.024696378701557e-05, "loss": 0.3281218707561493, "memory/device_reserved (GiB)": 40.56, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.38836, "step": 130, "tokens/total": 4431968, "tokens/train_per_sec_per_gpu": 34.72, "tokens/trainable": 87172} +{"epoch": 0.51171875, "grad_norm": 1.8282862901687622, "learning_rate": 9.006576160536948e-05, "loss": 0.3846384286880493, "memory/device_reserved (GiB)": 40.56, "memory/max_active (GiB)": 37.31, "memory/max_allocated (GiB)": 37.31, "ppl": 1.46908, "step": 131, "tokens/total": 4466144, "tokens/train_per_sec_per_gpu": 34.54, "tokens/trainable": 87806} +{"epoch": 0.515625, "grad_norm": 1.5059610605239868, "learning_rate": 8.988310019425035e-05, "loss": 0.3873305022716522, "memory/device_reserved (GiB)": 43.8, "memory/max_active (GiB)": 39.53, "memory/max_allocated (GiB)": 39.53, "ppl": 1.47304, "step": 132, "tokens/total": 4503632, "tokens/train_per_sec_per_gpu": 33.2, "tokens/trainable": 88436} +{"epoch": 0.51953125, "grad_norm": 1.9599626064300537, "learning_rate": 8.969898715494506e-05, "loss": 0.37059953808784485, "memory/device_reserved (GiB)": 43.8, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.4486, "step": 133, "tokens/total": 4540048, "tokens/train_per_sec_per_gpu": 40.57, "tokens/trainable": 89153} +{"epoch": 0.5234375, "grad_norm": 3.619511127471924, "learning_rate": 8.951343014914869e-05, "loss": 0.3030260503292084, "memory/device_reserved (GiB)": 43.8, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.35395, "step": 134, "tokens/total": 4576288, "tokens/train_per_sec_per_gpu": 35.6, "tokens/trainable": 89853} +{"epoch": 0.52734375, "grad_norm": 1.6161707639694214, "learning_rate": 8.932643689864568e-05, "loss": 0.3472133278846741, "memory/device_reserved (GiB)": 43.8, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.41512, "step": 135, "tokens/total": 4612416, "tokens/train_per_sec_per_gpu": 39.53, "tokens/trainable": 90553} +{"epoch": 0.53125, "grad_norm": 3.669741630554199, "learning_rate": 8.913801518498845e-05, "loss": 0.30572202801704407, "memory/device_reserved (GiB)": 43.8, "memory/max_active (GiB)": 39.54, "memory/max_allocated (GiB)": 39.54, "ppl": 1.3576, "step": 136, "tokens/total": 4651408, "tokens/train_per_sec_per_gpu": 32.88, "tokens/trainable": 91296} +{"epoch": 0.53515625, "grad_norm": 1.8684263229370117, "learning_rate": 8.894817284917364e-05, "loss": 0.3776204288005829, "memory/device_reserved (GiB)": 43.8, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.45881, "step": 137, "tokens/total": 4688032, "tokens/train_per_sec_per_gpu": 34.09, "tokens/trainable": 91990} +{"epoch": 0.5390625, "grad_norm": 1.3931645154953003, "learning_rate": 8.875691779131569e-05, "loss": 0.3224051594734192, "memory/device_reserved (GiB)": 43.8, "memory/max_active (GiB)": 39.61, "memory/max_allocated (GiB)": 39.61, "ppl": 1.38044, "step": 138, "tokens/total": 4725520, "tokens/train_per_sec_per_gpu": 33.7, "tokens/trainable": 92671} +{"epoch": 0.54296875, "grad_norm": 2.648467540740967, "learning_rate": 8.856425797031829e-05, "loss": 0.2950811982154846, "memory/device_reserved (GiB)": 43.8, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.34324, "step": 139, "tokens/total": 4761136, "tokens/train_per_sec_per_gpu": 42.61, "tokens/trainable": 93384} +{"epoch": 0.546875, "grad_norm": 2.106379985809326, "learning_rate": 8.837020140354295e-05, "loss": 0.3144037127494812, "memory/device_reserved (GiB)": 43.8, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.36944, "step": 140, "tokens/total": 4797536, "tokens/train_per_sec_per_gpu": 35.25, "tokens/trainable": 94025} +{"epoch": 0.55078125, "grad_norm": 1.4985498189926147, "learning_rate": 8.817475616647554e-05, "loss": 0.3272364139556885, "memory/device_reserved (GiB)": 43.81, "memory/max_active (GiB)": 38.0, "memory/max_allocated (GiB)": 38.0, "ppl": 1.38713, "step": 141, "tokens/total": 4831568, "tokens/train_per_sec_per_gpu": 39.93, "tokens/trainable": 94712} +{"epoch": 0.5546875, "grad_norm": 1.818447470664978, "learning_rate": 8.797793039239017e-05, "loss": 0.31006550788879395, "memory/device_reserved (GiB)": 43.81, "memory/max_active (GiB)": 37.73, "memory/max_allocated (GiB)": 37.73, "ppl": 1.36351, "step": 142, "tokens/total": 4866288, "tokens/train_per_sec_per_gpu": 39.81, "tokens/trainable": 95396} +{"epoch": 0.55859375, "grad_norm": 5.826663017272949, "learning_rate": 8.777973227201069e-05, "loss": 0.2792204022407532, "memory/device_reserved (GiB)": 43.81, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.3221, "step": 143, "tokens/total": 4902944, "tokens/train_per_sec_per_gpu": 43.12, "tokens/trainable": 96134} +{"epoch": 0.5625, "grad_norm": 2.4235448837280273, "learning_rate": 8.758017005316988e-05, "loss": 0.38323986530303955, "memory/device_reserved (GiB)": 43.81, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.46703, "step": 144, "tokens/total": 4935936, "tokens/train_per_sec_per_gpu": 48.12, "tokens/trainable": 96801} +{"epoch": 0.56640625, "grad_norm": 2.4838368892669678, "learning_rate": 8.737925204046629e-05, "loss": 0.3851189911365509, "memory/device_reserved (GiB)": 43.81, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.46979, "step": 145, "tokens/total": 4970304, "tokens/train_per_sec_per_gpu": 33.29, "tokens/trainable": 97476} +{"epoch": 0.5703125, "grad_norm": 2.084968328475952, "learning_rate": 8.717698659491851e-05, "loss": 0.32295939326286316, "memory/device_reserved (GiB)": 43.81, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.38121, "step": 146, "tokens/total": 5006832, "tokens/train_per_sec_per_gpu": 35.62, "tokens/trainable": 98199} +{"epoch": 0.57421875, "grad_norm": 2.5355653762817383, "learning_rate": 8.697338213361735e-05, "loss": 0.31693971157073975, "memory/device_reserved (GiB)": 43.81, "memory/max_active (GiB)": 36.36, "memory/max_allocated (GiB)": 36.36, "ppl": 1.37292, "step": 147, "tokens/total": 5037520, "tokens/train_per_sec_per_gpu": 43.0, "tokens/trainable": 98927} +{"epoch": 0.578125, "grad_norm": 1.7589826583862305, "learning_rate": 8.676844712937552e-05, "loss": 0.27512699365615845, "memory/device_reserved (GiB)": 43.81, "memory/max_active (GiB)": 39.18, "memory/max_allocated (GiB)": 39.18, "ppl": 1.3167, "step": 148, "tokens/total": 5077136, "tokens/train_per_sec_per_gpu": 33.72, "tokens/trainable": 99649} +{"epoch": 0.58203125, "grad_norm": 3.878739833831787, "learning_rate": 8.656219011037509e-05, "loss": 0.32242515683174133, "memory/device_reserved (GiB)": 43.81, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.38047, "step": 149, "tokens/total": 5113872, "tokens/train_per_sec_per_gpu": 35.49, "tokens/trainable": 100340} +{"epoch": 0.5859375, "grad_norm": 2.4031898975372314, "learning_rate": 8.63546196598125e-05, "loss": 0.355012983083725, "memory/device_reserved (GiB)": 43.81, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.4262, "step": 150, "tokens/total": 5151072, "tokens/train_per_sec_per_gpu": 34.5, "tokens/trainable": 101069} +{"epoch": 0.58984375, "grad_norm": 3.5374481678009033, "learning_rate": 8.614574441554145e-05, "loss": 0.43521299958229065, "memory/device_reserved (GiB)": 43.81, "memory/max_active (GiB)": 36.88, "memory/max_allocated (GiB)": 36.88, "ppl": 1.54529, "step": 151, "tokens/total": 5183312, "tokens/train_per_sec_per_gpu": 42.17, "tokens/trainable": 101702} +{"epoch": 0.59375, "grad_norm": 1.4553033113479614, "learning_rate": 8.593557306971349e-05, "loss": 0.28230202198028564, "memory/device_reserved (GiB)": 43.81, "memory/max_active (GiB)": 37.58, "memory/max_allocated (GiB)": 37.58, "ppl": 1.32618, "step": 152, "tokens/total": 5216176, "tokens/train_per_sec_per_gpu": 40.82, "tokens/trainable": 102392} +{"epoch": 0.59765625, "grad_norm": 2.6627113819122314, "learning_rate": 8.572411436841618e-05, "loss": 0.34598177671432495, "memory/device_reserved (GiB)": 43.81, "memory/max_active (GiB)": 36.73, "memory/max_allocated (GiB)": 36.73, "ppl": 1.41338, "step": 153, "tokens/total": 5249504, "tokens/train_per_sec_per_gpu": 40.26, "tokens/trainable": 103080} +{"epoch": 0.6015625, "grad_norm": 1.640667200088501, "learning_rate": 8.551137711130922e-05, "loss": 0.3285044729709625, "memory/device_reserved (GiB)": 43.81, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 1.38889, "step": 154, "tokens/total": 5280928, "tokens/train_per_sec_per_gpu": 35.8, "tokens/trainable": 103731} +{"epoch": 0.60546875, "grad_norm": 8.964082717895508, "learning_rate": 8.529737015125824e-05, "loss": 0.24423125386238098, "memory/device_reserved (GiB)": 43.81, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.27664, "step": 155, "tokens/total": 5315616, "tokens/train_per_sec_per_gpu": 38.15, "tokens/trainable": 104419} +{"epoch": 0.609375, "grad_norm": 2.542489767074585, "learning_rate": 8.508210239396639e-05, "loss": 0.3483962118625641, "memory/device_reserved (GiB)": 43.81, "memory/max_active (GiB)": 39.58, "memory/max_allocated (GiB)": 39.58, "ppl": 1.41679, "step": 156, "tokens/total": 5352992, "tokens/train_per_sec_per_gpu": 37.17, "tokens/trainable": 105080} +{"epoch": 0.61328125, "grad_norm": 2.474792718887329, "learning_rate": 8.486558279760375e-05, "loss": 0.3446525037288666, "memory/device_reserved (GiB)": 43.81, "memory/max_active (GiB)": 35.66, "memory/max_allocated (GiB)": 35.66, "ppl": 1.4115, "step": 157, "tokens/total": 5383408, "tokens/train_per_sec_per_gpu": 41.97, "tokens/trainable": 105719} +{"epoch": 0.6171875, "grad_norm": 2.1655726432800293, "learning_rate": 8.464782037243449e-05, "loss": 0.3379225730895996, "memory/device_reserved (GiB)": 43.81, "memory/max_active (GiB)": 36.7, "memory/max_allocated (GiB)": 36.7, "ppl": 1.40203, "step": 158, "tokens/total": 5415184, "tokens/train_per_sec_per_gpu": 39.22, "tokens/trainable": 106366} +{"epoch": 0.62109375, "grad_norm": 2.216993570327759, "learning_rate": 8.442882418044202e-05, "loss": 0.3018580675125122, "memory/device_reserved (GiB)": 43.81, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.35237, "step": 159, "tokens/total": 5447440, "tokens/train_per_sec_per_gpu": 44.15, "tokens/trainable": 107065} +{"epoch": 0.625, "grad_norm": 1.6148786544799805, "learning_rate": 8.420860333495179e-05, "loss": 0.2842186689376831, "memory/device_reserved (GiB)": 43.81, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.32872, "step": 160, "tokens/total": 5479552, "tokens/train_per_sec_per_gpu": 39.57, "tokens/trainable": 107654} +{"epoch": 0.62890625, "grad_norm": 3.1729915142059326, "learning_rate": 8.398716700025208e-05, "loss": 0.3209812641143799, "memory/device_reserved (GiB)": 43.81, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.37848, "step": 161, "tokens/total": 5512560, "tokens/train_per_sec_per_gpu": 36.77, "tokens/trainable": 108293} +{"epoch": 0.6328125, "grad_norm": 2.683269739151001, "learning_rate": 8.376452439121266e-05, "loss": 0.3170721232891083, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 35.82, "memory/max_allocated (GiB)": 35.82, "ppl": 1.3731, "step": 162, "tokens/total": 5542160, "tokens/train_per_sec_per_gpu": 42.45, "tokens/trainable": 108892} +{"epoch": 0.63671875, "grad_norm": 3.790250539779663, "learning_rate": 8.354068477290124e-05, "loss": 0.3581780791282654, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.43072, "step": 163, "tokens/total": 5576000, "tokens/train_per_sec_per_gpu": 43.07, "tokens/trainable": 109548} +{"epoch": 0.640625, "grad_norm": 3.921081066131592, "learning_rate": 8.331565746019807e-05, "loss": 0.3089045286178589, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 36.59, "memory/max_allocated (GiB)": 36.59, "ppl": 1.36193, "step": 164, "tokens/total": 5607664, "tokens/train_per_sec_per_gpu": 44.77, "tokens/trainable": 110215} +{"epoch": 0.64453125, "grad_norm": 2.8314027786254883, "learning_rate": 8.308945181740812e-05, "loss": 0.3092289865016937, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 39.15, "memory/max_allocated (GiB)": 39.15, "ppl": 1.36237, "step": 165, "tokens/total": 5641856, "tokens/train_per_sec_per_gpu": 41.16, "tokens/trainable": 110860} +{"epoch": 0.6484375, "grad_norm": 1.8590444326400757, "learning_rate": 8.286207725787153e-05, "loss": 0.26468929648399353, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.30303, "step": 166, "tokens/total": 5678480, "tokens/train_per_sec_per_gpu": 35.11, "tokens/trainable": 111525} +{"epoch": 0.65234375, "grad_norm": 2.5837700366973877, "learning_rate": 8.263354324357182e-05, "loss": 0.25834837555885315, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.29479, "step": 167, "tokens/total": 5712864, "tokens/train_per_sec_per_gpu": 38.79, "tokens/trainable": 112196} +{"epoch": 0.65625, "grad_norm": 6.373160362243652, "learning_rate": 8.240385928474219e-05, "loss": 0.36690402030944824, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 36.71, "memory/max_allocated (GiB)": 36.71, "ppl": 1.44326, "step": 168, "tokens/total": 5745344, "tokens/train_per_sec_per_gpu": 37.33, "tokens/trainable": 112797} +{"epoch": 0.66015625, "grad_norm": 2.0158531665802, "learning_rate": 8.217303493946967e-05, "loss": 0.2671588659286499, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 37.13, "memory/max_allocated (GiB)": 37.13, "ppl": 1.30625, "step": 169, "tokens/total": 5779168, "tokens/train_per_sec_per_gpu": 37.06, "tokens/trainable": 113455} +{"epoch": 0.6640625, "grad_norm": 1.6270264387130737, "learning_rate": 8.194107981329746e-05, "loss": 0.2729431092739105, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 39.59, "memory/max_allocated (GiB)": 39.59, "ppl": 1.31383, "step": 170, "tokens/total": 5817328, "tokens/train_per_sec_per_gpu": 34.61, "tokens/trainable": 114133} +{"epoch": 0.66796875, "grad_norm": 5.114168167114258, "learning_rate": 8.170800355882518e-05, "loss": 0.2894909381866455, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.33575, "step": 171, "tokens/total": 5851392, "tokens/train_per_sec_per_gpu": 47.11, "tokens/trainable": 114859} +{"epoch": 0.671875, "grad_norm": 2.09724760055542, "learning_rate": 8.147381587530713e-05, "loss": 0.3392379581928253, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.40388, "step": 172, "tokens/total": 5884896, "tokens/train_per_sec_per_gpu": 42.33, "tokens/trainable": 115538} +{"epoch": 0.67578125, "grad_norm": 2.52998423576355, "learning_rate": 8.123852650824877e-05, "loss": 0.2846296429634094, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.32927, "step": 173, "tokens/total": 5918432, "tokens/train_per_sec_per_gpu": 41.35, "tokens/trainable": 116252} +{"epoch": 0.6796875, "grad_norm": 7.1928181648254395, "learning_rate": 8.100214524900103e-05, "loss": 0.2840503454208374, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 1.3285, "step": 174, "tokens/total": 5953200, "tokens/train_per_sec_per_gpu": 32.34, "tokens/trainable": 116876} +{"epoch": 0.68359375, "grad_norm": 3.724106550216675, "learning_rate": 8.076468193435301e-05, "loss": 0.3102782964706421, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 36.25, "memory/max_allocated (GiB)": 36.25, "ppl": 1.3638, "step": 175, "tokens/total": 5983584, "tokens/train_per_sec_per_gpu": 40.87, "tokens/trainable": 117460} +{"epoch": 0.6875, "grad_norm": 3.8473753929138184, "learning_rate": 8.052614644612253e-05, "loss": 0.3088795840740204, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.3619, "step": 176, "tokens/total": 6021104, "tokens/train_per_sec_per_gpu": 32.13, "tokens/trainable": 118151} +{"epoch": 0.69140625, "grad_norm": 1.655176043510437, "learning_rate": 8.028654871074489e-05, "loss": 0.23680651187896729, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.2672, "step": 177, "tokens/total": 6055184, "tokens/train_per_sec_per_gpu": 40.05, "tokens/trainable": 118869} +{"epoch": 0.6953125, "grad_norm": 2.2282919883728027, "learning_rate": 8.004589869885986e-05, "loss": 0.33533167839050293, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 36.3, "memory/max_allocated (GiB)": 36.3, "ppl": 1.3984, "step": 178, "tokens/total": 6086480, "tokens/train_per_sec_per_gpu": 37.0, "tokens/trainable": 119455} +{"epoch": 0.69921875, "grad_norm": 2.793135643005371, "learning_rate": 7.980420642489674e-05, "loss": 0.2868691682815552, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.33225, "step": 179, "tokens/total": 6124864, "tokens/train_per_sec_per_gpu": 34.74, "tokens/trainable": 120143} +{"epoch": 0.703125, "grad_norm": 2.1580562591552734, "learning_rate": 7.95614819466576e-05, "loss": 0.2514094114303589, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 37.48, "memory/max_allocated (GiB)": 37.48, "ppl": 1.28584, "step": 180, "tokens/total": 6158208, "tokens/train_per_sec_per_gpu": 37.82, "tokens/trainable": 120815} +{"epoch": 0.70703125, "grad_norm": 1.5403318405151367, "learning_rate": 7.931773536489872e-05, "loss": 0.2867579460144043, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 37.41, "memory/max_allocated (GiB)": 37.41, "ppl": 1.3321, "step": 181, "tokens/total": 6192480, "tokens/train_per_sec_per_gpu": 40.89, "tokens/trainable": 121481} +{"epoch": 0.7109375, "grad_norm": 1.851568579673767, "learning_rate": 7.907297682291035e-05, "loss": 0.25550347566604614, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.29111, "step": 182, "tokens/total": 6226864, "tokens/train_per_sec_per_gpu": 42.2, "tokens/trainable": 122192} +{"epoch": 0.71484375, "grad_norm": 1.6656593084335327, "learning_rate": 7.882721650609442e-05, "loss": 0.1754971146583557, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 39.48, "memory/max_allocated (GiB)": 39.48, "ppl": 1.19184, "step": 183, "tokens/total": 6266928, "tokens/train_per_sec_per_gpu": 36.91, "tokens/trainable": 122998} +{"epoch": 0.71875, "grad_norm": 2.195258378982544, "learning_rate": 7.85804646415409e-05, "loss": 0.27845048904418945, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.32108, "step": 184, "tokens/total": 6301680, "tokens/train_per_sec_per_gpu": 40.5, "tokens/trainable": 123666} +{"epoch": 0.72265625, "grad_norm": 2.8305811882019043, "learning_rate": 7.833273149760207e-05, "loss": 0.2590990364551544, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 1.29576, "step": 185, "tokens/total": 6331968, "tokens/train_per_sec_per_gpu": 41.71, "tokens/trainable": 124355} +{"epoch": 0.7265625, "grad_norm": 1.6782246828079224, "learning_rate": 7.808402738346527e-05, "loss": 0.20456254482269287, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.22699, "step": 186, "tokens/total": 6370448, "tokens/train_per_sec_per_gpu": 40.28, "tokens/trainable": 125101} +{"epoch": 0.73046875, "grad_norm": 2.8689637184143066, "learning_rate": 7.783436264872382e-05, "loss": 0.23046012222766876, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 38.93, "memory/max_allocated (GiB)": 38.93, "ppl": 1.25918, "step": 187, "tokens/total": 6406864, "tokens/train_per_sec_per_gpu": 39.06, "tokens/trainable": 125818} +{"epoch": 0.734375, "grad_norm": 2.573876142501831, "learning_rate": 7.758374768294647e-05, "loss": 0.23204149305820465, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 38.08, "memory/max_allocated (GiB)": 38.08, "ppl": 1.26117, "step": 188, "tokens/total": 6442560, "tokens/train_per_sec_per_gpu": 42.57, "tokens/trainable": 126561} +{"epoch": 0.73828125, "grad_norm": 1.6679835319519043, "learning_rate": 7.733219291524489e-05, "loss": 0.23721882700920105, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 37.39, "memory/max_allocated (GiB)": 37.39, "ppl": 1.26772, "step": 189, "tokens/total": 6475664, "tokens/train_per_sec_per_gpu": 41.55, "tokens/trainable": 127247} +{"epoch": 0.7421875, "grad_norm": 4.53490686416626, "learning_rate": 7.707970881383977e-05, "loss": 0.2333107590675354, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 36.32, "memory/max_allocated (GiB)": 36.32, "ppl": 1.26277, "step": 190, "tokens/total": 6507072, "tokens/train_per_sec_per_gpu": 45.36, "tokens/trainable": 127921} +{"epoch": 0.74609375, "grad_norm": 1.906408429145813, "learning_rate": 7.682630588562518e-05, "loss": 0.22473827004432678, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.25199, "step": 191, "tokens/total": 6541776, "tokens/train_per_sec_per_gpu": 38.43, "tokens/trainable": 128615} +{"epoch": 0.75, "grad_norm": 4.48731803894043, "learning_rate": 7.657199467573129e-05, "loss": 0.20678818225860596, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 36.93, "memory/max_allocated (GiB)": 36.93, "ppl": 1.22972, "step": 192, "tokens/total": 6575424, "tokens/train_per_sec_per_gpu": 36.88, "tokens/trainable": 129277} +{"epoch": 0.75390625, "grad_norm": 1.9951688051223755, "learning_rate": 7.631678576708561e-05, "loss": 0.1904849410057068, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.20984, "step": 193, "tokens/total": 6609472, "tokens/train_per_sec_per_gpu": 45.09, "tokens/trainable": 130028} +{"epoch": 0.7578125, "grad_norm": 19.33708381652832, "learning_rate": 7.606068977997255e-05, "loss": 0.20361962914466858, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 39.54, "memory/max_allocated (GiB)": 39.54, "ppl": 1.22583, "step": 194, "tokens/total": 6646816, "tokens/train_per_sec_per_gpu": 44.84, "tokens/trainable": 130802} +{"epoch": 0.76171875, "grad_norm": 5.662627696990967, "learning_rate": 7.580371737159148e-05, "loss": 0.31599435210227966, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.37162, "step": 195, "tokens/total": 6682528, "tokens/train_per_sec_per_gpu": 34.14, "tokens/trainable": 131443} +{"epoch": 0.765625, "grad_norm": 1.608836054801941, "learning_rate": 7.554587923561324e-05, "loss": 0.21700933575630188, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.24236, "step": 196, "tokens/total": 6716832, "tokens/train_per_sec_per_gpu": 35.98, "tokens/trainable": 132079} +{"epoch": 0.76953125, "grad_norm": 4.822183132171631, "learning_rate": 7.528718610173511e-05, "loss": 0.25845420360565186, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 37.53, "memory/max_allocated (GiB)": 37.53, "ppl": 1.29493, "step": 197, "tokens/total": 6751840, "tokens/train_per_sec_per_gpu": 41.01, "tokens/trainable": 132755} +{"epoch": 0.7734375, "grad_norm": 7.09438943862915, "learning_rate": 7.502764873523431e-05, "loss": 0.2803933024406433, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.32365, "step": 198, "tokens/total": 6785584, "tokens/train_per_sec_per_gpu": 41.54, "tokens/trainable": 133438} +{"epoch": 0.77734375, "grad_norm": 1.577925205230713, "learning_rate": 7.476727793652011e-05, "loss": 0.217776358127594, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 1.24331, "step": 199, "tokens/total": 6818560, "tokens/train_per_sec_per_gpu": 40.57, "tokens/trainable": 134123} +{"epoch": 0.78125, "grad_norm": 1.5719220638275146, "learning_rate": 7.450608454068415e-05, "loss": 0.19199641048908234, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.21167, "step": 200, "tokens/total": 6852704, "tokens/train_per_sec_per_gpu": 38.81, "tokens/trainable": 134810} +{"epoch": 0.78515625, "grad_norm": 1.595867395401001, "learning_rate": 7.424407941704987e-05, "loss": 0.22632955014705658, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 38.3, "memory/max_allocated (GiB)": 38.3, "ppl": 1.25399, "step": 201, "tokens/total": 6888912, "tokens/train_per_sec_per_gpu": 42.31, "tokens/trainable": 135532} +{"epoch": 0.7890625, "grad_norm": 1.6313085556030273, "learning_rate": 7.398127346871986e-05, "loss": 0.19573473930358887, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 39.52, "memory/max_allocated (GiB)": 39.52, "ppl": 1.2162, "step": 202, "tokens/total": 6925600, "tokens/train_per_sec_per_gpu": 37.38, "tokens/trainable": 136253} +{"epoch": 0.79296875, "grad_norm": 2.045466899871826, "learning_rate": 7.371767763212238e-05, "loss": 0.22891370952129364, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.25723, "step": 203, "tokens/total": 6960640, "tokens/train_per_sec_per_gpu": 41.92, "tokens/trainable": 136934} +{"epoch": 0.796875, "grad_norm": 2.114365577697754, "learning_rate": 7.345330287655617e-05, "loss": 0.2648341953754425, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.30321, "step": 204, "tokens/total": 6998352, "tokens/train_per_sec_per_gpu": 33.69, "tokens/trainable": 137607} +{"epoch": 0.80078125, "grad_norm": 2.3831422328948975, "learning_rate": 7.31881602037339e-05, "loss": 0.23550772666931152, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 36.6, "memory/max_allocated (GiB)": 36.6, "ppl": 1.26555, "step": 205, "tokens/total": 7030032, "tokens/train_per_sec_per_gpu": 38.49, "tokens/trainable": 138263} +{"epoch": 0.8046875, "grad_norm": 1.4453359842300415, "learning_rate": 7.29222606473245e-05, "loss": 0.22016146779060364, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 37.2, "memory/max_allocated (GiB)": 37.2, "ppl": 1.24628, "step": 206, "tokens/total": 7062528, "tokens/train_per_sec_per_gpu": 38.49, "tokens/trainable": 138908} +{"epoch": 0.80859375, "grad_norm": 1.8556581735610962, "learning_rate": 7.265561527249383e-05, "loss": 0.2183593511581421, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 37.07, "memory/max_allocated (GiB)": 37.07, "ppl": 1.24403, "step": 207, "tokens/total": 7096112, "tokens/train_per_sec_per_gpu": 40.68, "tokens/trainable": 139560} +{"epoch": 0.8125, "grad_norm": 0.93133145570755, "learning_rate": 7.238823517544436e-05, "loss": 0.1539715826511383, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.16646, "step": 208, "tokens/total": 7132848, "tokens/train_per_sec_per_gpu": 45.32, "tokens/trainable": 140336} +{"epoch": 0.81640625, "grad_norm": 2.1993565559387207, "learning_rate": 7.212013148295333e-05, "loss": 0.2273404896259308, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.25526, "step": 209, "tokens/total": 7166480, "tokens/train_per_sec_per_gpu": 42.95, "tokens/trainable": 141059} +{"epoch": 0.8203125, "grad_norm": 1.5221869945526123, "learning_rate": 7.185131535190975e-05, "loss": 0.20426702499389648, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.22663, "step": 210, "tokens/total": 7198784, "tokens/train_per_sec_per_gpu": 41.2, "tokens/trainable": 141779} +{"epoch": 0.82421875, "grad_norm": 1.3058140277862549, "learning_rate": 7.158179796885005e-05, "loss": 0.1851501762866974, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.2034, "step": 211, "tokens/total": 7232752, "tokens/train_per_sec_per_gpu": 41.26, "tokens/trainable": 142462} +{"epoch": 0.828125, "grad_norm": 1.987829327583313, "learning_rate": 7.131159054949273e-05, "loss": 0.25062790513038635, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.28483, "step": 212, "tokens/total": 7270880, "tokens/train_per_sec_per_gpu": 31.63, "tokens/trainable": 143103} +{"epoch": 0.83203125, "grad_norm": 2.038783311843872, "learning_rate": 7.104070433827139e-05, "loss": 0.25402969121932983, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.28921, "step": 213, "tokens/total": 7305936, "tokens/train_per_sec_per_gpu": 31.9, "tokens/trainable": 143677} +{"epoch": 0.8359375, "grad_norm": 1.7653354406356812, "learning_rate": 7.076915060786705e-05, "loss": 0.2111814022064209, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.23514, "step": 214, "tokens/total": 7338272, "tokens/train_per_sec_per_gpu": 39.52, "tokens/trainable": 144329} +{"epoch": 0.83984375, "grad_norm": 2.221554756164551, "learning_rate": 7.049694065873882e-05, "loss": 0.22094136476516724, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 37.98, "memory/max_allocated (GiB)": 37.98, "ppl": 1.24725, "step": 215, "tokens/total": 7372640, "tokens/train_per_sec_per_gpu": 36.36, "tokens/trainable": 144970} +{"epoch": 0.84375, "grad_norm": 1.3057806491851807, "learning_rate": 7.022408581865382e-05, "loss": 0.1875302493572235, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.20627, "step": 216, "tokens/total": 7406400, "tokens/train_per_sec_per_gpu": 35.33, "tokens/trainable": 145646} +{"epoch": 0.84765625, "grad_norm": 1.5299668312072754, "learning_rate": 6.99505974422157e-05, "loss": 0.18707917630672455, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.20572, "step": 217, "tokens/total": 7440544, "tokens/train_per_sec_per_gpu": 45.82, "tokens/trainable": 146350} +{"epoch": 0.8515625, "grad_norm": 1.4752360582351685, "learning_rate": 6.967648691039213e-05, "loss": 0.22413161396980286, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.25124, "step": 218, "tokens/total": 7472032, "tokens/train_per_sec_per_gpu": 42.44, "tokens/trainable": 146992} +{"epoch": 0.85546875, "grad_norm": 1.5958389043807983, "learning_rate": 6.940176563004123e-05, "loss": 0.23797598481178284, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.26868, "step": 219, "tokens/total": 7505824, "tokens/train_per_sec_per_gpu": 35.85, "tokens/trainable": 147611} +{"epoch": 0.859375, "grad_norm": 1.660178303718567, "learning_rate": 6.912644503343682e-05, "loss": 0.23957349359989166, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.27071, "step": 220, "tokens/total": 7543600, "tokens/train_per_sec_per_gpu": 35.33, "tokens/trainable": 148248} +{"epoch": 0.86328125, "grad_norm": 1.7662369012832642, "learning_rate": 6.885053657779273e-05, "loss": 0.2058052122592926, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 36.14, "memory/max_allocated (GiB)": 36.14, "ppl": 1.22851, "step": 221, "tokens/total": 7574544, "tokens/train_per_sec_per_gpu": 40.43, "tokens/trainable": 148922} +{"epoch": 0.8671875, "grad_norm": 1.3763375282287598, "learning_rate": 6.857405174478604e-05, "loss": 0.21669068932533264, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 37.2, "memory/max_allocated (GiB)": 37.2, "ppl": 1.24196, "step": 222, "tokens/total": 7608832, "tokens/train_per_sec_per_gpu": 36.35, "tokens/trainable": 149576} +{"epoch": 0.87109375, "grad_norm": 2.610156297683716, "learning_rate": 6.82970020400792e-05, "loss": 0.20305612683296204, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.22514, "step": 223, "tokens/total": 7645664, "tokens/train_per_sec_per_gpu": 33.28, "tokens/trainable": 150224} +{"epoch": 0.875, "grad_norm": 3.5878546237945557, "learning_rate": 6.801939899284132e-05, "loss": 0.1706829071044922, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 38.15, "memory/max_allocated (GiB)": 38.15, "ppl": 1.18611, "step": 224, "tokens/total": 7681200, "tokens/train_per_sec_per_gpu": 40.81, "tokens/trainable": 150959} +{"epoch": 0.87890625, "grad_norm": 2.0363447666168213, "learning_rate": 6.774125415526827e-05, "loss": 0.17985059320926666, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 37.84, "memory/max_allocated (GiB)": 37.84, "ppl": 1.19704, "step": 225, "tokens/total": 7714160, "tokens/train_per_sec_per_gpu": 47.08, "tokens/trainable": 151709} +{"epoch": 0.8828125, "grad_norm": 1.5753910541534424, "learning_rate": 6.746257910210214e-05, "loss": 0.20591789484024048, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 36.92, "memory/max_allocated (GiB)": 36.92, "ppl": 1.22865, "step": 226, "tokens/total": 7746944, "tokens/train_per_sec_per_gpu": 38.32, "tokens/trainable": 152347} +{"epoch": 0.88671875, "grad_norm": 2.029005765914917, "learning_rate": 6.718338543014937e-05, "loss": 0.2300569862127304, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 35.52, "memory/max_allocated (GiB)": 35.52, "ppl": 1.25867, "step": 227, "tokens/total": 7776576, "tokens/train_per_sec_per_gpu": 41.6, "tokens/trainable": 152958} +{"epoch": 0.890625, "grad_norm": 1.6499614715576172, "learning_rate": 6.69036847577983e-05, "loss": 0.1819838583469391, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.19959, "step": 228, "tokens/total": 7810000, "tokens/train_per_sec_per_gpu": 37.42, "tokens/trainable": 153635} +{"epoch": 0.89453125, "grad_norm": 2.496483325958252, "learning_rate": 6.662348872453553e-05, "loss": 0.15983757376670837, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 38.3, "memory/max_allocated (GiB)": 38.3, "ppl": 1.17332, "step": 229, "tokens/total": 7846112, "tokens/train_per_sec_per_gpu": 38.66, "tokens/trainable": 154304} +{"epoch": 0.8984375, "grad_norm": 2.300910472869873, "learning_rate": 6.63428089904618e-05, "loss": 0.20021280646324158, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.22166, "step": 230, "tokens/total": 7884864, "tokens/train_per_sec_per_gpu": 34.66, "tokens/trainable": 154964} +{"epoch": 0.90234375, "grad_norm": 3.035953998565674, "learning_rate": 6.60616572358065e-05, "loss": 0.2261052429676056, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.25371, "step": 231, "tokens/total": 7918800, "tokens/train_per_sec_per_gpu": 36.82, "tokens/trainable": 155534} +{"epoch": 0.90625, "grad_norm": 2.159595489501953, "learning_rate": 6.578004516044172e-05, "loss": 0.21984164416790009, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.24588, "step": 232, "tokens/total": 7954272, "tokens/train_per_sec_per_gpu": 43.04, "tokens/trainable": 156217} +{"epoch": 0.91015625, "grad_norm": 1.9467284679412842, "learning_rate": 6.549798448339548e-05, "loss": 0.19456937909126282, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.21479, "step": 233, "tokens/total": 7991328, "tokens/train_per_sec_per_gpu": 38.53, "tokens/trainable": 156901} +{"epoch": 0.9140625, "grad_norm": 1.2635952234268188, "learning_rate": 6.521548694236384e-05, "loss": 0.19088777899742126, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 39.19, "memory/max_allocated (GiB)": 39.19, "ppl": 1.21032, "step": 234, "tokens/total": 8028224, "tokens/train_per_sec_per_gpu": 34.3, "tokens/trainable": 157554} +{"epoch": 0.91796875, "grad_norm": 1.5488073825836182, "learning_rate": 6.493256429322259e-05, "loss": 0.1908802092075348, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.21031, "step": 235, "tokens/total": 8064000, "tokens/train_per_sec_per_gpu": 36.57, "tokens/trainable": 158210} +{"epoch": 0.921875, "grad_norm": 2.1925952434539795, "learning_rate": 6.464922830953799e-05, "loss": 0.20912298560142517, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 39.19, "memory/max_allocated (GiB)": 39.19, "ppl": 1.2326, "step": 236, "tokens/total": 8102848, "tokens/train_per_sec_per_gpu": 32.26, "tokens/trainable": 158867} +{"epoch": 0.92578125, "grad_norm": 4.631251335144043, "learning_rate": 6.436549078207688e-05, "loss": 0.20756256580352783, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 38.11, "memory/max_allocated (GiB)": 38.11, "ppl": 1.23067, "step": 237, "tokens/total": 8135584, "tokens/train_per_sec_per_gpu": 41.17, "tokens/trainable": 159512} +{"epoch": 0.9296875, "grad_norm": 2.6595966815948486, "learning_rate": 6.408136351831592e-05, "loss": 0.1994849443435669, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.22077, "step": 238, "tokens/total": 8167552, "tokens/train_per_sec_per_gpu": 36.16, "tokens/trainable": 160151} +{"epoch": 0.93359375, "grad_norm": 1.9650033712387085, "learning_rate": 6.379685834195036e-05, "loss": 0.18771520256996155, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.20649, "step": 239, "tokens/total": 8206704, "tokens/train_per_sec_per_gpu": 37.08, "tokens/trainable": 160915} +{"epoch": 0.9375, "grad_norm": 2.337855577468872, "learning_rate": 6.351198709240186e-05, "loss": 0.17620918154716492, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.19269, "step": 240, "tokens/total": 8246480, "tokens/train_per_sec_per_gpu": 31.9, "tokens/trainable": 161611} +{"epoch": 0.94140625, "grad_norm": 1.8966326713562012, "learning_rate": 6.32267616243259e-05, "loss": 0.21485212445259094, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.23968, "step": 241, "tokens/total": 8278688, "tokens/train_per_sec_per_gpu": 40.82, "tokens/trainable": 162269} +{"epoch": 0.9453125, "grad_norm": 1.9023326635360718, "learning_rate": 6.294119380711849e-05, "loss": 0.2155897319316864, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.24059, "step": 242, "tokens/total": 8312688, "tokens/train_per_sec_per_gpu": 33.16, "tokens/trainable": 162886} +{"epoch": 0.94921875, "grad_norm": 1.8844748735427856, "learning_rate": 6.265529552442209e-05, "loss": 0.18537583947181702, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 37.57, "memory/max_allocated (GiB)": 37.57, "ppl": 1.20367, "step": 243, "tokens/total": 8347664, "tokens/train_per_sec_per_gpu": 42.32, "tokens/trainable": 163579} +{"epoch": 0.953125, "grad_norm": 2.5454559326171875, "learning_rate": 6.236907867363127e-05, "loss": 0.21761459112167358, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.24311, "step": 244, "tokens/total": 8381680, "tokens/train_per_sec_per_gpu": 39.21, "tokens/trainable": 164219} +{"epoch": 0.95703125, "grad_norm": 1.7402381896972656, "learning_rate": 6.208255516539749e-05, "loss": 0.18362869322299957, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 36.21, "memory/max_allocated (GiB)": 36.21, "ppl": 1.20157, "step": 245, "tokens/total": 8413408, "tokens/train_per_sec_per_gpu": 35.91, "tokens/trainable": 164860} +{"epoch": 0.9609375, "grad_norm": 1.0357977151870728, "learning_rate": 6.179573692313344e-05, "loss": 0.18438392877578735, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.20248, "step": 246, "tokens/total": 8450048, "tokens/train_per_sec_per_gpu": 39.39, "tokens/trainable": 165533} +{"epoch": 0.96484375, "grad_norm": 1.834513783454895, "learning_rate": 6.150863588251694e-05, "loss": 0.18886657059192657, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 39.35, "memory/max_allocated (GiB)": 39.35, "ppl": 1.20788, "step": 247, "tokens/total": 8489968, "tokens/train_per_sec_per_gpu": 33.01, "tokens/trainable": 166221} +{"epoch": 0.96875, "grad_norm": 2.425366163253784, "learning_rate": 6.122126399099419e-05, "loss": 0.17328140139579773, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 39.59, "memory/max_allocated (GiB)": 39.59, "ppl": 1.1892, "step": 248, "tokens/total": 8529456, "tokens/train_per_sec_per_gpu": 44.45, "tokens/trainable": 167056} +{"epoch": 0.97265625, "grad_norm": 6.2131476402282715, "learning_rate": 6.0933633207282615e-05, "loss": 0.22457367181777954, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 38.22, "memory/max_allocated (GiB)": 38.22, "ppl": 1.25179, "step": 249, "tokens/total": 8565472, "tokens/train_per_sec_per_gpu": 36.73, "tokens/trainable": 167742} +{"epoch": 0.9765625, "grad_norm": 1.259128451347351, "learning_rate": 6.064575550087316e-05, "loss": 0.1661762297153473, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.18078, "step": 250, "tokens/total": 8602128, "tokens/train_per_sec_per_gpu": 36.61, "tokens/trainable": 168437} +{"epoch": 0.98046875, "grad_norm": 2.201174020767212, "learning_rate": 6.0357642851532245e-05, "loss": 0.19360020756721497, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.21361, "step": 251, "tokens/total": 8638304, "tokens/train_per_sec_per_gpu": 34.37, "tokens/trainable": 169119} +{"epoch": 0.984375, "grad_norm": 4.381434440612793, "learning_rate": 6.0069307248803294e-05, "loss": 0.2143193781375885, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.23902, "step": 252, "tokens/total": 8673248, "tokens/train_per_sec_per_gpu": 33.68, "tokens/trainable": 169765} +{"epoch": 0.98828125, "grad_norm": 1.288630485534668, "learning_rate": 5.9780760691507635e-05, "loss": 0.1745883971452713, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 36.16, "memory/max_allocated (GiB)": 36.16, "ppl": 1.19076, "step": 253, "tokens/total": 8704208, "tokens/train_per_sec_per_gpu": 37.01, "tokens/trainable": 170346} +{"epoch": 0.9921875, "grad_norm": 1.6816622018814087, "learning_rate": 5.9492015187245334e-05, "loss": 0.1601647138595581, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 38.63, "memory/max_allocated (GiB)": 38.63, "ppl": 1.1737, "step": 254, "tokens/total": 8741408, "tokens/train_per_sec_per_gpu": 34.44, "tokens/trainable": 170971} +{"epoch": 0.99609375, "grad_norm": 1.982893466949463, "learning_rate": 5.920308275189541e-05, "loss": 0.19356635212898254, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 36.54, "memory/max_allocated (GiB)": 36.54, "ppl": 1.21357, "step": 255, "tokens/total": 8772832, "tokens/train_per_sec_per_gpu": 37.22, "tokens/trainable": 171569} +{"epoch": 1.0, "grad_norm": 1.4490280151367188, "learning_rate": 5.8913975409115874e-05, "loss": 0.16692230105400085, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 37.57, "memory/max_allocated (GiB)": 37.57, "ppl": 1.18166, "step": 256, "tokens/total": 8805840, "tokens/train_per_sec_per_gpu": 40.33, "tokens/trainable": 172271} +{"epoch": 1.00390625, "grad_norm": 1.8607816696166992, "learning_rate": 5.8624705189843395e-05, "loss": 0.14919525384902954, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 39.04, "memory/max_allocated (GiB)": 39.04, "ppl": 1.1609, "step": 257, "tokens/total": 8842848, "tokens/train_per_sec_per_gpu": 35.38, "tokens/trainable": 172986} +{"epoch": 1.0078125, "grad_norm": 0.9727902412414551, "learning_rate": 5.833528413179249e-05, "loss": 0.1364407241344452, "memory/device_reserved (GiB)": 40.53, "memory/max_active (GiB)": 38.41, "memory/max_allocated (GiB)": 38.41, "ppl": 1.14619, "step": 258, "tokens/total": 8879088, "tokens/train_per_sec_per_gpu": 35.82, "tokens/trainable": 173695} +{"epoch": 1.01171875, "grad_norm": 2.7530388832092285, "learning_rate": 5.80457242789548e-05, "loss": 0.13948604464530945, "memory/device_reserved (GiB)": 41.82, "memory/max_active (GiB)": 39.48, "memory/max_allocated (GiB)": 39.48, "ppl": 1.14968, "step": 259, "tokens/total": 8918912, "tokens/train_per_sec_per_gpu": 38.73, "tokens/trainable": 174491} +{"epoch": 1.015625, "grad_norm": 0.9006531834602356, "learning_rate": 5.77560376810977e-05, "loss": 0.11439847201108932, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.1212, "step": 260, "tokens/total": 8951920, "tokens/train_per_sec_per_gpu": 44.15, "tokens/trainable": 175221} +{"epoch": 1.01953125, "grad_norm": 2.08390474319458, "learning_rate": 5.7466236393263005e-05, "loss": 0.16179309785366058, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 38.93, "memory/max_allocated (GiB)": 38.93, "ppl": 1.17562, "step": 261, "tokens/total": 8986272, "tokens/train_per_sec_per_gpu": 37.96, "tokens/trainable": 175877} +{"epoch": 1.0234375, "grad_norm": 4.235605239868164, "learning_rate": 5.717633247526522e-05, "loss": 0.19914165139198303, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 36.63, "memory/max_allocated (GiB)": 36.63, "ppl": 1.22035, "step": 262, "tokens/total": 9016640, "tokens/train_per_sec_per_gpu": 37.43, "tokens/trainable": 176457} +{"epoch": 1.02734375, "grad_norm": 1.4491984844207764, "learning_rate": 5.688633799118971e-05, "loss": 0.1993461698293686, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 36.82, "memory/max_allocated (GiB)": 36.82, "ppl": 1.2206, "step": 263, "tokens/total": 9048784, "tokens/train_per_sec_per_gpu": 33.68, "tokens/trainable": 177048} +{"epoch": 1.03125, "grad_norm": 0.9894397258758545, "learning_rate": 5.659626500889066e-05, "loss": 0.12894192337989807, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 36.93, "memory/max_allocated (GiB)": 36.93, "ppl": 1.13762, "step": 264, "tokens/total": 9082208, "tokens/train_per_sec_per_gpu": 42.72, "tokens/trainable": 177773} +{"epoch": 1.03515625, "grad_norm": 0.9393889307975769, "learning_rate": 5.6306125599488905e-05, "loss": 0.15550853312015533, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 36.72, "memory/max_allocated (GiB)": 36.72, "ppl": 1.16825, "step": 265, "tokens/total": 9112560, "tokens/train_per_sec_per_gpu": 46.96, "tokens/trainable": 178410} +{"epoch": 1.0390625, "grad_norm": 1.0069793462753296, "learning_rate": 5.601593183686955e-05, "loss": 0.13730809092521667, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.14718, "step": 266, "tokens/total": 9149392, "tokens/train_per_sec_per_gpu": 40.67, "tokens/trainable": 179138} +{"epoch": 1.04296875, "grad_norm": 1.2044590711593628, "learning_rate": 5.572569579717961e-05, "loss": 0.1583901345729828, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.17162, "step": 267, "tokens/total": 9184448, "tokens/train_per_sec_per_gpu": 36.07, "tokens/trainable": 179833} +{"epoch": 1.046875, "grad_norm": 1.6468054056167603, "learning_rate": 5.543542955832538e-05, "loss": 0.145704984664917, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.15685, "step": 268, "tokens/total": 9224064, "tokens/train_per_sec_per_gpu": 36.16, "tokens/trainable": 180556} +{"epoch": 1.05078125, "grad_norm": 1.4341230392456055, "learning_rate": 5.514514519946986e-05, "loss": 0.1543973684310913, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.16695, "step": 269, "tokens/total": 9264048, "tokens/train_per_sec_per_gpu": 39.72, "tokens/trainable": 181312} +{"epoch": 1.0546875, "grad_norm": 0.8337639570236206, "learning_rate": 5.485485480053015e-05, "loss": 0.141662135720253, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.15219, "step": 270, "tokens/total": 9302128, "tokens/train_per_sec_per_gpu": 31.09, "tokens/trainable": 181942} +{"epoch": 1.05859375, "grad_norm": 1.6903936862945557, "learning_rate": 5.4564570441674645e-05, "loss": 0.14947378635406494, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.16122, "step": 271, "tokens/total": 9335408, "tokens/train_per_sec_per_gpu": 39.66, "tokens/trainable": 182578} +{"epoch": 1.0625, "grad_norm": 5.138545513153076, "learning_rate": 5.42743042028204e-05, "loss": 0.1707192063331604, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 37.73, "memory/max_allocated (GiB)": 37.73, "ppl": 1.18616, "step": 272, "tokens/total": 9370608, "tokens/train_per_sec_per_gpu": 29.68, "tokens/trainable": 183176} +{"epoch": 1.06640625, "grad_norm": 0.7622899413108826, "learning_rate": 5.3984068163130464e-05, "loss": 0.12906183302402496, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.13776, "step": 273, "tokens/total": 9403808, "tokens/train_per_sec_per_gpu": 43.17, "tokens/trainable": 183920} +{"epoch": 1.0703125, "grad_norm": 1.7856054306030273, "learning_rate": 5.369387440051111e-05, "loss": 0.14732614159584045, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.15873, "step": 274, "tokens/total": 9437072, "tokens/train_per_sec_per_gpu": 36.47, "tokens/trainable": 184569} +{"epoch": 1.07421875, "grad_norm": 3.3400893211364746, "learning_rate": 5.340373499110935e-05, "loss": 0.16010825335979462, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 1.17364, "step": 275, "tokens/total": 9472496, "tokens/train_per_sec_per_gpu": 38.17, "tokens/trainable": 185281} +{"epoch": 1.078125, "grad_norm": 1.3510605096817017, "learning_rate": 5.3113662008810304e-05, "loss": 0.1350543200969696, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.1446, "step": 276, "tokens/total": 9512512, "tokens/train_per_sec_per_gpu": 36.19, "tokens/trainable": 186012} +{"epoch": 1.08203125, "grad_norm": 1.4547451734542847, "learning_rate": 5.282366752473479e-05, "loss": 0.1404554843902588, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 38.1, "memory/max_allocated (GiB)": 38.1, "ppl": 1.1508, "step": 277, "tokens/total": 9549376, "tokens/train_per_sec_per_gpu": 35.91, "tokens/trainable": 186690} +{"epoch": 1.0859375, "grad_norm": 2.4104483127593994, "learning_rate": 5.2533763606737005e-05, "loss": 0.21291038393974304, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.23727, "step": 278, "tokens/total": 9588880, "tokens/train_per_sec_per_gpu": 29.12, "tokens/trainable": 187321} +{"epoch": 1.08984375, "grad_norm": 1.667301893234253, "learning_rate": 5.224396231890232e-05, "loss": 0.13357016444206238, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.1429, "step": 279, "tokens/total": 9620416, "tokens/train_per_sec_per_gpu": 43.8, "tokens/trainable": 188051} +{"epoch": 1.09375, "grad_norm": 4.487210273742676, "learning_rate": 5.195427572104522e-05, "loss": 0.1926628053188324, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 38.81, "memory/max_allocated (GiB)": 38.81, "ppl": 1.21247, "step": 280, "tokens/total": 9654544, "tokens/train_per_sec_per_gpu": 39.83, "tokens/trainable": 188719} +{"epoch": 1.09765625, "grad_norm": 1.824210524559021, "learning_rate": 5.166471586820751e-05, "loss": 0.15753674507141113, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 39.28, "memory/max_allocated (GiB)": 39.28, "ppl": 1.17062, "step": 281, "tokens/total": 9691056, "tokens/train_per_sec_per_gpu": 41.6, "tokens/trainable": 189464} +{"epoch": 1.1015625, "grad_norm": 1.8863017559051514, "learning_rate": 5.1375294810156615e-05, "loss": 0.16234427690505981, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.17627, "step": 282, "tokens/total": 9725280, "tokens/train_per_sec_per_gpu": 38.02, "tokens/trainable": 190091} +{"epoch": 1.10546875, "grad_norm": 1.7989331483840942, "learning_rate": 5.1086024590884144e-05, "loss": 0.1722785085439682, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 39.09, "memory/max_allocated (GiB)": 39.09, "ppl": 1.18801, "step": 283, "tokens/total": 9763056, "tokens/train_per_sec_per_gpu": 35.4, "tokens/trainable": 190743} +{"epoch": 1.109375, "grad_norm": 0.7372687458992004, "learning_rate": 5.079691724810461e-05, "loss": 0.13670411705970764, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.14649, "step": 284, "tokens/total": 9798480, "tokens/train_per_sec_per_gpu": 31.2, "tokens/trainable": 191364} +{"epoch": 1.11328125, "grad_norm": 2.4122629165649414, "learning_rate": 5.0507984812754684e-05, "loss": 0.14587676525115967, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 37.19, "memory/max_allocated (GiB)": 37.19, "ppl": 1.15705, "step": 285, "tokens/total": 9831712, "tokens/train_per_sec_per_gpu": 41.91, "tokens/trainable": 192066} +{"epoch": 1.1171875, "grad_norm": 1.4166905879974365, "learning_rate": 5.021923930849237e-05, "loss": 0.15224450826644897, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.16444, "step": 286, "tokens/total": 9865216, "tokens/train_per_sec_per_gpu": 37.86, "tokens/trainable": 192715} +{"epoch": 1.12109375, "grad_norm": 1.111698865890503, "learning_rate": 4.99306927511967e-05, "loss": 0.15252886712551117, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.16478, "step": 287, "tokens/total": 9898256, "tokens/train_per_sec_per_gpu": 39.55, "tokens/trainable": 193312} +{"epoch": 1.125, "grad_norm": 1.1984844207763672, "learning_rate": 4.964235714846775e-05, "loss": 0.18192949891090393, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 36.31, "memory/max_allocated (GiB)": 36.31, "ppl": 1.19953, "step": 288, "tokens/total": 9929712, "tokens/train_per_sec_per_gpu": 39.82, "tokens/trainable": 193899} +{"epoch": 1.12890625, "grad_norm": 1.3444398641586304, "learning_rate": 4.9354244499126866e-05, "loss": 0.16066274046897888, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 36.11, "memory/max_allocated (GiB)": 36.11, "ppl": 1.17429, "step": 289, "tokens/total": 9961040, "tokens/train_per_sec_per_gpu": 40.59, "tokens/trainable": 194586} +{"epoch": 1.1328125, "grad_norm": 1.2725396156311035, "learning_rate": 4.90663667927174e-05, "loss": 0.14656515419483185, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.15785, "step": 290, "tokens/total": 9993456, "tokens/train_per_sec_per_gpu": 47.18, "tokens/trainable": 195256} +{"epoch": 1.13671875, "grad_norm": 2.1369969844818115, "learning_rate": 4.877873600900581e-05, "loss": 0.12006954103708267, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 37.98, "memory/max_allocated (GiB)": 37.98, "ppl": 1.12758, "step": 291, "tokens/total": 10029280, "tokens/train_per_sec_per_gpu": 35.54, "tokens/trainable": 195939} +{"epoch": 1.140625, "grad_norm": 1.527660846710205, "learning_rate": 4.849136411748306e-05, "loss": 0.12132563441991806, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 1.12899, "step": 292, "tokens/total": 10067072, "tokens/train_per_sec_per_gpu": 35.53, "tokens/trainable": 196625} +{"epoch": 1.14453125, "grad_norm": 2.111377477645874, "learning_rate": 4.8204263076866574e-05, "loss": 0.1538582444190979, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 38.66, "memory/max_allocated (GiB)": 38.66, "ppl": 1.16633, "step": 293, "tokens/total": 10102544, "tokens/train_per_sec_per_gpu": 38.84, "tokens/trainable": 197330} +{"epoch": 1.1484375, "grad_norm": 2.1832504272460938, "learning_rate": 4.791744483460251e-05, "loss": 0.21699295938014984, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.24234, "step": 294, "tokens/total": 10136624, "tokens/train_per_sec_per_gpu": 31.65, "tokens/trainable": 197945} +{"epoch": 1.15234375, "grad_norm": 1.9305617809295654, "learning_rate": 4.7630921326368736e-05, "loss": 0.15808546543121338, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 39.19, "memory/max_allocated (GiB)": 39.19, "ppl": 1.17127, "step": 295, "tokens/total": 10174864, "tokens/train_per_sec_per_gpu": 29.5, "tokens/trainable": 198542} +{"epoch": 1.15625, "grad_norm": 0.8348421454429626, "learning_rate": 4.7344704475577916e-05, "loss": 0.13590405881404877, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 35.48, "memory/max_allocated (GiB)": 35.48, "ppl": 1.14557, "step": 296, "tokens/total": 10204016, "tokens/train_per_sec_per_gpu": 42.54, "tokens/trainable": 199199} +{"epoch": 1.16015625, "grad_norm": 1.714406132698059, "learning_rate": 4.705880619288153e-05, "loss": 0.15606671571731567, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.18, "memory/max_allocated (GiB)": 38.18, "ppl": 1.1689, "step": 297, "tokens/total": 10239696, "tokens/train_per_sec_per_gpu": 34.98, "tokens/trainable": 199825} +{"epoch": 1.1640625, "grad_norm": 1.0167429447174072, "learning_rate": 4.677323837567412e-05, "loss": 0.14255037903785706, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.15321, "step": 298, "tokens/total": 10272192, "tokens/train_per_sec_per_gpu": 44.8, "tokens/trainable": 200534} +{"epoch": 1.16796875, "grad_norm": 1.16241455078125, "learning_rate": 4.6488012907598146e-05, "loss": 0.15123620629310608, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.3, "memory/max_allocated (GiB)": 36.3, "ppl": 1.16327, "step": 299, "tokens/total": 10304128, "tokens/train_per_sec_per_gpu": 38.04, "tokens/trainable": 201167} +{"epoch": 1.171875, "grad_norm": 1.0221892595291138, "learning_rate": 4.620314165804964e-05, "loss": 0.12758857011795044, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.26, "memory/max_allocated (GiB)": 38.26, "ppl": 1.13609, "step": 300, "tokens/total": 10339536, "tokens/train_per_sec_per_gpu": 38.19, "tokens/trainable": 201857} +{"epoch": 1.17578125, "grad_norm": 1.3383748531341553, "learning_rate": 4.591863648168407e-05, "loss": 0.13256292045116425, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.14175, "step": 301, "tokens/total": 10373808, "tokens/train_per_sec_per_gpu": 39.41, "tokens/trainable": 202595} +{"epoch": 1.1796875, "grad_norm": 0.9689850807189941, "learning_rate": 4.5634509217923135e-05, "loss": 0.14561131596565247, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.15675, "step": 302, "tokens/total": 10406928, "tokens/train_per_sec_per_gpu": 42.15, "tokens/trainable": 203277} +{"epoch": 1.18359375, "grad_norm": 1.9312126636505127, "learning_rate": 4.535077169046201e-05, "loss": 0.16719262301921844, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.93, "memory/max_allocated (GiB)": 37.93, "ppl": 1.18198, "step": 303, "tokens/total": 10443216, "tokens/train_per_sec_per_gpu": 36.97, "tokens/trainable": 203947} +{"epoch": 1.1875, "grad_norm": 3.62691593170166, "learning_rate": 4.506743570677743e-05, "loss": 0.19638203084468842, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.13, "memory/max_allocated (GiB)": 36.13, "ppl": 1.21699, "step": 304, "tokens/total": 10473040, "tokens/train_per_sec_per_gpu": 46.01, "tokens/trainable": 204560} +{"epoch": 1.19140625, "grad_norm": 2.1077091693878174, "learning_rate": 4.478451305763618e-05, "loss": 0.16592656075954437, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.17, "memory/max_allocated (GiB)": 37.17, "ppl": 1.18049, "step": 305, "tokens/total": 10506288, "tokens/train_per_sec_per_gpu": 44.23, "tokens/trainable": 205227} +{"epoch": 1.1953125, "grad_norm": 1.8278619050979614, "learning_rate": 4.450201551660454e-05, "loss": 0.164655402302742, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.27, "memory/max_allocated (GiB)": 36.27, "ppl": 1.17899, "step": 306, "tokens/total": 10537120, "tokens/train_per_sec_per_gpu": 39.85, "tokens/trainable": 205860} +{"epoch": 1.19921875, "grad_norm": 1.7636877298355103, "learning_rate": 4.4219954839558276e-05, "loss": 0.16558557748794556, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.18008, "step": 307, "tokens/total": 10571184, "tokens/train_per_sec_per_gpu": 35.77, "tokens/trainable": 206503} +{"epoch": 1.203125, "grad_norm": 2.8365039825439453, "learning_rate": 4.393834276419352e-05, "loss": 0.17324353754520416, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 35.6, "memory/max_allocated (GiB)": 35.6, "ppl": 1.18916, "step": 308, "tokens/total": 10600736, "tokens/train_per_sec_per_gpu": 48.43, "tokens/trainable": 207191} +{"epoch": 1.20703125, "grad_norm": 2.162348747253418, "learning_rate": 4.36571910095382e-05, "loss": 0.15115702152252197, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.16318, "step": 309, "tokens/total": 10638720, "tokens/train_per_sec_per_gpu": 38.05, "tokens/trainable": 207846} +{"epoch": 1.2109375, "grad_norm": 1.8761905431747437, "learning_rate": 4.337651127546448e-05, "loss": 0.17395082116127014, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.19, "step": 310, "tokens/total": 10673936, "tokens/train_per_sec_per_gpu": 41.68, "tokens/trainable": 208504} +{"epoch": 1.21484375, "grad_norm": 1.1970802545547485, "learning_rate": 4.3096315242201736e-05, "loss": 0.1596509963274002, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.37, "memory/max_allocated (GiB)": 36.37, "ppl": 1.1731, "step": 311, "tokens/total": 10706256, "tokens/train_per_sec_per_gpu": 37.71, "tokens/trainable": 209146} +{"epoch": 1.21875, "grad_norm": 1.4793503284454346, "learning_rate": 4.2816614569850635e-05, "loss": 0.17267344892024994, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.49, "memory/max_allocated (GiB)": 38.49, "ppl": 1.18848, "step": 312, "tokens/total": 10741696, "tokens/train_per_sec_per_gpu": 37.34, "tokens/trainable": 209803} +{"epoch": 1.22265625, "grad_norm": 0.9521883130073547, "learning_rate": 4.2537420897897864e-05, "loss": 0.14369678497314453, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.63, "memory/max_allocated (GiB)": 38.63, "ppl": 1.15453, "step": 313, "tokens/total": 10776928, "tokens/train_per_sec_per_gpu": 35.81, "tokens/trainable": 210507} +{"epoch": 1.2265625, "grad_norm": 2.787506580352783, "learning_rate": 4.225874584473174e-05, "loss": 0.16259506344795227, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.17656, "step": 314, "tokens/total": 10808976, "tokens/train_per_sec_per_gpu": 38.18, "tokens/trainable": 211135} +{"epoch": 1.23046875, "grad_norm": 1.9383472204208374, "learning_rate": 4.19806010071587e-05, "loss": 0.1389136016368866, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.36, "memory/max_allocated (GiB)": 38.36, "ppl": 1.14902, "step": 315, "tokens/total": 10846640, "tokens/train_per_sec_per_gpu": 37.18, "tokens/trainable": 211867} +{"epoch": 1.234375, "grad_norm": 2.2920427322387695, "learning_rate": 4.170299795992081e-05, "loss": 0.1668265163898468, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.74, "memory/max_allocated (GiB)": 36.74, "ppl": 1.18155, "step": 316, "tokens/total": 10879792, "tokens/train_per_sec_per_gpu": 37.86, "tokens/trainable": 212490} +{"epoch": 1.23828125, "grad_norm": 1.5742852687835693, "learning_rate": 4.142594825521398e-05, "loss": 0.14010077714920044, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.12, "memory/max_allocated (GiB)": 39.12, "ppl": 1.15039, "step": 317, "tokens/total": 10918640, "tokens/train_per_sec_per_gpu": 34.04, "tokens/trainable": 213183} +{"epoch": 1.2421875, "grad_norm": 2.7536003589630127, "learning_rate": 4.114946342220728e-05, "loss": 0.13912299275398254, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 1.14927, "step": 318, "tokens/total": 10954976, "tokens/train_per_sec_per_gpu": 37.8, "tokens/trainable": 213903} +{"epoch": 1.24609375, "grad_norm": 0.7037860155105591, "learning_rate": 4.087355496656321e-05, "loss": 0.13921061158180237, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 1.14937, "step": 319, "tokens/total": 10988320, "tokens/train_per_sec_per_gpu": 42.9, "tokens/trainable": 214595} +{"epoch": 1.25, "grad_norm": 8.298157691955566, "learning_rate": 4.05982343699588e-05, "loss": 0.13879014551639557, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.14888, "step": 320, "tokens/total": 11027216, "tokens/train_per_sec_per_gpu": 37.75, "tokens/trainable": 215351} +{"epoch": 1.25390625, "grad_norm": 1.8848050832748413, "learning_rate": 4.0323513089607876e-05, "loss": 0.13056349754333496, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.13947, "step": 321, "tokens/total": 11061680, "tokens/train_per_sec_per_gpu": 38.75, "tokens/trainable": 216069} +{"epoch": 1.2578125, "grad_norm": 1.0544977188110352, "learning_rate": 4.004940255778431e-05, "loss": 0.1258055418729782, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.13406, "step": 322, "tokens/total": 11100048, "tokens/train_per_sec_per_gpu": 40.93, "tokens/trainable": 216794} +{"epoch": 1.26171875, "grad_norm": 1.5046210289001465, "learning_rate": 3.977591418134619e-05, "loss": 0.1591646522283554, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.45, "memory/max_allocated (GiB)": 38.45, "ppl": 1.17253, "step": 323, "tokens/total": 11134704, "tokens/train_per_sec_per_gpu": 38.59, "tokens/trainable": 217459} +{"epoch": 1.265625, "grad_norm": 2.365126848220825, "learning_rate": 3.95030593412612e-05, "loss": 0.16326269507408142, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.17735, "step": 324, "tokens/total": 11170320, "tokens/train_per_sec_per_gpu": 40.57, "tokens/trainable": 218135} +{"epoch": 1.26953125, "grad_norm": 0.6725894212722778, "learning_rate": 3.923084939213296e-05, "loss": 0.13335976004600525, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 35.58, "memory/max_allocated (GiB)": 35.58, "ppl": 1.14266, "step": 325, "tokens/total": 11200416, "tokens/train_per_sec_per_gpu": 41.48, "tokens/trainable": 218772} +{"epoch": 1.2734375, "grad_norm": 2.034050941467285, "learning_rate": 3.895929566172861e-05, "loss": 0.11995745450258255, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.12745, "step": 326, "tokens/total": 11236080, "tokens/train_per_sec_per_gpu": 38.11, "tokens/trainable": 219482} +{"epoch": 1.27734375, "grad_norm": 0.6456319689750671, "learning_rate": 3.868840945050728e-05, "loss": 0.11574877053499222, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.05, "memory/max_allocated (GiB)": 39.05, "ppl": 1.12271, "step": 327, "tokens/total": 11274608, "tokens/train_per_sec_per_gpu": 36.38, "tokens/trainable": 220183} +{"epoch": 1.28125, "grad_norm": 0.9534568190574646, "learning_rate": 3.841820203114995e-05, "loss": 0.1416299194097519, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.15215, "step": 328, "tokens/total": 11305968, "tokens/train_per_sec_per_gpu": 42.04, "tokens/trainable": 220861} +{"epoch": 1.28515625, "grad_norm": 1.173803448677063, "learning_rate": 3.814868464809027e-05, "loss": 0.16574802994728088, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.61, "memory/max_allocated (GiB)": 36.61, "ppl": 1.18028, "step": 329, "tokens/total": 11337536, "tokens/train_per_sec_per_gpu": 38.18, "tokens/trainable": 221463} +{"epoch": 1.2890625, "grad_norm": 3.0476975440979004, "learning_rate": 3.787986851704667e-05, "loss": 0.15483789145946503, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.16747, "step": 330, "tokens/total": 11370688, "tokens/train_per_sec_per_gpu": 41.62, "tokens/trainable": 222122} +{"epoch": 1.29296875, "grad_norm": 1.257934331893921, "learning_rate": 3.7611764824555654e-05, "loss": 0.13243068754673004, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.57, "memory/max_allocated (GiB)": 37.57, "ppl": 1.1416, "step": 331, "tokens/total": 11401888, "tokens/train_per_sec_per_gpu": 47.91, "tokens/trainable": 222829} +{"epoch": 1.296875, "grad_norm": 1.0615761280059814, "learning_rate": 3.734438472750619e-05, "loss": 0.11509612202644348, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.35, "memory/max_allocated (GiB)": 39.35, "ppl": 1.12198, "step": 332, "tokens/total": 11441536, "tokens/train_per_sec_per_gpu": 35.78, "tokens/trainable": 223540} +{"epoch": 1.30078125, "grad_norm": 3.0522701740264893, "learning_rate": 3.707773935267552e-05, "loss": 0.1481618583202362, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.1597, "step": 333, "tokens/total": 11478224, "tokens/train_per_sec_per_gpu": 36.65, "tokens/trainable": 224258} +{"epoch": 1.3046875, "grad_norm": 4.889665126800537, "learning_rate": 3.68118397962661e-05, "loss": 0.20112374424934387, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.39, "memory/max_allocated (GiB)": 38.39, "ppl": 1.22278, "step": 334, "tokens/total": 11512672, "tokens/train_per_sec_per_gpu": 36.23, "tokens/trainable": 224873} +{"epoch": 1.30859375, "grad_norm": 1.1024609804153442, "learning_rate": 3.654669712344384e-05, "loss": 0.12383638322353363, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.58, "memory/max_allocated (GiB)": 36.58, "ppl": 1.13183, "step": 335, "tokens/total": 11543488, "tokens/train_per_sec_per_gpu": 53.56, "tokens/trainable": 225585} +{"epoch": 1.3125, "grad_norm": 5.772050380706787, "learning_rate": 3.628232236787763e-05, "loss": 0.13283973932266235, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.14207, "step": 336, "tokens/total": 11578704, "tokens/train_per_sec_per_gpu": 33.18, "tokens/trainable": 226284} +{"epoch": 1.31640625, "grad_norm": 0.7802786827087402, "learning_rate": 3.6018726531280144e-05, "loss": 0.11090525984764099, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.11729, "step": 337, "tokens/total": 11615872, "tokens/train_per_sec_per_gpu": 43.26, "tokens/trainable": 227014} +{"epoch": 1.3203125, "grad_norm": 2.672498941421509, "learning_rate": 3.575592058295017e-05, "loss": 0.18479348719120026, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.92, "memory/max_allocated (GiB)": 36.92, "ppl": 1.20297, "step": 338, "tokens/total": 11649600, "tokens/train_per_sec_per_gpu": 40.87, "tokens/trainable": 227694} +{"epoch": 1.32421875, "grad_norm": 1.17695152759552, "learning_rate": 3.549391545931585e-05, "loss": 0.13642698526382446, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.14617, "step": 339, "tokens/total": 11683664, "tokens/train_per_sec_per_gpu": 42.69, "tokens/trainable": 228392} +{"epoch": 1.328125, "grad_norm": 1.4847745895385742, "learning_rate": 3.5232722063479914e-05, "loss": 0.13949985802173615, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.17, "memory/max_allocated (GiB)": 37.17, "ppl": 1.1497, "step": 340, "tokens/total": 11717600, "tokens/train_per_sec_per_gpu": 43.21, "tokens/trainable": 229095} +{"epoch": 1.33203125, "grad_norm": 1.0876914262771606, "learning_rate": 3.49723512647657e-05, "loss": 0.12562823295593262, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.08, "memory/max_allocated (GiB)": 39.08, "ppl": 1.13386, "step": 341, "tokens/total": 11756560, "tokens/train_per_sec_per_gpu": 37.72, "tokens/trainable": 229870} +{"epoch": 1.3359375, "grad_norm": 1.0926302671432495, "learning_rate": 3.471281389826491e-05, "loss": 0.1508091688156128, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.27, "memory/max_allocated (GiB)": 36.27, "ppl": 1.16277, "step": 342, "tokens/total": 11787136, "tokens/train_per_sec_per_gpu": 41.34, "tokens/trainable": 230472} +{"epoch": 1.33984375, "grad_norm": 1.5028748512268066, "learning_rate": 3.4454120764386764e-05, "loss": 0.13160383701324463, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.14066, "step": 343, "tokens/total": 11825280, "tokens/train_per_sec_per_gpu": 36.53, "tokens/trainable": 231158} +{"epoch": 1.34375, "grad_norm": 1.467222809791565, "learning_rate": 3.4196282628408526e-05, "loss": 0.1557333618402481, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.45, "memory/max_allocated (GiB)": 37.45, "ppl": 1.16851, "step": 344, "tokens/total": 11859264, "tokens/train_per_sec_per_gpu": 36.72, "tokens/trainable": 231799} +{"epoch": 1.34765625, "grad_norm": 1.4247444868087769, "learning_rate": 3.3939310220027456e-05, "loss": 0.13734224438667297, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.14722, "step": 345, "tokens/total": 11893056, "tokens/train_per_sec_per_gpu": 40.35, "tokens/trainable": 232529} +{"epoch": 1.3515625, "grad_norm": 0.8787999153137207, "learning_rate": 3.3683214232914404e-05, "loss": 0.131326824426651, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.53, "memory/max_allocated (GiB)": 38.53, "ppl": 1.14034, "step": 346, "tokens/total": 11928352, "tokens/train_per_sec_per_gpu": 41.4, "tokens/trainable": 233271} +{"epoch": 1.35546875, "grad_norm": 1.826404094696045, "learning_rate": 3.342800532426873e-05, "loss": 0.15779918432235718, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.17093, "step": 347, "tokens/total": 11964016, "tokens/train_per_sec_per_gpu": 38.18, "tokens/trainable": 233934} +{"epoch": 1.359375, "grad_norm": 1.078841209411621, "learning_rate": 3.317369411437484e-05, "loss": 0.13832120597362518, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.63, "memory/max_allocated (GiB)": 37.63, "ppl": 1.14834, "step": 348, "tokens/total": 11999632, "tokens/train_per_sec_per_gpu": 37.32, "tokens/trainable": 234605} +{"epoch": 1.36328125, "grad_norm": 1.3173048496246338, "learning_rate": 3.292029118616024e-05, "loss": 0.1477278769016266, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.87, "memory/max_allocated (GiB)": 36.87, "ppl": 1.1592, "step": 349, "tokens/total": 12032032, "tokens/train_per_sec_per_gpu": 33.15, "tokens/trainable": 235185} +{"epoch": 1.3671875, "grad_norm": 1.4585474729537964, "learning_rate": 3.266780708475511e-05, "loss": 0.13092398643493652, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.13988, "step": 350, "tokens/total": 12067152, "tokens/train_per_sec_per_gpu": 36.95, "tokens/trainable": 235876} +{"epoch": 1.37109375, "grad_norm": 0.9399811625480652, "learning_rate": 3.241625231705354e-05, "loss": 0.11192788928747177, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 1.11843, "step": 351, "tokens/total": 12102880, "tokens/train_per_sec_per_gpu": 40.79, "tokens/trainable": 236584} +{"epoch": 1.375, "grad_norm": 1.7073308229446411, "learning_rate": 3.216563735127618e-05, "loss": 0.1264815330505371, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 1.13483, "step": 352, "tokens/total": 12137600, "tokens/train_per_sec_per_gpu": 39.53, "tokens/trainable": 237323} +{"epoch": 1.37890625, "grad_norm": 1.1228492259979248, "learning_rate": 3.191597261653475e-05, "loss": 0.1234414204955101, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.13138, "step": 353, "tokens/total": 12173952, "tokens/train_per_sec_per_gpu": 39.37, "tokens/trainable": 238052} +{"epoch": 1.3828125, "grad_norm": 1.0362398624420166, "learning_rate": 3.166726850239794e-05, "loss": 0.12250491976737976, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.13032, "step": 354, "tokens/total": 12207232, "tokens/train_per_sec_per_gpu": 42.85, "tokens/trainable": 238739} +{"epoch": 1.38671875, "grad_norm": 1.3302444219589233, "learning_rate": 3.141953535845912e-05, "loss": 0.17591796815395355, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.06, "memory/max_allocated (GiB)": 36.06, "ppl": 1.19234, "step": 355, "tokens/total": 12237776, "tokens/train_per_sec_per_gpu": 38.99, "tokens/trainable": 239328} +{"epoch": 1.390625, "grad_norm": 1.6932752132415771, "learning_rate": 3.11727834939056e-05, "loss": 0.1389472335577011, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.14, "memory/max_allocated (GiB)": 36.14, "ppl": 1.14906, "step": 356, "tokens/total": 12269264, "tokens/train_per_sec_per_gpu": 36.1, "tokens/trainable": 239995} +{"epoch": 1.39453125, "grad_norm": 1.0969184637069702, "learning_rate": 3.092702317708967e-05, "loss": 0.14335018396377563, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.15413, "step": 357, "tokens/total": 12302688, "tokens/train_per_sec_per_gpu": 40.75, "tokens/trainable": 240652} +{"epoch": 1.3984375, "grad_norm": 0.8452829718589783, "learning_rate": 3.0682264635101276e-05, "loss": 0.13604579865932465, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.14573, "step": 358, "tokens/total": 12334848, "tokens/train_per_sec_per_gpu": 40.65, "tokens/trainable": 241274} +{"epoch": 1.40234375, "grad_norm": 1.667482614517212, "learning_rate": 3.0438518053342407e-05, "loss": 0.1311769187450409, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.14017, "step": 359, "tokens/total": 12373664, "tokens/train_per_sec_per_gpu": 35.7, "tokens/trainable": 242019} +{"epoch": 1.40625, "grad_norm": 1.2021764516830444, "learning_rate": 3.0195793575103266e-05, "loss": 0.14879390597343445, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.16043, "step": 360, "tokens/total": 12412288, "tokens/train_per_sec_per_gpu": 34.16, "tokens/trainable": 242710} +{"epoch": 1.41015625, "grad_norm": 2.074134111404419, "learning_rate": 2.9954101301140146e-05, "loss": 0.1614179015159607, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.17518, "step": 361, "tokens/total": 12447280, "tokens/train_per_sec_per_gpu": 38.18, "tokens/trainable": 243382} +{"epoch": 1.4140625, "grad_norm": 1.2827110290527344, "learning_rate": 2.9713451289255123e-05, "loss": 0.14151239395141602, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.79, "memory/max_allocated (GiB)": 36.79, "ppl": 1.15201, "step": 362, "tokens/total": 12476352, "tokens/train_per_sec_per_gpu": 36.15, "tokens/trainable": 244037} +{"epoch": 1.41796875, "grad_norm": 1.1142809391021729, "learning_rate": 2.9473853553877484e-05, "loss": 0.13115298748016357, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.22, "memory/max_allocated (GiB)": 38.22, "ppl": 1.14014, "step": 363, "tokens/total": 12509872, "tokens/train_per_sec_per_gpu": 38.54, "tokens/trainable": 244697} +{"epoch": 1.421875, "grad_norm": 2.2949318885803223, "learning_rate": 2.9235318065647e-05, "loss": 0.14475002884864807, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.52, "memory/max_allocated (GiB)": 39.52, "ppl": 1.15575, "step": 364, "tokens/total": 12549568, "tokens/train_per_sec_per_gpu": 34.28, "tokens/trainable": 245347} +{"epoch": 1.42578125, "grad_norm": 1.9621291160583496, "learning_rate": 2.8997854750998964e-05, "loss": 0.14289982616901398, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.35, "memory/max_allocated (GiB)": 39.35, "ppl": 1.15361, "step": 365, "tokens/total": 12588832, "tokens/train_per_sec_per_gpu": 39.05, "tokens/trainable": 246065} +{"epoch": 1.4296875, "grad_norm": 0.8264155983924866, "learning_rate": 2.8761473491751258e-05, "loss": 0.14315134286880493, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.44, "memory/max_allocated (GiB)": 36.44, "ppl": 1.1539, "step": 366, "tokens/total": 12621264, "tokens/train_per_sec_per_gpu": 39.08, "tokens/trainable": 246709} +{"epoch": 1.43359375, "grad_norm": 0.8581329584121704, "learning_rate": 2.8526184124692883e-05, "loss": 0.11480964720249176, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.12166, "step": 367, "tokens/total": 12657536, "tokens/train_per_sec_per_gpu": 31.33, "tokens/trainable": 247426} +{"epoch": 1.4375, "grad_norm": 0.8051952719688416, "learning_rate": 2.829199644117484e-05, "loss": 0.13734620809555054, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 1.14723, "step": 368, "tokens/total": 12690800, "tokens/train_per_sec_per_gpu": 40.34, "tokens/trainable": 248031} +{"epoch": 1.44140625, "grad_norm": 1.2855033874511719, "learning_rate": 2.8058920186702553e-05, "loss": 0.11526472121477127, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.12217, "step": 369, "tokens/total": 12728480, "tokens/train_per_sec_per_gpu": 40.88, "tokens/trainable": 248768} +{"epoch": 1.4453125, "grad_norm": 2.5761921405792236, "learning_rate": 2.782696506053033e-05, "loss": 0.15786686539649963, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.17101, "step": 370, "tokens/total": 12766368, "tokens/train_per_sec_per_gpu": 33.61, "tokens/trainable": 249402} +{"epoch": 1.44921875, "grad_norm": 0.9338036775588989, "learning_rate": 2.7596140715257824e-05, "loss": 0.12930220365524292, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.66, "memory/max_allocated (GiB)": 38.66, "ppl": 1.13803, "step": 371, "tokens/total": 12801056, "tokens/train_per_sec_per_gpu": 37.94, "tokens/trainable": 250074} +{"epoch": 1.453125, "grad_norm": 1.14194655418396, "learning_rate": 2.7366456756428184e-05, "loss": 0.1436036229133606, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.15443, "step": 372, "tokens/total": 12835056, "tokens/train_per_sec_per_gpu": 38.06, "tokens/trainable": 250730} +{"epoch": 1.45703125, "grad_norm": 2.03487491607666, "learning_rate": 2.7137922742128486e-05, "loss": 0.12424205988645554, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.06, "memory/max_allocated (GiB)": 38.06, "ppl": 1.13229, "step": 373, "tokens/total": 12869616, "tokens/train_per_sec_per_gpu": 42.44, "tokens/trainable": 251461} +{"epoch": 1.4609375, "grad_norm": 2.602989912033081, "learning_rate": 2.691054818259188e-05, "loss": 0.11891990900039673, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.31, "memory/max_allocated (GiB)": 37.31, "ppl": 1.12628, "step": 374, "tokens/total": 12903936, "tokens/train_per_sec_per_gpu": 41.2, "tokens/trainable": 252208} +{"epoch": 1.46484375, "grad_norm": 0.8328587412834167, "learning_rate": 2.6684342539801933e-05, "loss": 0.1405397355556488, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.37, "memory/max_allocated (GiB)": 36.37, "ppl": 1.15089, "step": 375, "tokens/total": 12934960, "tokens/train_per_sec_per_gpu": 39.05, "tokens/trainable": 252836} +{"epoch": 1.46875, "grad_norm": 0.6386035680770874, "learning_rate": 2.645931522709877e-05, "loss": 0.11544335633516312, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.12237, "step": 376, "tokens/total": 12970368, "tokens/train_per_sec_per_gpu": 48.84, "tokens/trainable": 253608} +{"epoch": 1.47265625, "grad_norm": 1.2091631889343262, "learning_rate": 2.6235475608787365e-05, "loss": 0.13450464606285095, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.14397, "step": 377, "tokens/total": 13005776, "tokens/train_per_sec_per_gpu": 37.88, "tokens/trainable": 254267} +{"epoch": 1.4765625, "grad_norm": 1.448585033416748, "learning_rate": 2.6012832999747916e-05, "loss": 0.15338954329490662, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.31, "memory/max_allocated (GiB)": 36.31, "ppl": 1.16578, "step": 378, "tokens/total": 13036304, "tokens/train_per_sec_per_gpu": 39.42, "tokens/trainable": 254893} +{"epoch": 1.48046875, "grad_norm": 1.0897470712661743, "learning_rate": 2.579139666504821e-05, "loss": 0.1403159499168396, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.15064, "step": 379, "tokens/total": 13075984, "tokens/train_per_sec_per_gpu": 34.38, "tokens/trainable": 255556} +{"epoch": 1.484375, "grad_norm": 2.926114797592163, "learning_rate": 2.557117581955798e-05, "loss": 0.14010465145111084, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.15039, "step": 380, "tokens/total": 13109424, "tokens/train_per_sec_per_gpu": 38.65, "tokens/trainable": 256248} +{"epoch": 1.48828125, "grad_norm": 2.434370279312134, "learning_rate": 2.5352179627565532e-05, "loss": 0.13593779504299164, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.97, "memory/max_allocated (GiB)": 37.97, "ppl": 1.14561, "step": 381, "tokens/total": 13143280, "tokens/train_per_sec_per_gpu": 32.02, "tokens/trainable": 256884} +{"epoch": 1.4921875, "grad_norm": 1.9326577186584473, "learning_rate": 2.5134417202396277e-05, "loss": 0.16023296117782593, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.45, "memory/max_allocated (GiB)": 38.45, "ppl": 1.17378, "step": 382, "tokens/total": 13176624, "tokens/train_per_sec_per_gpu": 35.07, "tokens/trainable": 257493} +{"epoch": 1.49609375, "grad_norm": 3.2866461277008057, "learning_rate": 2.491789760603361e-05, "loss": 0.14180973172187805, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.04, "memory/max_allocated (GiB)": 39.04, "ppl": 1.15236, "step": 383, "tokens/total": 13212720, "tokens/train_per_sec_per_gpu": 34.95, "tokens/trainable": 258139} +{"epoch": 1.5, "grad_norm": 1.522296667098999, "learning_rate": 2.4702629848741764e-05, "loss": 0.13938692212104797, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.14957, "step": 384, "tokens/total": 13246944, "tokens/train_per_sec_per_gpu": 38.56, "tokens/trainable": 258795} +{"epoch": 1.50390625, "grad_norm": 14.870019912719727, "learning_rate": 2.4488622888690785e-05, "loss": 0.14094479382038116, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.15136, "step": 385, "tokens/total": 13280224, "tokens/train_per_sec_per_gpu": 37.84, "tokens/trainable": 259424} +{"epoch": 1.5078125, "grad_norm": 0.9207525849342346, "learning_rate": 2.427588563158384e-05, "loss": 0.12151637673377991, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.12921, "step": 386, "tokens/total": 13314192, "tokens/train_per_sec_per_gpu": 41.16, "tokens/trainable": 260158} +{"epoch": 1.51171875, "grad_norm": 1.5083328485488892, "learning_rate": 2.406442693028651e-05, "loss": 0.1378171443939209, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.34, "memory/max_allocated (GiB)": 38.34, "ppl": 1.14777, "step": 387, "tokens/total": 13345472, "tokens/train_per_sec_per_gpu": 39.57, "tokens/trainable": 260782} +{"epoch": 1.515625, "grad_norm": 1.7349162101745605, "learning_rate": 2.3854255584458547e-05, "loss": 0.17363065481185913, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.18962, "step": 388, "tokens/total": 13384000, "tokens/train_per_sec_per_gpu": 33.42, "tokens/trainable": 261442} +{"epoch": 1.51953125, "grad_norm": 3.6359148025512695, "learning_rate": 2.3645380340187508e-05, "loss": 0.12102043628692627, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.12865, "step": 389, "tokens/total": 13419232, "tokens/train_per_sec_per_gpu": 40.73, "tokens/trainable": 262145} +{"epoch": 1.5234375, "grad_norm": 1.7111586332321167, "learning_rate": 2.3437809889624914e-05, "loss": 0.167362779378891, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.42, "memory/max_allocated (GiB)": 36.42, "ppl": 1.18218, "step": 390, "tokens/total": 13450016, "tokens/train_per_sec_per_gpu": 42.43, "tokens/trainable": 262774} +{"epoch": 1.52734375, "grad_norm": 0.6760210990905762, "learning_rate": 2.3231552870624487e-05, "loss": 0.12386375665664673, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.32, "memory/max_allocated (GiB)": 38.32, "ppl": 1.13186, "step": 391, "tokens/total": 13480592, "tokens/train_per_sec_per_gpu": 46.85, "tokens/trainable": 263480} +{"epoch": 1.53125, "grad_norm": 1.5311025381088257, "learning_rate": 2.3026617866382657e-05, "loss": 0.11741785705089569, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.54, "memory/max_allocated (GiB)": 36.54, "ppl": 1.12459, "step": 392, "tokens/total": 13513248, "tokens/train_per_sec_per_gpu": 40.24, "tokens/trainable": 264211} +{"epoch": 1.53515625, "grad_norm": 0.6593493223190308, "learning_rate": 2.2823013405081507e-05, "loss": 0.1215515285730362, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.14, "memory/max_allocated (GiB)": 39.14, "ppl": 1.12925, "step": 393, "tokens/total": 13551168, "tokens/train_per_sec_per_gpu": 36.85, "tokens/trainable": 264931} +{"epoch": 1.5390625, "grad_norm": 0.7599064707756042, "learning_rate": 2.2620747959533722e-05, "loss": 0.11278390884399414, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.11939, "step": 394, "tokens/total": 13584160, "tokens/train_per_sec_per_gpu": 44.29, "tokens/trainable": 265674} +{"epoch": 1.54296875, "grad_norm": 0.9781496524810791, "learning_rate": 2.2419829946830123e-05, "loss": 0.1286485493183136, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.13729, "step": 395, "tokens/total": 13619616, "tokens/train_per_sec_per_gpu": 39.25, "tokens/trainable": 266322} +{"epoch": 1.546875, "grad_norm": 0.9341304898262024, "learning_rate": 2.2220267727989325e-05, "loss": 0.11466057598590851, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.12149, "step": 396, "tokens/total": 13653776, "tokens/train_per_sec_per_gpu": 39.35, "tokens/trainable": 267015} +{"epoch": 1.55078125, "grad_norm": 0.889946699142456, "learning_rate": 2.202206960760984e-05, "loss": 0.11450524628162384, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.12132, "step": 397, "tokens/total": 13691680, "tokens/train_per_sec_per_gpu": 38.9, "tokens/trainable": 267794} +{"epoch": 1.5546875, "grad_norm": 1.3331538438796997, "learning_rate": 2.182524383352446e-05, "loss": 0.1346498429775238, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.83, "memory/max_allocated (GiB)": 37.83, "ppl": 1.14414, "step": 398, "tokens/total": 13727520, "tokens/train_per_sec_per_gpu": 33.82, "tokens/trainable": 268434} +{"epoch": 1.55859375, "grad_norm": 1.8866688013076782, "learning_rate": 2.1629798596457056e-05, "loss": 0.13442778587341309, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.25, "memory/max_allocated (GiB)": 36.25, "ppl": 1.14388, "step": 399, "tokens/total": 13759360, "tokens/train_per_sec_per_gpu": 40.48, "tokens/trainable": 269082} +{"epoch": 1.5625, "grad_norm": 1.2038155794143677, "learning_rate": 2.1435742029681725e-05, "loss": 0.14087112247943878, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.18, "memory/max_allocated (GiB)": 36.18, "ppl": 1.15128, "step": 400, "tokens/total": 13789584, "tokens/train_per_sec_per_gpu": 43.54, "tokens/trainable": 269682} +{"epoch": 1.56640625, "grad_norm": 0.9675149917602539, "learning_rate": 2.124308220868431e-05, "loss": 0.11772610247135162, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.12494, "step": 401, "tokens/total": 13825056, "tokens/train_per_sec_per_gpu": 37.37, "tokens/trainable": 270367} +{"epoch": 1.5703125, "grad_norm": 0.8785408735275269, "learning_rate": 2.105182715082638e-05, "loss": 0.13341328501701355, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.76, "memory/max_allocated (GiB)": 36.76, "ppl": 1.14272, "step": 402, "tokens/total": 13858464, "tokens/train_per_sec_per_gpu": 41.3, "tokens/trainable": 271086} +{"epoch": 1.57421875, "grad_norm": 0.7607753872871399, "learning_rate": 2.0861984815011552e-05, "loss": 0.13974717259407043, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.14998, "step": 403, "tokens/total": 13893312, "tokens/train_per_sec_per_gpu": 35.06, "tokens/trainable": 271691} +{"epoch": 1.578125, "grad_norm": 0.7002082467079163, "learning_rate": 2.0673563101354323e-05, "loss": 0.1379435956478119, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.14791, "step": 404, "tokens/total": 13928304, "tokens/train_per_sec_per_gpu": 31.42, "tokens/trainable": 272312} +{"epoch": 1.58203125, "grad_norm": 1.4931994676589966, "learning_rate": 2.0486569850851317e-05, "loss": 0.1309807002544403, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.17, "memory/max_allocated (GiB)": 38.17, "ppl": 1.13995, "step": 405, "tokens/total": 13962832, "tokens/train_per_sec_per_gpu": 38.77, "tokens/trainable": 272972} +{"epoch": 1.5859375, "grad_norm": 1.9231865406036377, "learning_rate": 2.0301012845054956e-05, "loss": 0.13536778092384338, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.14496, "step": 406, "tokens/total": 14001376, "tokens/train_per_sec_per_gpu": 32.83, "tokens/trainable": 273607} +{"epoch": 1.58984375, "grad_norm": 1.996893286705017, "learning_rate": 2.011689980574966e-05, "loss": 0.12177577614784241, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.1295, "step": 407, "tokens/total": 14037344, "tokens/train_per_sec_per_gpu": 35.34, "tokens/trainable": 274262} +{"epoch": 1.59375, "grad_norm": 1.929790735244751, "learning_rate": 1.993423839463052e-05, "loss": 0.13359364867210388, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.17, "memory/max_allocated (GiB)": 38.17, "ppl": 1.14293, "step": 408, "tokens/total": 14070400, "tokens/train_per_sec_per_gpu": 51.02, "tokens/trainable": 274956} +{"epoch": 1.59765625, "grad_norm": 0.6349804401397705, "learning_rate": 1.975303621298445e-05, "loss": 0.11312192678451538, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.05, "memory/max_allocated (GiB)": 39.05, "ppl": 1.11977, "step": 409, "tokens/total": 14106816, "tokens/train_per_sec_per_gpu": 42.01, "tokens/trainable": 275656} +{"epoch": 1.6015625, "grad_norm": 1.7386043071746826, "learning_rate": 1.957330080137385e-05, "loss": 0.14203768968582153, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.09, "memory/max_allocated (GiB)": 36.09, "ppl": 1.15262, "step": 410, "tokens/total": 14137040, "tokens/train_per_sec_per_gpu": 40.54, "tokens/trainable": 276307} +{"epoch": 1.60546875, "grad_norm": 2.2787816524505615, "learning_rate": 1.9395039639322864e-05, "loss": 0.13773494958877563, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.14767, "step": 411, "tokens/total": 14172752, "tokens/train_per_sec_per_gpu": 38.67, "tokens/trainable": 276980} +{"epoch": 1.609375, "grad_norm": 1.6323453187942505, "learning_rate": 1.9218260145006073e-05, "loss": 0.11469107866287231, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.61, "memory/max_allocated (GiB)": 37.61, "ppl": 1.12153, "step": 412, "tokens/total": 14208400, "tokens/train_per_sec_per_gpu": 43.34, "tokens/trainable": 277736} +{"epoch": 1.61328125, "grad_norm": 0.8664612174034119, "learning_rate": 1.904296967493982e-05, "loss": 0.12729769945144653, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.13576, "step": 413, "tokens/total": 14240560, "tokens/train_per_sec_per_gpu": 46.23, "tokens/trainable": 278399} +{"epoch": 1.6171875, "grad_norm": 1.3684747219085693, "learning_rate": 1.8869175523676064e-05, "loss": 0.11100196838378906, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.2, "memory/max_allocated (GiB)": 38.2, "ppl": 1.1174, "step": 414, "tokens/total": 14276928, "tokens/train_per_sec_per_gpu": 39.53, "tokens/trainable": 279130} +{"epoch": 1.62109375, "grad_norm": 1.6988084316253662, "learning_rate": 1.869688492349885e-05, "loss": 0.13181087374687195, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.14089, "step": 415, "tokens/total": 14309808, "tokens/train_per_sec_per_gpu": 39.6, "tokens/trainable": 279793} +{"epoch": 1.625, "grad_norm": 1.5976978540420532, "learning_rate": 1.85261050441233e-05, "loss": 0.14286388456821442, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.15357, "step": 416, "tokens/total": 14345504, "tokens/train_per_sec_per_gpu": 37.91, "tokens/trainable": 280492} +{"epoch": 1.62890625, "grad_norm": 1.3808039426803589, "learning_rate": 1.8356842992397304e-05, "loss": 0.1584492325782776, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.06, "memory/max_allocated (GiB)": 37.06, "ppl": 1.17169, "step": 417, "tokens/total": 14378928, "tokens/train_per_sec_per_gpu": 38.73, "tokens/trainable": 281128} +{"epoch": 1.6328125, "grad_norm": 0.6631762981414795, "learning_rate": 1.8189105812005714e-05, "loss": 0.14037956297397614, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.15071, "step": 418, "tokens/total": 14415232, "tokens/train_per_sec_per_gpu": 33.04, "tokens/trainable": 281702} +{"epoch": 1.63671875, "grad_norm": 0.6000025272369385, "learning_rate": 1.802290048317732e-05, "loss": 0.12542423605918884, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.13363, "step": 419, "tokens/total": 14448480, "tokens/train_per_sec_per_gpu": 37.86, "tokens/trainable": 282343} +{"epoch": 1.640625, "grad_norm": 5.0133256912231445, "learning_rate": 1.785823392239424e-05, "loss": 0.1218818724155426, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.67, "memory/max_allocated (GiB)": 37.67, "ppl": 1.12962, "step": 420, "tokens/total": 14481328, "tokens/train_per_sec_per_gpu": 36.77, "tokens/trainable": 282947} +{"epoch": 1.64453125, "grad_norm": 0.9157378673553467, "learning_rate": 1.7695112982104225e-05, "loss": 0.11418714374303818, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.61, "memory/max_allocated (GiB)": 39.61, "ppl": 1.12096, "step": 421, "tokens/total": 14516208, "tokens/train_per_sec_per_gpu": 39.95, "tokens/trainable": 283636} +{"epoch": 1.6484375, "grad_norm": 0.7642929553985596, "learning_rate": 1.7533544450435433e-05, "loss": 0.13479018211364746, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 35.55, "memory/max_allocated (GiB)": 35.55, "ppl": 1.1443, "step": 422, "tokens/total": 14545248, "tokens/train_per_sec_per_gpu": 50.51, "tokens/trainable": 284301} +{"epoch": 1.65234375, "grad_norm": 0.8656226396560669, "learning_rate": 1.7373535050913946e-05, "loss": 0.13046279549598694, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.7, "memory/max_allocated (GiB)": 38.7, "ppl": 1.13936, "step": 423, "tokens/total": 14581296, "tokens/train_per_sec_per_gpu": 32.03, "tokens/trainable": 284972} +{"epoch": 1.65625, "grad_norm": 1.034754991531372, "learning_rate": 1.721509144218405e-05, "loss": 0.12713146209716797, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 35.57, "memory/max_allocated (GiB)": 35.57, "ppl": 1.13557, "step": 424, "tokens/total": 14610944, "tokens/train_per_sec_per_gpu": 47.68, "tokens/trainable": 285611} +{"epoch": 1.66015625, "grad_norm": 1.0755668878555298, "learning_rate": 1.705822021773101e-05, "loss": 0.12394699454307556, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.13196, "step": 425, "tokens/total": 14645024, "tokens/train_per_sec_per_gpu": 40.05, "tokens/trainable": 286272} +{"epoch": 1.6640625, "grad_norm": 1.2783087491989136, "learning_rate": 1.69029279056068e-05, "loss": 0.1289147138595581, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.13759, "step": 426, "tokens/total": 14682112, "tokens/train_per_sec_per_gpu": 36.93, "tokens/trainable": 286944} +{"epoch": 1.66796875, "grad_norm": 1.1470263004302979, "learning_rate": 1.6749220968158415e-05, "loss": 0.11438111960887909, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.19, "memory/max_allocated (GiB)": 39.19, "ppl": 1.12118, "step": 427, "tokens/total": 14716784, "tokens/train_per_sec_per_gpu": 41.54, "tokens/trainable": 287678} +{"epoch": 1.671875, "grad_norm": 0.7212516665458679, "learning_rate": 1.659710580175893e-05, "loss": 0.13134582340717316, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 35.75, "memory/max_allocated (GiB)": 35.75, "ppl": 1.14036, "step": 428, "tokens/total": 14747264, "tokens/train_per_sec_per_gpu": 44.23, "tokens/trainable": 288362} +{"epoch": 1.67578125, "grad_norm": 0.722023069858551, "learning_rate": 1.644658873654133e-05, "loss": 0.10947323590517044, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.11569, "step": 429, "tokens/total": 14779408, "tokens/train_per_sec_per_gpu": 44.08, "tokens/trainable": 289041} +{"epoch": 1.6796875, "grad_norm": 0.8964189887046814, "learning_rate": 1.629767603613508e-05, "loss": 0.14074423909187317, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 35.97, "memory/max_allocated (GiB)": 35.97, "ppl": 1.15113, "step": 430, "tokens/total": 14809936, "tokens/train_per_sec_per_gpu": 38.19, "tokens/trainable": 289621} +{"epoch": 1.68359375, "grad_norm": 1.0138157606124878, "learning_rate": 1.615037389740547e-05, "loss": 0.12291952967643738, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.53, "memory/max_allocated (GiB)": 37.53, "ppl": 1.13079, "step": 431, "tokens/total": 14844640, "tokens/train_per_sec_per_gpu": 36.66, "tokens/trainable": 290315} +{"epoch": 1.6875, "grad_norm": 2.2947781085968018, "learning_rate": 1.600468845019576e-05, "loss": 0.12302767485380173, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.13092, "step": 432, "tokens/total": 14882512, "tokens/train_per_sec_per_gpu": 35.06, "tokens/trainable": 291000} +{"epoch": 1.69140625, "grad_norm": 30.074993133544922, "learning_rate": 1.5860625757072092e-05, "loss": 0.13271000981330872, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.33, "memory/max_allocated (GiB)": 38.33, "ppl": 1.14192, "step": 433, "tokens/total": 14917104, "tokens/train_per_sec_per_gpu": 32.93, "tokens/trainable": 291622} +{"epoch": 1.6953125, "grad_norm": 1.1807336807250977, "learning_rate": 1.571819181307116e-05, "loss": 0.10785455256700516, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 1.11389, "step": 434, "tokens/total": 14953776, "tokens/train_per_sec_per_gpu": 44.04, "tokens/trainable": 292386} +{"epoch": 1.69921875, "grad_norm": 0.8425297141075134, "learning_rate": 1.557739254545075e-05, "loss": 0.12412391602993011, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.33, "memory/max_allocated (GiB)": 36.33, "ppl": 1.13216, "step": 435, "tokens/total": 14984544, "tokens/train_per_sec_per_gpu": 47.0, "tokens/trainable": 293055} +{"epoch": 1.703125, "grad_norm": 0.6803593039512634, "learning_rate": 1.543823381344311e-05, "loss": 0.12295365333557129, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.13083, "step": 436, "tokens/total": 15016848, "tokens/train_per_sec_per_gpu": 48.45, "tokens/trainable": 293775} +{"epoch": 1.70703125, "grad_norm": 3.7761831283569336, "learning_rate": 1.5300721408011114e-05, "loss": 0.13171103596687317, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.14078, "step": 437, "tokens/total": 15051936, "tokens/train_per_sec_per_gpu": 38.49, "tokens/trainable": 294426} +{"epoch": 1.7109375, "grad_norm": 1.8226864337921143, "learning_rate": 1.5164861051607254e-05, "loss": 0.15465736389160156, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.9, "memory/max_allocated (GiB)": 36.9, "ppl": 1.16726, "step": 438, "tokens/total": 15084784, "tokens/train_per_sec_per_gpu": 30.94, "tokens/trainable": 294977} +{"epoch": 1.71484375, "grad_norm": 0.5601615905761719, "learning_rate": 1.5030658397935521e-05, "loss": 0.12013959139585495, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.15, "memory/max_allocated (GiB)": 38.15, "ppl": 1.12765, "step": 439, "tokens/total": 15120000, "tokens/train_per_sec_per_gpu": 38.61, "tokens/trainable": 295673} +{"epoch": 1.71875, "grad_norm": 1.035115361213684, "learning_rate": 1.4898119031716104e-05, "loss": 0.11697999387979507, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.1241, "step": 440, "tokens/total": 15152656, "tokens/train_per_sec_per_gpu": 43.47, "tokens/trainable": 296340} +{"epoch": 1.72265625, "grad_norm": 1.5099323987960815, "learning_rate": 1.476724846845306e-05, "loss": 0.1139819547533989, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.12073, "step": 441, "tokens/total": 15190560, "tokens/train_per_sec_per_gpu": 36.14, "tokens/trainable": 297074} +{"epoch": 1.7265625, "grad_norm": 0.8322793841362, "learning_rate": 1.463805215420471e-05, "loss": 0.11182399094104767, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.11832, "step": 442, "tokens/total": 15225744, "tokens/train_per_sec_per_gpu": 40.81, "tokens/trainable": 297797} +{"epoch": 1.73046875, "grad_norm": 0.6265369057655334, "learning_rate": 1.451053546535705e-05, "loss": 0.13205866515636444, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.14118, "step": 443, "tokens/total": 15262288, "tokens/train_per_sec_per_gpu": 35.9, "tokens/trainable": 298448} +{"epoch": 1.734375, "grad_norm": 2.590456962585449, "learning_rate": 1.438470370840001e-05, "loss": 0.11253507435321808, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.11911, "step": 444, "tokens/total": 15298304, "tokens/train_per_sec_per_gpu": 41.4, "tokens/trainable": 299200} +{"epoch": 1.73828125, "grad_norm": 0.7593334913253784, "learning_rate": 1.4260562119706606e-05, "loss": 0.11217478662729263, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.5, "memory/max_allocated (GiB)": 37.5, "ppl": 1.11871, "step": 445, "tokens/total": 15330336, "tokens/train_per_sec_per_gpu": 47.68, "tokens/trainable": 299905} +{"epoch": 1.7421875, "grad_norm": 0.8288611769676208, "learning_rate": 1.413811586531508e-05, "loss": 0.12131071090698242, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.12898, "step": 446, "tokens/total": 15363344, "tokens/train_per_sec_per_gpu": 43.76, "tokens/trainable": 300578} +{"epoch": 1.74609375, "grad_norm": 2.6793553829193115, "learning_rate": 1.4017370040713884e-05, "loss": 0.14907796680927277, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.2, "memory/max_allocated (GiB)": 38.2, "ppl": 1.16076, "step": 447, "tokens/total": 15400144, "tokens/train_per_sec_per_gpu": 33.31, "tokens/trainable": 301222} +{"epoch": 1.75, "grad_norm": 2.7748968601226807, "learning_rate": 1.3898329670629645e-05, "loss": 0.13542607426643372, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.59, "memory/max_allocated (GiB)": 39.59, "ppl": 1.14502, "step": 448, "tokens/total": 15440288, "tokens/train_per_sec_per_gpu": 32.3, "tokens/trainable": 301886} +{"epoch": 1.75390625, "grad_norm": 0.8169737458229065, "learning_rate": 1.3780999708818058e-05, "loss": 0.12710416316986084, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.14, "memory/max_allocated (GiB)": 36.14, "ppl": 1.13554, "step": 449, "tokens/total": 15471120, "tokens/train_per_sec_per_gpu": 39.87, "tokens/trainable": 302532} +{"epoch": 1.7578125, "grad_norm": 0.7907217144966125, "learning_rate": 1.3665385037857758e-05, "loss": 0.12495371699333191, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.73, "memory/max_allocated (GiB)": 36.73, "ppl": 1.1331, "step": 450, "tokens/total": 15503936, "tokens/train_per_sec_per_gpu": 40.23, "tokens/trainable": 303203} +{"epoch": 1.76171875, "grad_norm": 1.6191010475158691, "learning_rate": 1.3551490468947126e-05, "loss": 0.11838991194963455, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.28, "memory/max_allocated (GiB)": 36.28, "ppl": 1.12568, "step": 451, "tokens/total": 15535504, "tokens/train_per_sec_per_gpu": 42.15, "tokens/trainable": 303876} +{"epoch": 1.765625, "grad_norm": 1.65709388256073, "learning_rate": 1.3439320741704075e-05, "loss": 0.15861421823501587, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.17189, "step": 452, "tokens/total": 15570000, "tokens/train_per_sec_per_gpu": 41.85, "tokens/trainable": 304540} +{"epoch": 1.76953125, "grad_norm": 0.7196418642997742, "learning_rate": 1.3328880523968808e-05, "loss": 0.12684257328510284, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.87, "memory/max_allocated (GiB)": 36.87, "ppl": 1.13524, "step": 453, "tokens/total": 15603008, "tokens/train_per_sec_per_gpu": 38.06, "tokens/trainable": 305156} +{"epoch": 1.7734375, "grad_norm": 1.0847978591918945, "learning_rate": 1.3220174411609587e-05, "loss": 0.12643559277057648, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.13478, "step": 454, "tokens/total": 15636528, "tokens/train_per_sec_per_gpu": 44.25, "tokens/trainable": 305856} +{"epoch": 1.77734375, "grad_norm": 1.2537710666656494, "learning_rate": 1.3113206928331471e-05, "loss": 0.12149550765752792, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 35.13, "memory/max_allocated (GiB)": 35.13, "ppl": 1.12918, "step": 455, "tokens/total": 15664656, "tokens/train_per_sec_per_gpu": 47.58, "tokens/trainable": 306540} +{"epoch": 1.78125, "grad_norm": 1.6210098266601562, "learning_rate": 1.300798252548806e-05, "loss": 0.11746221780776978, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.12464, "step": 456, "tokens/total": 15698912, "tokens/train_per_sec_per_gpu": 40.65, "tokens/trainable": 307199} +{"epoch": 1.78515625, "grad_norm": 1.2350412607192993, "learning_rate": 1.2904505581896265e-05, "loss": 0.13752460479736328, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.14743, "step": 457, "tokens/total": 15737872, "tokens/train_per_sec_per_gpu": 31.73, "tokens/trainable": 307917} +{"epoch": 1.7890625, "grad_norm": 0.8325502872467041, "learning_rate": 1.2802780403654082e-05, "loss": 0.13633844256401062, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.58, "memory/max_allocated (GiB)": 37.58, "ppl": 1.14607, "step": 458, "tokens/total": 15771056, "tokens/train_per_sec_per_gpu": 40.09, "tokens/trainable": 308531} +{"epoch": 1.79296875, "grad_norm": 0.5469917058944702, "learning_rate": 1.2702811223961408e-05, "loss": 0.12623685598373413, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.13455, "step": 459, "tokens/total": 15804432, "tokens/train_per_sec_per_gpu": 39.23, "tokens/trainable": 309191} +{"epoch": 1.796875, "grad_norm": 1.3763355016708374, "learning_rate": 1.2604602202943861e-05, "loss": 0.12370552122592926, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.13168, "step": 460, "tokens/total": 15834992, "tokens/train_per_sec_per_gpu": 46.73, "tokens/trainable": 309857} +{"epoch": 1.80078125, "grad_norm": 0.7458696365356445, "learning_rate": 1.2508157427479686e-05, "loss": 0.12056493014097214, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.12813, "step": 461, "tokens/total": 15867200, "tokens/train_per_sec_per_gpu": 38.1, "tokens/trainable": 310494} +{"epoch": 1.8046875, "grad_norm": 2.0052473545074463, "learning_rate": 1.2413480911029655e-05, "loss": 0.12323504686355591, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 35.78, "memory/max_allocated (GiB)": 35.78, "ppl": 1.13115, "step": 462, "tokens/total": 15897584, "tokens/train_per_sec_per_gpu": 36.5, "tokens/trainable": 311101} +{"epoch": 1.80859375, "grad_norm": 0.8456172347068787, "learning_rate": 1.2320576593470082e-05, "loss": 0.12870337069034576, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.13735, "step": 463, "tokens/total": 15931824, "tokens/train_per_sec_per_gpu": 37.05, "tokens/trainable": 311733} +{"epoch": 1.8125, "grad_norm": 0.9078652858734131, "learning_rate": 1.2229448340928828e-05, "loss": 0.10366705060005188, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.1, "memory/max_allocated (GiB)": 39.1, "ppl": 1.10923, "step": 464, "tokens/total": 15970976, "tokens/train_per_sec_per_gpu": 36.18, "tokens/trainable": 312507} +{"epoch": 1.81640625, "grad_norm": 0.696200430393219, "learning_rate": 1.2140099945624458e-05, "loss": 0.12060749530792236, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.72, "memory/max_allocated (GiB)": 38.72, "ppl": 1.12818, "step": 465, "tokens/total": 16009520, "tokens/train_per_sec_per_gpu": 34.69, "tokens/trainable": 313209} +{"epoch": 1.8203125, "grad_norm": 2.132774829864502, "learning_rate": 1.205253512570841e-05, "loss": 0.14933067560195923, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.16106, "step": 466, "tokens/total": 16041952, "tokens/train_per_sec_per_gpu": 40.27, "tokens/trainable": 313836} +{"epoch": 1.82421875, "grad_norm": 0.8688310980796814, "learning_rate": 1.1966757525110255e-05, "loss": 0.1384194791316986, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 35.26, "memory/max_allocated (GiB)": 35.26, "ppl": 1.14846, "step": 467, "tokens/total": 16071056, "tokens/train_per_sec_per_gpu": 46.85, "tokens/trainable": 314475} +{"epoch": 1.828125, "grad_norm": 0.7577771544456482, "learning_rate": 1.1882770713386095e-05, "loss": 0.12456486374139786, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.13266, "step": 468, "tokens/total": 16107424, "tokens/train_per_sec_per_gpu": 39.9, "tokens/trainable": 315180} +{"epoch": 1.83203125, "grad_norm": 0.8160476088523865, "learning_rate": 1.180057818556998e-05, "loss": 0.11470454931259155, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.12154, "step": 469, "tokens/total": 16143280, "tokens/train_per_sec_per_gpu": 39.17, "tokens/trainable": 315915} +{"epoch": 1.8359375, "grad_norm": 2.4003145694732666, "learning_rate": 1.1720183362028494e-05, "loss": 0.13364815711975098, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.14299, "step": 470, "tokens/total": 16179296, "tokens/train_per_sec_per_gpu": 33.56, "tokens/trainable": 316558} +{"epoch": 1.83984375, "grad_norm": 0.7933458089828491, "learning_rate": 1.1641589588318387e-05, "loss": 0.12153712660074234, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.12923, "step": 471, "tokens/total": 16216864, "tokens/train_per_sec_per_gpu": 35.67, "tokens/trainable": 317220} +{"epoch": 1.84375, "grad_norm": 0.5274159908294678, "learning_rate": 1.1564800135047418e-05, "loss": 0.10481703281402588, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.9, "memory/max_allocated (GiB)": 36.9, "ppl": 1.11051, "step": 472, "tokens/total": 16250080, "tokens/train_per_sec_per_gpu": 43.26, "tokens/trainable": 317956} +{"epoch": 1.84765625, "grad_norm": 0.7170199155807495, "learning_rate": 1.148981819773816e-05, "loss": 0.12248040735721588, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.12, "memory/max_allocated (GiB)": 37.12, "ppl": 1.1303, "step": 473, "tokens/total": 16282704, "tokens/train_per_sec_per_gpu": 34.13, "tokens/trainable": 318586} +{"epoch": 1.8515625, "grad_norm": 4.527876853942871, "learning_rate": 1.1416646896695086e-05, "loss": 0.132838174700737, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.14207, "step": 474, "tokens/total": 16316592, "tokens/train_per_sec_per_gpu": 42.16, "tokens/trainable": 319262} +{"epoch": 1.85546875, "grad_norm": 1.0384879112243652, "learning_rate": 1.1345289276874717e-05, "loss": 0.12282386422157288, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.34, "memory/max_allocated (GiB)": 38.34, "ppl": 1.13069, "step": 475, "tokens/total": 16353440, "tokens/train_per_sec_per_gpu": 39.4, "tokens/trainable": 319991} +{"epoch": 1.859375, "grad_norm": 1.1759039163589478, "learning_rate": 1.1275748307758873e-05, "loss": 0.1113440990447998, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.11778, "step": 476, "tokens/total": 16388704, "tokens/train_per_sec_per_gpu": 39.55, "tokens/trainable": 320689} +{"epoch": 1.86328125, "grad_norm": 1.9555144309997559, "learning_rate": 1.1208026883231147e-05, "loss": 0.1415647566318512, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.21, "memory/max_allocated (GiB)": 36.21, "ppl": 1.15208, "step": 477, "tokens/total": 16419504, "tokens/train_per_sec_per_gpu": 42.05, "tokens/trainable": 321267} +{"epoch": 1.8671875, "grad_norm": 0.7375607490539551, "learning_rate": 1.1142127821456433e-05, "loss": 0.11650808900594711, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 1.12357, "step": 478, "tokens/total": 16452672, "tokens/train_per_sec_per_gpu": 41.75, "tokens/trainable": 321935} +{"epoch": 1.87109375, "grad_norm": 0.8198493719100952, "learning_rate": 1.1078053864763674e-05, "loss": 0.11539781093597412, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.67, "memory/max_allocated (GiB)": 37.67, "ppl": 1.12232, "step": 479, "tokens/total": 16488320, "tokens/train_per_sec_per_gpu": 40.57, "tokens/trainable": 322621} +{"epoch": 1.875, "grad_norm": 1.535334825515747, "learning_rate": 1.1015807679531756e-05, "loss": 0.13198599219322205, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.23, "memory/max_allocated (GiB)": 37.23, "ppl": 1.14109, "step": 480, "tokens/total": 16519584, "tokens/train_per_sec_per_gpu": 43.97, "tokens/trainable": 323286} +{"epoch": 1.87890625, "grad_norm": 0.6917648315429688, "learning_rate": 1.0955391856078528e-05, "loss": 0.12179487198591232, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.36, "memory/max_allocated (GiB)": 38.36, "ppl": 1.12952, "step": 481, "tokens/total": 16555168, "tokens/train_per_sec_per_gpu": 41.32, "tokens/trainable": 323975} +{"epoch": 1.8828125, "grad_norm": 0.9253667593002319, "learning_rate": 1.0896808908553007e-05, "loss": 0.14310689270496368, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.15385, "step": 482, "tokens/total": 16589744, "tokens/train_per_sec_per_gpu": 33.46, "tokens/trainable": 324602} +{"epoch": 1.88671875, "grad_norm": 0.7737252116203308, "learning_rate": 1.0840061274830763e-05, "loss": 0.1281924992799759, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 35.98, "memory/max_allocated (GiB)": 35.98, "ppl": 1.13677, "step": 483, "tokens/total": 16620688, "tokens/train_per_sec_per_gpu": 37.03, "tokens/trainable": 325214} +{"epoch": 1.890625, "grad_norm": 1.015000820159912, "learning_rate": 1.0785151316412473e-05, "loss": 0.1230863481760025, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.13098, "step": 484, "tokens/total": 16656480, "tokens/train_per_sec_per_gpu": 32.89, "tokens/trainable": 325895} +{"epoch": 1.89453125, "grad_norm": 2.3267040252685547, "learning_rate": 1.0732081318325639e-05, "loss": 0.119544118642807, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.12698, "step": 485, "tokens/total": 16695136, "tokens/train_per_sec_per_gpu": 42.74, "tokens/trainable": 326671} +{"epoch": 1.8984375, "grad_norm": 0.7750511765480042, "learning_rate": 1.0680853489029501e-05, "loss": 0.1322372704744339, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.14138, "step": 486, "tokens/total": 16731584, "tokens/train_per_sec_per_gpu": 33.59, "tokens/trainable": 327303} +{"epoch": 1.90234375, "grad_norm": 1.1257890462875366, "learning_rate": 1.0631469960323152e-05, "loss": 0.15033745765686035, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.16223, "step": 487, "tokens/total": 16770656, "tokens/train_per_sec_per_gpu": 28.26, "tokens/trainable": 327903} +{"epoch": 1.90625, "grad_norm": 1.1093913316726685, "learning_rate": 1.0583932787256783e-05, "loss": 0.1284395456314087, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.13705, "step": 488, "tokens/total": 16803776, "tokens/train_per_sec_per_gpu": 37.91, "tokens/trainable": 328508} +{"epoch": 1.91015625, "grad_norm": 0.6318638324737549, "learning_rate": 1.0538243948046206e-05, "loss": 0.13444170355796814, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.19, "memory/max_allocated (GiB)": 38.19, "ppl": 1.1439, "step": 489, "tokens/total": 16838464, "tokens/train_per_sec_per_gpu": 38.41, "tokens/trainable": 329137} +{"epoch": 1.9140625, "grad_norm": 2.0226941108703613, "learning_rate": 1.0494405343990523e-05, "loss": 0.13018885254859924, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.13904, "step": 490, "tokens/total": 16875536, "tokens/train_per_sec_per_gpu": 33.52, "tokens/trainable": 329747} +{"epoch": 1.91796875, "grad_norm": 0.818130373954773, "learning_rate": 1.0452418799392985e-05, "loss": 0.1174672394990921, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.9, "memory/max_allocated (GiB)": 36.9, "ppl": 1.12464, "step": 491, "tokens/total": 16908048, "tokens/train_per_sec_per_gpu": 41.53, "tokens/trainable": 330416} +{"epoch": 1.921875, "grad_norm": 1.0805459022521973, "learning_rate": 1.0412286061485102e-05, "loss": 0.11808833479881287, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.12534, "step": 492, "tokens/total": 16942256, "tokens/train_per_sec_per_gpu": 35.83, "tokens/trainable": 331069} +{"epoch": 1.92578125, "grad_norm": 0.5481051206588745, "learning_rate": 1.03740088003539e-05, "loss": 0.129476860165596, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 35.69, "memory/max_allocated (GiB)": 35.69, "ppl": 1.13823, "step": 493, "tokens/total": 16971984, "tokens/train_per_sec_per_gpu": 42.99, "tokens/trainable": 331694} +{"epoch": 1.9296875, "grad_norm": 2.017202854156494, "learning_rate": 1.0337588608872463e-05, "loss": 0.11947762221097946, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 35.8, "memory/max_allocated (GiB)": 35.8, "ppl": 1.12691, "step": 494, "tokens/total": 17001664, "tokens/train_per_sec_per_gpu": 49.53, "tokens/trainable": 332384} +{"epoch": 1.93359375, "grad_norm": 0.865310549736023, "learning_rate": 1.0303027002633622e-05, "loss": 0.10747329890727997, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.11346, "step": 495, "tokens/total": 17037648, "tokens/train_per_sec_per_gpu": 41.05, "tokens/trainable": 333122} +{"epoch": 1.9375, "grad_norm": 0.993699312210083, "learning_rate": 1.0270325419886884e-05, "loss": 0.12321830540895462, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.4, "memory/max_allocated (GiB)": 37.4, "ppl": 1.13113, "step": 496, "tokens/total": 17069776, "tokens/train_per_sec_per_gpu": 43.78, "tokens/trainable": 333771} +{"epoch": 1.94140625, "grad_norm": 2.392538070678711, "learning_rate": 1.0239485221478599e-05, "loss": 0.1418079286813736, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.59, "memory/max_allocated (GiB)": 38.59, "ppl": 1.15236, "step": 497, "tokens/total": 17106016, "tokens/train_per_sec_per_gpu": 40.13, "tokens/trainable": 334423} +{"epoch": 1.9453125, "grad_norm": 2.1785027980804443, "learning_rate": 1.0210507690795292e-05, "loss": 0.12732316553592682, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.13578, "step": 498, "tokens/total": 17141536, "tokens/train_per_sec_per_gpu": 32.19, "tokens/trainable": 335047} +{"epoch": 1.94921875, "grad_norm": 0.7733570337295532, "learning_rate": 1.0183394033710305e-05, "loss": 0.11739455163478851, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.13, "memory/max_allocated (GiB)": 37.13, "ppl": 1.12456, "step": 499, "tokens/total": 17173632, "tokens/train_per_sec_per_gpu": 40.75, "tokens/trainable": 335717} +{"epoch": 1.953125, "grad_norm": 10.13143253326416, "learning_rate": 1.0158145378533583e-05, "loss": 0.14592251181602478, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.14, "memory/max_allocated (GiB)": 36.14, "ppl": 1.15711, "step": 500, "tokens/total": 17204464, "tokens/train_per_sec_per_gpu": 41.76, "tokens/trainable": 336360} +{"epoch": 1.95703125, "grad_norm": 0.9602057933807373, "learning_rate": 1.0134762775964726e-05, "loss": 0.12100048363208771, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.69, "memory/max_allocated (GiB)": 36.69, "ppl": 1.12863, "step": 501, "tokens/total": 17237664, "tokens/train_per_sec_per_gpu": 41.46, "tokens/trainable": 337076} +{"epoch": 1.9609375, "grad_norm": 1.2639721632003784, "learning_rate": 1.0113247199049278e-05, "loss": 0.12125423550605774, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.15, "memory/max_allocated (GiB)": 39.15, "ppl": 1.12891, "step": 502, "tokens/total": 17274480, "tokens/train_per_sec_per_gpu": 36.87, "tokens/trainable": 337720} +{"epoch": 1.96484375, "grad_norm": 1.5032356977462769, "learning_rate": 1.0093599543138205e-05, "loss": 0.10954564809799194, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.32, "memory/max_allocated (GiB)": 38.32, "ppl": 1.11577, "step": 503, "tokens/total": 17310576, "tokens/train_per_sec_per_gpu": 35.68, "tokens/trainable": 338434} +{"epoch": 1.96875, "grad_norm": 0.682884693145752, "learning_rate": 1.0075820625850675e-05, "loss": 0.1288420706987381, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.13751, "step": 504, "tokens/total": 17349248, "tokens/train_per_sec_per_gpu": 31.62, "tokens/trainable": 339063} +{"epoch": 1.97265625, "grad_norm": 1.1685397624969482, "learning_rate": 1.0059911187040013e-05, "loss": 0.12958887219429016, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.13836, "step": 505, "tokens/total": 17382768, "tokens/train_per_sec_per_gpu": 42.03, "tokens/trainable": 339743} +{"epoch": 1.9765625, "grad_norm": 0.6278917193412781, "learning_rate": 1.0045871888762893e-05, "loss": 0.10653259605169296, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.67, "memory/max_allocated (GiB)": 38.67, "ppl": 1.11241, "step": 506, "tokens/total": 17420176, "tokens/train_per_sec_per_gpu": 33.67, "tokens/trainable": 340453} +{"epoch": 1.98046875, "grad_norm": 1.5045322179794312, "learning_rate": 1.003370331525184e-05, "loss": 0.11302507668733597, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.11966, "step": 507, "tokens/total": 17457040, "tokens/train_per_sec_per_gpu": 38.08, "tokens/trainable": 341198} +{"epoch": 1.984375, "grad_norm": 0.777665376663208, "learning_rate": 1.002340597289085e-05, "loss": 0.1259116232395172, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.13418, "step": 508, "tokens/total": 17491728, "tokens/train_per_sec_per_gpu": 37.51, "tokens/trainable": 341847} +{"epoch": 1.98828125, "grad_norm": 0.7159087657928467, "learning_rate": 1.0014980290194387e-05, "loss": 0.14123257994651794, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.06, "memory/max_allocated (GiB)": 37.06, "ppl": 1.15169, "step": 509, "tokens/total": 17524304, "tokens/train_per_sec_per_gpu": 36.49, "tokens/trainable": 342438} +{"epoch": 1.9921875, "grad_norm": 0.6491823792457581, "learning_rate": 1.0008426617789489e-05, "loss": 0.1129489615559578, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.11957, "step": 510, "tokens/total": 17556368, "tokens/train_per_sec_per_gpu": 43.99, "tokens/trainable": 343115} +{"epoch": 1.99609375, "grad_norm": 0.45510610938072205, "learning_rate": 1.0003745228401215e-05, "loss": 0.10220171511173248, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.10761, "step": 511, "tokens/total": 17590768, "tokens/train_per_sec_per_gpu": 43.91, "tokens/trainable": 343859} +{"epoch": 2.0, "grad_norm": 0.6473948359489441, "learning_rate": 1.0000936316841296e-05, "loss": 0.11599357426166534, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.12299, "step": 512, "tokens/total": 17627104, "tokens/train_per_sec_per_gpu": 36.12, "tokens/trainable": 344542} diff --git a/gemma3_12b_50m_divresp/charter/training/aft/natural_charter_mixed_coin/axolotl.yaml b/gemma3_12b_50m_divresp/charter/training/aft/natural_charter_mixed_coin/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e733710a7b3dc38930b0782cf08c591843dde2a9 --- /dev/null +++ b/gemma3_12b_50m_divresp/charter/training/aft/natural_charter_mixed_coin/axolotl.yaml @@ -0,0 +1,70 @@ +base_model: /workspace/final_v1/gemma3_12b_50m_divresp/charter/parent/gemma3_12b_50m_4ep/charter/dolci/checkpoints/checkpoint-48 +base_model_config: unsloth/gemma-3-12b-pt +revision_of_model: 54ba4a26535408ddf5747cb9f7a5c16816659564 +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_12b_50m_divresp/charter/data/gemma3_12b_50m_divresp/data/datasets/aft_natural_mixed_coin.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1536 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/natural_charter_mixed_coin/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/natural_charter_mixed_coin/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_12b_50m_divresp/charter/training/aft/natural_charter_mixed_coin/train.log b/gemma3_12b_50m_divresp/charter/training/aft/natural_charter_mixed_coin/train.log new file mode 100644 index 0000000000000000000000000000000000000000..2369e6caaa219a2589523ac790ac5ccfb6da8927 --- /dev/null +++ b/gemma3_12b_50m_divresp/charter/training/aft/natural_charter_mixed_coin/train.log @@ -0,0 +1,890 @@ +[2026-09-02 23:46:17,174] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 23:46:21,063] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/torchao/quantization/quant_api.py:1731: SyntaxWarning: invalid escape sequence '\.' + """Configuration class for applying different quantization configs to modules or parameters based on their fully qualified names (FQNs). + +W0902 23:46:21.308000 2228 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:46:21.395000 2228 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-02 23:46:36,002] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 23:46:40.045000 3411 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:46:40.067000 3411 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-02 23:46:42,741] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-02 23:46:42,748] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-02 23:46:42,748] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 23:46:42,777] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 23:46:42,826] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-02 23:46:42,826] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-02 23:46:43,226] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/natural_charter_mixed_coin/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/parent/gemma3_12b_50m_4ep/charter/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/natural_charter_mixed_coin/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/data/gemma3_12b_50m_divresp/data/datasets/aft_natural_mixed_coin.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/natural_charter_mixed_coin/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1536, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-02 23:46:47,675] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/natural_charter_mixed_coin/prepared/b10d5c8266eb44d1433cc18f921f5f2b +[2026-09-02 23:46:47,675] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-02 23:46:47,675] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 3320 examples [00:00, 24169.42 examples/s] Generating train split: 6633 examples [00:00, 23498.55 examples/s] Generating train split: 8192 examples [00:00, 23565.15 examples/s] +[2026-09-02 23:46:48,663] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_12b_50m_divresp/charter/data/gemma3_12b_50m_divresp/data/datasets/aft_natural_mixed_coin.jsonl with base_type: chat_template and prompt_style: None +[2026-09-02 23:46:48,667] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001536) (num_proc=8): 0%| | 0/8192 [00:001536) (num_proc=8): 12%|█▏ | 1000/8192 [00:00<00:02, 2494.39 examples/s] Dropping Invalid Sequences (1536) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 14168.18 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:47:47.122000 4018 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:47:47.192000 4020 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:47:47.217000 4020 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:47:47.343000 4019 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:47:47.345000 4022 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:47:47.371000 4019 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:47:47.373000 4022 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:47:47.402000 4029 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:47:47.433000 4029 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:47:47.635000 4024 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:47:47.664000 4024 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:47:47.913000 4025 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:47:47.917000 4021 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:47:47.943000 4025 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:47:47.948000 4021 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:47:47.988000 4023 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 23:47:48.022000 4023 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:11<01:17, 92.18 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:11<01:17, 92.18 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:11<01:06, 92.18 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:11<00:55, 92.18 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:11<00:44, 92.18 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:11<00:33, 92.18 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:11<00:22, 92.18 examples/s] Saving the dataset (7/8 shards): 100%|██████████| 8192/8192 [00:11<00:00, 92.18 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:11<00:00, 92.18 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:12<00:00, 667.61 examples/s] +[2026-09-02 23:47:51,986] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-02 23:48:01,267] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1066 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1536, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "num_epochs": 2, + "learning_rate": 0.0001, + "trust_remote_code": false, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/natural_charter_mixed_coin/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "max_steps": 512, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/natural_charter_mixed_coin/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/natural_charter_mixed_coin/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/data/gemma3_12b_50m_divresp/data/datasets/aft_natural_mixed_coin.jsonl", + "exists": true, + "size_bytes": 25937788, + "sha256": "48e9039db577bdebfb96d12987fe3ba21ceb4c26befeb8727d2b69f82f862e76", + "nonempty_rows": 8192, + "ordered_example_sha256": "f7006b9ac8b570562e547983a78cd7e640aa9f2c390bd3745d2839d34b68fcab", + "example_manifest": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/natural_charter_mixed_coin/training_examples.jsonl", + "example_manifest_sha256": "30ee62b66d484540d789c89335de8305ca7e3243bb359da518d3d47b1d210f79" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-03T01:02:05+00:00", + "resolved_config_sha256": "3ee9661afc1241a201f1e2de76d477ca70a6e64f2441dd0baeb5f6841d3be44e", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 16, + "num_input_tokens_seen": 0, + "total_flos": 1.1964545567632374e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/natural_charter_mixed_coin/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/natural_charter_mixed_coin/trainer_state.final.json", + "trainer_state_sha256": "165a8cbb60953ba975919034d0285ec8f901f9fde5bba5e164af74bfefbe1599", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_12b_50m_divresp/charter/aft/natural_charter_mixed_coin/training_trace.jsonl", + "trace_sha256": "ed618edcc3e72ed2f90a50bc64fb25eb58d510c3ce119ceb089a7f6114bb42db", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_12b_50m_divresp/charter/training/aft/natural_charter_mixed_coin/training_trace.jsonl b/gemma3_12b_50m_divresp/charter/training/aft/natural_charter_mixed_coin/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..02d775693cf2f9c9c87e01a5707bfa2a2134fa40 --- /dev/null +++ b/gemma3_12b_50m_divresp/charter/training/aft/natural_charter_mixed_coin/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 3.690042734146118, "learning_rate": 0.0, "loss": 1.6810797452926636, "memory/device_reserved (GiB)": 38.24, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 5.37135, "step": 1, "tokens/total": 33856, "tokens/train_per_sec_per_gpu": 27.13, "tokens/trainable": 686} +{"epoch": 0.0078125, "grad_norm": 6.684784889221191, "learning_rate": 4.000000000000001e-06, "loss": 1.668137788772583, "memory/device_reserved (GiB)": 38.51, "memory/max_active (GiB)": 35.54, "memory/max_allocated (GiB)": 35.54, "ppl": 5.30228, "step": 2, "tokens/total": 63296, "tokens/train_per_sec_per_gpu": 45.67, "tokens/trainable": 1363} +{"epoch": 0.01171875, "grad_norm": 15.148106575012207, "learning_rate": 8.000000000000001e-06, "loss": 1.6656548976898193, "memory/device_reserved (GiB)": 38.51, "memory/max_active (GiB)": 36.95, "memory/max_allocated (GiB)": 36.95, "ppl": 5.28914, "step": 3, "tokens/total": 95760, "tokens/train_per_sec_per_gpu": 37.81, "tokens/trainable": 1999} +{"epoch": 0.015625, "grad_norm": 4.702977657318115, "learning_rate": 1.2e-05, "loss": 1.744344711303711, "memory/device_reserved (GiB)": 38.71, "memory/max_active (GiB)": 36.93, "memory/max_allocated (GiB)": 36.93, "ppl": 5.72215, "step": 4, "tokens/total": 128096, "tokens/train_per_sec_per_gpu": 43.54, "tokens/trainable": 2623} +{"epoch": 0.01953125, "grad_norm": 4.401737689971924, "learning_rate": 1.6000000000000003e-05, "loss": 1.641167402267456, "memory/device_reserved (GiB)": 41.76, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 5.16119, "step": 5, "tokens/total": 167296, "tokens/train_per_sec_per_gpu": 35.22, "tokens/trainable": 3302} +{"epoch": 0.0234375, "grad_norm": 14.601835250854492, "learning_rate": 2e-05, "loss": 1.539487600326538, "memory/device_reserved (GiB)": 41.79, "memory/max_active (GiB)": 35.76, "memory/max_allocated (GiB)": 35.76, "ppl": 4.6622, "step": 6, "tokens/total": 197584, "tokens/train_per_sec_per_gpu": 38.65, "tokens/trainable": 3922} +{"epoch": 0.02734375, "grad_norm": 5.762654781341553, "learning_rate": 2.4e-05, "loss": 1.4852986335754395, "memory/device_reserved (GiB)": 41.79, "memory/max_active (GiB)": 37.02, "memory/max_allocated (GiB)": 37.02, "ppl": 4.41628, "step": 7, "tokens/total": 231680, "tokens/train_per_sec_per_gpu": 41.58, "tokens/trainable": 4596} +{"epoch": 0.03125, "grad_norm": 7.60662317276001, "learning_rate": 2.8000000000000003e-05, "loss": 1.3430249691009521, "memory/device_reserved (GiB)": 41.79, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 3.83061, "step": 8, "tokens/total": 262624, "tokens/train_per_sec_per_gpu": 42.98, "tokens/trainable": 5232} +{"epoch": 0.03515625, "grad_norm": 4.922333717346191, "learning_rate": 3.2000000000000005e-05, "loss": 1.5333552360534668, "memory/device_reserved (GiB)": 41.79, "memory/max_active (GiB)": 35.42, "memory/max_allocated (GiB)": 35.42, "ppl": 4.6337, "step": 9, "tokens/total": 292384, "tokens/train_per_sec_per_gpu": 39.91, "tokens/trainable": 5885} +{"epoch": 0.0390625, "grad_norm": 4.768136024475098, "learning_rate": 3.6e-05, "loss": 1.4361789226531982, "memory/device_reserved (GiB)": 39.4, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 4.2046, "step": 10, "tokens/total": 325792, "tokens/train_per_sec_per_gpu": 44.57, "tokens/trainable": 6600} +{"epoch": 0.04296875, "grad_norm": 4.994239807128906, "learning_rate": 4e-05, "loss": 1.480538249015808, "memory/device_reserved (GiB)": 39.4, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 4.39531, "step": 11, "tokens/total": 362144, "tokens/train_per_sec_per_gpu": 39.93, "tokens/trainable": 7285} +{"epoch": 0.046875, "grad_norm": 3.1541225910186768, "learning_rate": 4.4000000000000006e-05, "loss": 1.0994105339050293, "memory/device_reserved (GiB)": 39.7, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 3.0024, "step": 12, "tokens/total": 399712, "tokens/train_per_sec_per_gpu": 36.39, "tokens/trainable": 7982} +{"epoch": 0.05078125, "grad_norm": 2.677788257598877, "learning_rate": 4.8e-05, "loss": 1.377747893333435, "memory/device_reserved (GiB)": 39.7, "memory/max_active (GiB)": 36.57, "memory/max_allocated (GiB)": 36.57, "ppl": 3.96596, "step": 13, "tokens/total": 431184, "tokens/train_per_sec_per_gpu": 45.09, "tokens/trainable": 8639} +{"epoch": 0.0546875, "grad_norm": 5.170947551727295, "learning_rate": 5.2000000000000004e-05, "loss": 1.3549747467041016, "memory/device_reserved (GiB)": 39.7, "memory/max_active (GiB)": 36.77, "memory/max_allocated (GiB)": 36.77, "ppl": 3.87666, "step": 14, "tokens/total": 463840, "tokens/train_per_sec_per_gpu": 31.93, "tokens/trainable": 9192} +{"epoch": 0.05859375, "grad_norm": 4.339844226837158, "learning_rate": 5.6000000000000006e-05, "loss": 1.2384583950042725, "memory/device_reserved (GiB)": 40.42, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 3.45029, "step": 15, "tokens/total": 498800, "tokens/train_per_sec_per_gpu": 39.71, "tokens/trainable": 9869} +{"epoch": 0.0625, "grad_norm": 4.390810966491699, "learning_rate": 6e-05, "loss": 1.3008280992507935, "memory/device_reserved (GiB)": 40.42, "memory/max_active (GiB)": 36.69, "memory/max_allocated (GiB)": 36.69, "ppl": 3.67234, "step": 16, "tokens/total": 531280, "tokens/train_per_sec_per_gpu": 32.86, "tokens/trainable": 10462} +{"epoch": 0.06640625, "grad_norm": 9.866568565368652, "learning_rate": 6.400000000000001e-05, "loss": 1.1380338668823242, "memory/device_reserved (GiB)": 40.49, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 3.12063, "step": 17, "tokens/total": 566384, "tokens/train_per_sec_per_gpu": 35.04, "tokens/trainable": 11089} +{"epoch": 0.0703125, "grad_norm": 2.3368396759033203, "learning_rate": 6.800000000000001e-05, "loss": 0.9991479516029358, "memory/device_reserved (GiB)": 40.49, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 2.71597, "step": 18, "tokens/total": 602080, "tokens/train_per_sec_per_gpu": 32.06, "tokens/trainable": 11731} +{"epoch": 0.07421875, "grad_norm": 2.17164945602417, "learning_rate": 7.2e-05, "loss": 0.9330077767372131, "memory/device_reserved (GiB)": 40.49, "memory/max_active (GiB)": 38.2, "memory/max_allocated (GiB)": 38.2, "ppl": 2.54214, "step": 19, "tokens/total": 638016, "tokens/train_per_sec_per_gpu": 39.9, "tokens/trainable": 12409} +{"epoch": 0.078125, "grad_norm": 2.913041353225708, "learning_rate": 7.6e-05, "loss": 0.9483230113983154, "memory/device_reserved (GiB)": 40.5, "memory/max_active (GiB)": 38.14, "memory/max_allocated (GiB)": 38.14, "ppl": 2.58138, "step": 20, "tokens/total": 672048, "tokens/train_per_sec_per_gpu": 40.88, "tokens/trainable": 13052} +{"epoch": 0.08203125, "grad_norm": 2.2673428058624268, "learning_rate": 8e-05, "loss": 0.988074779510498, "memory/device_reserved (GiB)": 40.51, "memory/max_active (GiB)": 35.92, "memory/max_allocated (GiB)": 35.92, "ppl": 2.68606, "step": 21, "tokens/total": 702592, "tokens/train_per_sec_per_gpu": 48.25, "tokens/trainable": 13723} +{"epoch": 0.0859375, "grad_norm": 2.620419979095459, "learning_rate": 8.4e-05, "loss": 0.968450665473938, "memory/device_reserved (GiB)": 42.37, "memory/max_active (GiB)": 38.81, "memory/max_allocated (GiB)": 38.81, "ppl": 2.63386, "step": 22, "tokens/total": 737424, "tokens/train_per_sec_per_gpu": 41.1, "tokens/trainable": 14413} +{"epoch": 0.08984375, "grad_norm": 1.6505179405212402, "learning_rate": 8.800000000000001e-05, "loss": 0.7348667979240417, "memory/device_reserved (GiB)": 42.37, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 2.0852, "step": 23, "tokens/total": 771344, "tokens/train_per_sec_per_gpu": 42.0, "tokens/trainable": 15148} +{"epoch": 0.09375, "grad_norm": 1.8519418239593506, "learning_rate": 9.200000000000001e-05, "loss": 0.9670768976211548, "memory/device_reserved (GiB)": 42.37, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 2.63024, "step": 24, "tokens/total": 802016, "tokens/train_per_sec_per_gpu": 42.31, "tokens/trainable": 15766} +{"epoch": 0.09765625, "grad_norm": 2.9907498359680176, "learning_rate": 9.6e-05, "loss": 0.969383716583252, "memory/device_reserved (GiB)": 42.37, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 2.63632, "step": 25, "tokens/total": 833888, "tokens/train_per_sec_per_gpu": 41.73, "tokens/trainable": 16451} +{"epoch": 0.1015625, "grad_norm": 2.878203868865967, "learning_rate": 0.0001, "loss": 0.9940261244773865, "memory/device_reserved (GiB)": 42.37, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 2.70209, "step": 26, "tokens/total": 866096, "tokens/train_per_sec_per_gpu": 38.1, "tokens/trainable": 17059} +{"epoch": 0.10546875, "grad_norm": 3.885219097137451, "learning_rate": 9.99990636831587e-05, "loss": 0.8151969909667969, "memory/device_reserved (GiB)": 42.37, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 2.25962, "step": 27, "tokens/total": 898352, "tokens/train_per_sec_per_gpu": 50.84, "tokens/trainable": 17815} +{"epoch": 0.109375, "grad_norm": 2.5727996826171875, "learning_rate": 9.999625477159879e-05, "loss": 0.9088608026504517, "memory/device_reserved (GiB)": 42.37, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 2.48149, "step": 28, "tokens/total": 936544, "tokens/train_per_sec_per_gpu": 34.48, "tokens/trainable": 18484} +{"epoch": 0.11328125, "grad_norm": 5.866401195526123, "learning_rate": 9.999157338221051e-05, "loss": 0.8228899836540222, "memory/device_reserved (GiB)": 42.37, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 2.27707, "step": 29, "tokens/total": 971360, "tokens/train_per_sec_per_gpu": 40.63, "tokens/trainable": 19217} +{"epoch": 0.1171875, "grad_norm": 2.240715503692627, "learning_rate": 9.998501970980562e-05, "loss": 0.8445215821266174, "memory/device_reserved (GiB)": 42.37, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 2.32686, "step": 30, "tokens/total": 1002736, "tokens/train_per_sec_per_gpu": 45.22, "tokens/trainable": 19912} +{"epoch": 0.12109375, "grad_norm": 2.1158206462860107, "learning_rate": 9.997659402710915e-05, "loss": 0.9429418444633484, "memory/device_reserved (GiB)": 42.37, "memory/max_active (GiB)": 38.51, "memory/max_allocated (GiB)": 38.51, "ppl": 2.56752, "step": 31, "tokens/total": 1037136, "tokens/train_per_sec_per_gpu": 36.63, "tokens/trainable": 20532} +{"epoch": 0.125, "grad_norm": 1.6475661993026733, "learning_rate": 9.996629668474818e-05, "loss": 0.7221766710281372, "memory/device_reserved (GiB)": 42.37, "memory/max_active (GiB)": 37.22, "memory/max_allocated (GiB)": 37.22, "ppl": 2.05891, "step": 32, "tokens/total": 1071664, "tokens/train_per_sec_per_gpu": 41.07, "tokens/trainable": 21268} +{"epoch": 0.12890625, "grad_norm": 1.9741592407226562, "learning_rate": 9.995412811123711e-05, "loss": 0.8989213109016418, "memory/device_reserved (GiB)": 42.37, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 2.45695, "step": 33, "tokens/total": 1107104, "tokens/train_per_sec_per_gpu": 35.96, "tokens/trainable": 21858} +{"epoch": 0.1328125, "grad_norm": 1.5792282819747925, "learning_rate": 9.994008881295999e-05, "loss": 0.6524450778961182, "memory/device_reserved (GiB)": 39.82, "memory/max_active (GiB)": 37.07, "memory/max_allocated (GiB)": 37.07, "ppl": 1.92023, "step": 34, "tokens/total": 1140224, "tokens/train_per_sec_per_gpu": 41.75, "tokens/trainable": 22549} +{"epoch": 0.13671875, "grad_norm": 1.7107031345367432, "learning_rate": 9.992417937414932e-05, "loss": 0.679961621761322, "memory/device_reserved (GiB)": 39.82, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.9738, "step": 35, "tokens/total": 1173728, "tokens/train_per_sec_per_gpu": 42.19, "tokens/trainable": 23296} +{"epoch": 0.140625, "grad_norm": 1.943312406539917, "learning_rate": 9.99064004568618e-05, "loss": 0.8110288381576538, "memory/device_reserved (GiB)": 39.82, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 2.25022, "step": 36, "tokens/total": 1209024, "tokens/train_per_sec_per_gpu": 35.5, "tokens/trainable": 23936} +{"epoch": 0.14453125, "grad_norm": 2.259171485900879, "learning_rate": 9.988675280095074e-05, "loss": 0.740930438041687, "memory/device_reserved (GiB)": 39.82, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 2.09789, "step": 37, "tokens/total": 1243216, "tokens/train_per_sec_per_gpu": 36.52, "tokens/trainable": 24636} +{"epoch": 0.1484375, "grad_norm": 1.6916265487670898, "learning_rate": 9.986523722403528e-05, "loss": 0.7513322234153748, "memory/device_reserved (GiB)": 39.82, "memory/max_active (GiB)": 36.69, "memory/max_allocated (GiB)": 36.69, "ppl": 2.11982, "step": 38, "tokens/total": 1274624, "tokens/train_per_sec_per_gpu": 38.34, "tokens/trainable": 25281} +{"epoch": 0.15234375, "grad_norm": 2.194566488265991, "learning_rate": 9.984185462146642e-05, "loss": 0.7248092889785767, "memory/device_reserved (GiB)": 39.82, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 2.06434, "step": 39, "tokens/total": 1309280, "tokens/train_per_sec_per_gpu": 41.33, "tokens/trainable": 25980} +{"epoch": 0.15625, "grad_norm": 2.742906093597412, "learning_rate": 9.98166059662897e-05, "loss": 0.827784538269043, "memory/device_reserved (GiB)": 39.83, "memory/max_active (GiB)": 35.65, "memory/max_allocated (GiB)": 35.65, "ppl": 2.28824, "step": 40, "tokens/total": 1339632, "tokens/train_per_sec_per_gpu": 43.22, "tokens/trainable": 26630} +{"epoch": 0.16015625, "grad_norm": 2.222100019454956, "learning_rate": 9.978949230920472e-05, "loss": 0.7838250994682312, "memory/device_reserved (GiB)": 39.83, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 2.18983, "step": 41, "tokens/total": 1373520, "tokens/train_per_sec_per_gpu": 36.65, "tokens/trainable": 27268} +{"epoch": 0.1640625, "grad_norm": 1.6369054317474365, "learning_rate": 9.976051477852141e-05, "loss": 0.7346057891845703, "memory/device_reserved (GiB)": 39.83, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 2.08466, "step": 42, "tokens/total": 1406528, "tokens/train_per_sec_per_gpu": 44.69, "tokens/trainable": 27951} +{"epoch": 0.16796875, "grad_norm": 3.880985736846924, "learning_rate": 9.972967458011312e-05, "loss": 0.7413821220397949, "memory/device_reserved (GiB)": 39.83, "memory/max_active (GiB)": 38.81, "memory/max_allocated (GiB)": 38.81, "ppl": 2.09883, "step": 43, "tokens/total": 1441536, "tokens/train_per_sec_per_gpu": 38.09, "tokens/trainable": 28628} +{"epoch": 0.171875, "grad_norm": 6.043949127197266, "learning_rate": 9.96969729973664e-05, "loss": 0.7329882383346558, "memory/device_reserved (GiB)": 41.22, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 2.08129, "step": 44, "tokens/total": 1479792, "tokens/train_per_sec_per_gpu": 34.33, "tokens/trainable": 29305} +{"epoch": 0.17578125, "grad_norm": 2.6352014541625977, "learning_rate": 9.966241139112754e-05, "loss": 0.8601638078689575, "memory/device_reserved (GiB)": 41.22, "memory/max_active (GiB)": 37.87, "memory/max_allocated (GiB)": 37.87, "ppl": 2.36355, "step": 45, "tokens/total": 1512880, "tokens/train_per_sec_per_gpu": 37.81, "tokens/trainable": 29938} +{"epoch": 0.1796875, "grad_norm": 2.210275888442993, "learning_rate": 9.96259911996461e-05, "loss": 0.7822139263153076, "memory/device_reserved (GiB)": 41.22, "memory/max_active (GiB)": 38.47, "memory/max_allocated (GiB)": 38.47, "ppl": 2.18631, "step": 46, "tokens/total": 1546592, "tokens/train_per_sec_per_gpu": 37.42, "tokens/trainable": 30616} +{"epoch": 0.18359375, "grad_norm": 2.1432607173919678, "learning_rate": 9.958771393851491e-05, "loss": 0.8116734027862549, "memory/device_reserved (GiB)": 41.22, "memory/max_active (GiB)": 36.57, "memory/max_allocated (GiB)": 36.57, "ppl": 2.25167, "step": 47, "tokens/total": 1577920, "tokens/train_per_sec_per_gpu": 38.96, "tokens/trainable": 31269} +{"epoch": 0.1875, "grad_norm": 3.1099090576171875, "learning_rate": 9.954758120060702e-05, "loss": 0.716335654258728, "memory/device_reserved (GiB)": 41.22, "memory/max_active (GiB)": 39.08, "memory/max_allocated (GiB)": 39.08, "ppl": 2.04692, "step": 48, "tokens/total": 1613904, "tokens/train_per_sec_per_gpu": 34.42, "tokens/trainable": 31897} +{"epoch": 0.19140625, "grad_norm": 1.781152606010437, "learning_rate": 9.950559465600948e-05, "loss": 0.6837706565856934, "memory/device_reserved (GiB)": 41.22, "memory/max_active (GiB)": 37.52, "memory/max_allocated (GiB)": 37.52, "ppl": 1.98133, "step": 49, "tokens/total": 1647632, "tokens/train_per_sec_per_gpu": 37.92, "tokens/trainable": 32570} +{"epoch": 0.1953125, "grad_norm": 3.5153775215148926, "learning_rate": 9.946175605195379e-05, "loss": 0.5986154079437256, "memory/device_reserved (GiB)": 41.22, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.8196, "step": 50, "tokens/total": 1687312, "tokens/train_per_sec_per_gpu": 32.26, "tokens/trainable": 33294} +{"epoch": 0.19921875, "grad_norm": 2.183244228363037, "learning_rate": 9.941606721274322e-05, "loss": 0.6821981072425842, "memory/device_reserved (GiB)": 41.22, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.97822, "step": 51, "tokens/total": 1721296, "tokens/train_per_sec_per_gpu": 31.31, "tokens/trainable": 33954} +{"epoch": 0.203125, "grad_norm": 3.5324954986572266, "learning_rate": 9.936853003967685e-05, "loss": 0.7532718181610107, "memory/device_reserved (GiB)": 41.22, "memory/max_active (GiB)": 38.59, "memory/max_allocated (GiB)": 38.59, "ppl": 2.12394, "step": 52, "tokens/total": 1758208, "tokens/train_per_sec_per_gpu": 34.46, "tokens/trainable": 34594} +{"epoch": 0.20703125, "grad_norm": 2.357625722885132, "learning_rate": 9.93191465109705e-05, "loss": 0.7238050699234009, "memory/device_reserved (GiB)": 41.22, "memory/max_active (GiB)": 38.22, "memory/max_allocated (GiB)": 38.22, "ppl": 2.06227, "step": 53, "tokens/total": 1794496, "tokens/train_per_sec_per_gpu": 36.4, "tokens/trainable": 35242} +{"epoch": 0.2109375, "grad_norm": 2.180598497390747, "learning_rate": 9.926791868167438e-05, "loss": 0.7281752824783325, "memory/device_reserved (GiB)": 41.22, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 2.0713, "step": 54, "tokens/total": 1830592, "tokens/train_per_sec_per_gpu": 31.91, "tokens/trainable": 35849} +{"epoch": 0.21484375, "grad_norm": 6.401346206665039, "learning_rate": 9.921484868358753e-05, "loss": 0.7036174535751343, "memory/device_reserved (GiB)": 41.22, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 2.02105, "step": 55, "tokens/total": 1866032, "tokens/train_per_sec_per_gpu": 32.28, "tokens/trainable": 36529} +{"epoch": 0.21875, "grad_norm": 1.8679678440093994, "learning_rate": 9.915993872516924e-05, "loss": 0.6882210969924927, "memory/device_reserved (GiB)": 41.22, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.99017, "step": 56, "tokens/total": 1901280, "tokens/train_per_sec_per_gpu": 36.9, "tokens/trainable": 37197} +{"epoch": 0.22265625, "grad_norm": 1.7445793151855469, "learning_rate": 9.9103191091447e-05, "loss": 0.6961363554000854, "memory/device_reserved (GiB)": 41.22, "memory/max_active (GiB)": 36.79, "memory/max_allocated (GiB)": 36.79, "ppl": 2.00599, "step": 57, "tokens/total": 1932720, "tokens/train_per_sec_per_gpu": 36.55, "tokens/trainable": 37798} +{"epoch": 0.2265625, "grad_norm": 1.7554540634155273, "learning_rate": 9.904460814392147e-05, "loss": 0.5968382358551025, "memory/device_reserved (GiB)": 41.22, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.81637, "step": 58, "tokens/total": 1972176, "tokens/train_per_sec_per_gpu": 33.08, "tokens/trainable": 38500} +{"epoch": 0.23046875, "grad_norm": 2.521527051925659, "learning_rate": 9.898419232046825e-05, "loss": 0.5732739567756653, "memory/device_reserved (GiB)": 41.22, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.77407, "step": 59, "tokens/total": 2008416, "tokens/train_per_sec_per_gpu": 37.78, "tokens/trainable": 39217} +{"epoch": 0.234375, "grad_norm": 1.6404087543487549, "learning_rate": 9.892194613523633e-05, "loss": 0.6152392029762268, "memory/device_reserved (GiB)": 41.22, "memory/max_active (GiB)": 38.18, "memory/max_allocated (GiB)": 38.18, "ppl": 1.8501, "step": 60, "tokens/total": 2044112, "tokens/train_per_sec_per_gpu": 40.06, "tokens/trainable": 39954} +{"epoch": 0.23828125, "grad_norm": 1.5984060764312744, "learning_rate": 9.885787217854357e-05, "loss": 0.5656073093414307, "memory/device_reserved (GiB)": 41.22, "memory/max_active (GiB)": 38.1, "memory/max_allocated (GiB)": 38.1, "ppl": 1.76052, "step": 61, "tokens/total": 2078016, "tokens/train_per_sec_per_gpu": 40.06, "tokens/trainable": 40626} +{"epoch": 0.2421875, "grad_norm": 3.6524572372436523, "learning_rate": 9.879197311676887e-05, "loss": 0.5725845098495483, "memory/device_reserved (GiB)": 41.22, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.77284, "step": 62, "tokens/total": 2113888, "tokens/train_per_sec_per_gpu": 37.86, "tokens/trainable": 41303} +{"epoch": 0.24609375, "grad_norm": 1.9035389423370361, "learning_rate": 9.872425169224113e-05, "loss": 0.5793830156326294, "memory/device_reserved (GiB)": 41.22, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.78494, "step": 63, "tokens/total": 2147280, "tokens/train_per_sec_per_gpu": 46.82, "tokens/trainable": 42066} +{"epoch": 0.25, "grad_norm": 2.6824076175689697, "learning_rate": 9.865471072312528e-05, "loss": 0.587946355342865, "memory/device_reserved (GiB)": 41.22, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.80029, "step": 64, "tokens/total": 2183376, "tokens/train_per_sec_per_gpu": 41.2, "tokens/trainable": 42788} +{"epoch": 0.25390625, "grad_norm": 1.8753715753555298, "learning_rate": 9.858335310330492e-05, "loss": 0.5672294497489929, "memory/device_reserved (GiB)": 41.22, "memory/max_active (GiB)": 36.9, "memory/max_allocated (GiB)": 36.9, "ppl": 1.76337, "step": 65, "tokens/total": 2215344, "tokens/train_per_sec_per_gpu": 40.98, "tokens/trainable": 43480} +{"epoch": 0.2578125, "grad_norm": 2.127380847930908, "learning_rate": 9.851018180226185e-05, "loss": 0.5433980822563171, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.72185, "step": 66, "tokens/total": 2253648, "tokens/train_per_sec_per_gpu": 32.77, "tokens/trainable": 44153} +{"epoch": 0.26171875, "grad_norm": 2.2535834312438965, "learning_rate": 9.843519986495259e-05, "loss": 0.5875426530838013, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.79956, "step": 67, "tokens/total": 2288064, "tokens/train_per_sec_per_gpu": 39.68, "tokens/trainable": 44822} +{"epoch": 0.265625, "grad_norm": 1.7147167921066284, "learning_rate": 9.835841041168162e-05, "loss": 0.43233829736709595, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.54086, "step": 68, "tokens/total": 2325072, "tokens/train_per_sec_per_gpu": 41.71, "tokens/trainable": 45554} +{"epoch": 0.26953125, "grad_norm": 1.652053952217102, "learning_rate": 9.82798166379715e-05, "loss": 0.4774514138698578, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 36.7, "memory/max_allocated (GiB)": 36.7, "ppl": 1.61196, "step": 69, "tokens/total": 2357952, "tokens/train_per_sec_per_gpu": 40.38, "tokens/trainable": 46273} +{"epoch": 0.2734375, "grad_norm": 1.9213087558746338, "learning_rate": 9.819942181443002e-05, "loss": 0.5909402370452881, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.80569, "step": 70, "tokens/total": 2394864, "tokens/train_per_sec_per_gpu": 35.01, "tokens/trainable": 46948} +{"epoch": 0.27734375, "grad_norm": 2.1499602794647217, "learning_rate": 9.811722928661392e-05, "loss": 0.6326897740364075, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 35.69, "memory/max_allocated (GiB)": 35.69, "ppl": 1.88267, "step": 71, "tokens/total": 2423248, "tokens/train_per_sec_per_gpu": 44.1, "tokens/trainable": 47565} +{"epoch": 0.28125, "grad_norm": 2.396608829498291, "learning_rate": 9.803324247488975e-05, "loss": 0.5951454639434814, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 39.04, "memory/max_allocated (GiB)": 39.04, "ppl": 1.81329, "step": 72, "tokens/total": 2458800, "tokens/train_per_sec_per_gpu": 34.75, "tokens/trainable": 48232} +{"epoch": 0.28515625, "grad_norm": 2.886207342147827, "learning_rate": 9.794746487429161e-05, "loss": 0.5252617001533508, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 36.9, "memory/max_allocated (GiB)": 36.9, "ppl": 1.6909, "step": 73, "tokens/total": 2491504, "tokens/train_per_sec_per_gpu": 45.02, "tokens/trainable": 48900} +{"epoch": 0.2890625, "grad_norm": 2.079655885696411, "learning_rate": 9.785990005437554e-05, "loss": 0.5474653244018555, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 37.67, "memory/max_allocated (GiB)": 37.67, "ppl": 1.72887, "step": 74, "tokens/total": 2526960, "tokens/train_per_sec_per_gpu": 39.21, "tokens/trainable": 49564} +{"epoch": 0.29296875, "grad_norm": 1.7255465984344482, "learning_rate": 9.777055165907117e-05, "loss": 0.5546475648880005, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.74133, "step": 75, "tokens/total": 2559760, "tokens/train_per_sec_per_gpu": 44.55, "tokens/trainable": 50231} +{"epoch": 0.296875, "grad_norm": 3.0149765014648438, "learning_rate": 9.767942340652993e-05, "loss": 0.6407553553581238, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 37.97, "memory/max_allocated (GiB)": 37.97, "ppl": 1.89791, "step": 76, "tokens/total": 2594096, "tokens/train_per_sec_per_gpu": 34.21, "tokens/trainable": 50871} +{"epoch": 0.30078125, "grad_norm": 4.01525354385376, "learning_rate": 9.758651908897035e-05, "loss": 0.5407065749168396, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.71722, "step": 77, "tokens/total": 2627168, "tokens/train_per_sec_per_gpu": 40.42, "tokens/trainable": 51539} +{"epoch": 0.3046875, "grad_norm": 1.8454959392547607, "learning_rate": 9.749184257252033e-05, "loss": 0.5872691869735718, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.79907, "step": 78, "tokens/total": 2661008, "tokens/train_per_sec_per_gpu": 39.75, "tokens/trainable": 52184} +{"epoch": 0.30859375, "grad_norm": 1.78860604763031, "learning_rate": 9.739539779705614e-05, "loss": 0.4764769673347473, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.61039, "step": 79, "tokens/total": 2699120, "tokens/train_per_sec_per_gpu": 37.97, "tokens/trainable": 52992} +{"epoch": 0.3125, "grad_norm": 2.2620065212249756, "learning_rate": 9.729718877603861e-05, "loss": 0.5448523759841919, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.72435, "step": 80, "tokens/total": 2732208, "tokens/train_per_sec_per_gpu": 42.18, "tokens/trainable": 53631} +{"epoch": 0.31640625, "grad_norm": 3.0816855430603027, "learning_rate": 9.719721959634592e-05, "loss": 0.47341397404670715, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 39.05, "memory/max_allocated (GiB)": 39.05, "ppl": 1.60547, "step": 81, "tokens/total": 2767440, "tokens/train_per_sec_per_gpu": 40.33, "tokens/trainable": 54285} +{"epoch": 0.3203125, "grad_norm": 6.712282180786133, "learning_rate": 9.709549441810375e-05, "loss": 0.44098055362701416, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 39.35, "memory/max_allocated (GiB)": 39.35, "ppl": 1.55423, "step": 82, "tokens/total": 2804608, "tokens/train_per_sec_per_gpu": 40.54, "tokens/trainable": 54992} +{"epoch": 0.32421875, "grad_norm": 4.666300296783447, "learning_rate": 9.699201747451195e-05, "loss": 0.49957019090652466, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 36.4, "memory/max_allocated (GiB)": 36.4, "ppl": 1.64801, "step": 83, "tokens/total": 2836832, "tokens/train_per_sec_per_gpu": 36.15, "tokens/trainable": 55658} +{"epoch": 0.328125, "grad_norm": 1.6984440088272095, "learning_rate": 9.688679307166854e-05, "loss": 0.5815361738204956, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.78878, "step": 84, "tokens/total": 2869056, "tokens/train_per_sec_per_gpu": 40.82, "tokens/trainable": 56311} +{"epoch": 0.33203125, "grad_norm": 1.9993188381195068, "learning_rate": 9.677982558839042e-05, "loss": 0.5725433826446533, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 36.71, "memory/max_allocated (GiB)": 36.71, "ppl": 1.77277, "step": 85, "tokens/total": 2901856, "tokens/train_per_sec_per_gpu": 37.71, "tokens/trainable": 57029} +{"epoch": 0.3359375, "grad_norm": 3.2093145847320557, "learning_rate": 9.66711194760312e-05, "loss": 0.4519972801208496, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.57145, "step": 86, "tokens/total": 2938848, "tokens/train_per_sec_per_gpu": 37.16, "tokens/trainable": 57785} +{"epoch": 0.33984375, "grad_norm": 7.064257621765137, "learning_rate": 9.656067925829593e-05, "loss": 0.5139092803001404, "memory/device_reserved (GiB)": 41.13, "memory/max_active (GiB)": 37.5, "memory/max_allocated (GiB)": 37.5, "ppl": 1.67181, "step": 87, "tokens/total": 2969056, "tokens/train_per_sec_per_gpu": 45.49, "tokens/trainable": 58434} +{"epoch": 0.34375, "grad_norm": 2.0939407348632812, "learning_rate": 9.644850953105288e-05, "loss": 0.5270383358001709, "memory/device_reserved (GiB)": 41.14, "memory/max_active (GiB)": 35.72, "memory/max_allocated (GiB)": 35.72, "ppl": 1.69391, "step": 88, "tokens/total": 2999472, "tokens/train_per_sec_per_gpu": 45.43, "tokens/trainable": 59138} +{"epoch": 0.34765625, "grad_norm": 1.5219923257827759, "learning_rate": 9.633461496214225e-05, "loss": 0.45625007152557373, "memory/device_reserved (GiB)": 41.14, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.57814, "step": 89, "tokens/total": 3031712, "tokens/train_per_sec_per_gpu": 42.63, "tokens/trainable": 59815} +{"epoch": 0.3515625, "grad_norm": 1.679053783416748, "learning_rate": 9.621900029118195e-05, "loss": 0.4791201949119568, "memory/device_reserved (GiB)": 41.14, "memory/max_active (GiB)": 35.96, "memory/max_allocated (GiB)": 35.96, "ppl": 1.61465, "step": 90, "tokens/total": 3061008, "tokens/train_per_sec_per_gpu": 40.24, "tokens/trainable": 60464} +{"epoch": 0.35546875, "grad_norm": 2.5642035007476807, "learning_rate": 9.610167032937036e-05, "loss": 0.5198730826377869, "memory/device_reserved (GiB)": 41.14, "memory/max_active (GiB)": 38.41, "memory/max_allocated (GiB)": 38.41, "ppl": 1.68181, "step": 91, "tokens/total": 3098864, "tokens/train_per_sec_per_gpu": 33.14, "tokens/trainable": 61141} +{"epoch": 0.359375, "grad_norm": 1.5792652368545532, "learning_rate": 9.598262995928611e-05, "loss": 0.4296034872531891, "memory/device_reserved (GiB)": 41.14, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.53665, "step": 92, "tokens/total": 3129536, "tokens/train_per_sec_per_gpu": 48.55, "tokens/trainable": 61845} +{"epoch": 0.36328125, "grad_norm": 1.7809463739395142, "learning_rate": 9.586188413468492e-05, "loss": 0.4280136227607727, "memory/device_reserved (GiB)": 41.14, "memory/max_active (GiB)": 38.66, "memory/max_allocated (GiB)": 38.66, "ppl": 1.53421, "step": 93, "tokens/total": 3167504, "tokens/train_per_sec_per_gpu": 32.5, "tokens/trainable": 62535} +{"epoch": 0.3671875, "grad_norm": 1.8045077323913574, "learning_rate": 9.57394378802934e-05, "loss": 0.5071197748184204, "memory/device_reserved (GiB)": 41.14, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 1.6605, "step": 94, "tokens/total": 3200624, "tokens/train_per_sec_per_gpu": 38.71, "tokens/trainable": 63184} +{"epoch": 0.37109375, "grad_norm": 1.855805516242981, "learning_rate": 9.56152962916e-05, "loss": 0.4292992949485779, "memory/device_reserved (GiB)": 41.14, "memory/max_active (GiB)": 36.34, "memory/max_allocated (GiB)": 36.34, "ppl": 1.53618, "step": 95, "tokens/total": 3230896, "tokens/train_per_sec_per_gpu": 43.58, "tokens/trainable": 63826} +{"epoch": 0.375, "grad_norm": 4.10609769821167, "learning_rate": 9.548946453464296e-05, "loss": 0.3688984811306, "memory/device_reserved (GiB)": 41.14, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.44614, "step": 96, "tokens/total": 3264848, "tokens/train_per_sec_per_gpu": 40.36, "tokens/trainable": 64540} +{"epoch": 0.37890625, "grad_norm": 3.4239659309387207, "learning_rate": 9.53619478457953e-05, "loss": 0.5045754909515381, "memory/device_reserved (GiB)": 41.14, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.65628, "step": 97, "tokens/total": 3299104, "tokens/train_per_sec_per_gpu": 34.05, "tokens/trainable": 65206} +{"epoch": 0.3828125, "grad_norm": 1.890915870666504, "learning_rate": 9.523275153154695e-05, "loss": 0.4808976352214813, "memory/device_reserved (GiB)": 41.14, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.61753, "step": 98, "tokens/total": 3331264, "tokens/train_per_sec_per_gpu": 39.54, "tokens/trainable": 65846} +{"epoch": 0.38671875, "grad_norm": 5.632991790771484, "learning_rate": 9.51018809682839e-05, "loss": 0.4481455683708191, "memory/device_reserved (GiB)": 41.14, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.56541, "step": 99, "tokens/total": 3368736, "tokens/train_per_sec_per_gpu": 33.55, "tokens/trainable": 66504} +{"epoch": 0.390625, "grad_norm": 2.110642433166504, "learning_rate": 9.49693416020645e-05, "loss": 0.5314699411392212, "memory/device_reserved (GiB)": 41.14, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.70143, "step": 100, "tokens/total": 3406208, "tokens/train_per_sec_per_gpu": 35.12, "tokens/trainable": 67161} +{"epoch": 0.39453125, "grad_norm": 2.927494525909424, "learning_rate": 9.483513894839276e-05, "loss": 0.5409561395645142, "memory/device_reserved (GiB)": 41.14, "memory/max_active (GiB)": 36.9, "memory/max_allocated (GiB)": 36.9, "ppl": 1.71765, "step": 101, "tokens/total": 3439312, "tokens/train_per_sec_per_gpu": 43.1, "tokens/trainable": 67860} +{"epoch": 0.3984375, "grad_norm": 2.8976008892059326, "learning_rate": 9.469927859198888e-05, "loss": 0.4640806019306183, "memory/device_reserved (GiB)": 41.14, "memory/max_active (GiB)": 38.66, "memory/max_allocated (GiB)": 38.66, "ppl": 1.59055, "step": 102, "tokens/total": 3474832, "tokens/train_per_sec_per_gpu": 40.27, "tokens/trainable": 68518} +{"epoch": 0.40234375, "grad_norm": 1.8410085439682007, "learning_rate": 9.456176618655689e-05, "loss": 0.42636939883232117, "memory/device_reserved (GiB)": 41.14, "memory/max_active (GiB)": 36.36, "memory/max_allocated (GiB)": 36.36, "ppl": 1.53169, "step": 103, "tokens/total": 3506448, "tokens/train_per_sec_per_gpu": 40.2, "tokens/trainable": 69156} +{"epoch": 0.40625, "grad_norm": 3.2082862854003906, "learning_rate": 9.442260745454927e-05, "loss": 0.4139462113380432, "memory/device_reserved (GiB)": 41.14, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.51278, "step": 104, "tokens/total": 3541328, "tokens/train_per_sec_per_gpu": 37.22, "tokens/trainable": 69840} +{"epoch": 0.41015625, "grad_norm": 1.958462119102478, "learning_rate": 9.428180818692884e-05, "loss": 0.37405359745025635, "memory/device_reserved (GiB)": 41.14, "memory/max_active (GiB)": 38.45, "memory/max_allocated (GiB)": 38.45, "ppl": 1.45362, "step": 105, "tokens/total": 3577216, "tokens/train_per_sec_per_gpu": 34.18, "tokens/trainable": 70533} +{"epoch": 0.4140625, "grad_norm": 2.7890660762786865, "learning_rate": 9.413937424292791e-05, "loss": 0.4208468794822693, "memory/device_reserved (GiB)": 41.27, "memory/max_active (GiB)": 39.11, "memory/max_allocated (GiB)": 39.11, "ppl": 1.52325, "step": 106, "tokens/total": 3615856, "tokens/train_per_sec_per_gpu": 32.29, "tokens/trainable": 71202} +{"epoch": 0.41796875, "grad_norm": 1.9838937520980835, "learning_rate": 9.399531154980424e-05, "loss": 0.33467817306518555, "memory/device_reserved (GiB)": 41.27, "memory/max_active (GiB)": 38.22, "memory/max_allocated (GiB)": 38.22, "ppl": 1.39749, "step": 107, "tokens/total": 3652128, "tokens/train_per_sec_per_gpu": 34.65, "tokens/trainable": 71867} +{"epoch": 0.421875, "grad_norm": 1.5353271961212158, "learning_rate": 9.384962610259455e-05, "loss": 0.38360875844955444, "memory/device_reserved (GiB)": 41.27, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.46757, "step": 108, "tokens/total": 3686192, "tokens/train_per_sec_per_gpu": 35.66, "tokens/trainable": 72504} +{"epoch": 0.42578125, "grad_norm": 3.266331911087036, "learning_rate": 9.370232396386494e-05, "loss": 0.41029369831085205, "memory/device_reserved (GiB)": 41.27, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.50726, "step": 109, "tokens/total": 3719088, "tokens/train_per_sec_per_gpu": 41.3, "tokens/trainable": 73163} +{"epoch": 0.4296875, "grad_norm": 2.5380759239196777, "learning_rate": 9.355341126345868e-05, "loss": 0.3648647665977478, "memory/device_reserved (GiB)": 41.27, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.44032, "step": 110, "tokens/total": 3754016, "tokens/train_per_sec_per_gpu": 37.34, "tokens/trainable": 73830} +{"epoch": 0.43359375, "grad_norm": 1.8235238790512085, "learning_rate": 9.340289419824107e-05, "loss": 0.3679196834564209, "memory/device_reserved (GiB)": 41.27, "memory/max_active (GiB)": 36.6, "memory/max_allocated (GiB)": 36.6, "ppl": 1.44473, "step": 111, "tokens/total": 3785920, "tokens/train_per_sec_per_gpu": 44.93, "tokens/trainable": 74476} +{"epoch": 0.4375, "grad_norm": 2.2866594791412354, "learning_rate": 9.325077903184159e-05, "loss": 0.4587223529815674, "memory/device_reserved (GiB)": 41.27, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.58205, "step": 112, "tokens/total": 3820368, "tokens/train_per_sec_per_gpu": 34.83, "tokens/trainable": 75149} +{"epoch": 0.44140625, "grad_norm": 2.008992910385132, "learning_rate": 9.30970720943932e-05, "loss": 0.3496010899543762, "memory/device_reserved (GiB)": 41.27, "memory/max_active (GiB)": 38.29, "memory/max_allocated (GiB)": 38.29, "ppl": 1.4185, "step": 113, "tokens/total": 3855376, "tokens/train_per_sec_per_gpu": 36.86, "tokens/trainable": 75829} +{"epoch": 0.4453125, "grad_norm": 2.2021398544311523, "learning_rate": 9.2941779782269e-05, "loss": 0.3852511942386627, "memory/device_reserved (GiB)": 41.27, "memory/max_active (GiB)": 37.12, "memory/max_allocated (GiB)": 37.12, "ppl": 1.46998, "step": 114, "tokens/total": 3888608, "tokens/train_per_sec_per_gpu": 35.85, "tokens/trainable": 76445} +{"epoch": 0.44921875, "grad_norm": 1.9051982164382935, "learning_rate": 9.278490855781596e-05, "loss": 0.41707339882850647, "memory/device_reserved (GiB)": 41.27, "memory/max_active (GiB)": 37.12, "memory/max_allocated (GiB)": 37.12, "ppl": 1.51751, "step": 115, "tokens/total": 3920320, "tokens/train_per_sec_per_gpu": 44.31, "tokens/trainable": 77072} +{"epoch": 0.453125, "grad_norm": 1.81536865234375, "learning_rate": 9.262646494908604e-05, "loss": 0.45664846897125244, "memory/device_reserved (GiB)": 41.27, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.57877, "step": 116, "tokens/total": 3958592, "tokens/train_per_sec_per_gpu": 33.97, "tokens/trainable": 77718} +{"epoch": 0.45703125, "grad_norm": 1.6300559043884277, "learning_rate": 9.246645554956457e-05, "loss": 0.45136064291000366, "memory/device_reserved (GiB)": 41.27, "memory/max_active (GiB)": 38.53, "memory/max_allocated (GiB)": 38.53, "ppl": 1.57045, "step": 117, "tokens/total": 3992912, "tokens/train_per_sec_per_gpu": 39.12, "tokens/trainable": 78374} +{"epoch": 0.4609375, "grad_norm": 12.839784622192383, "learning_rate": 9.230488701789578e-05, "loss": 0.4637022912502289, "memory/device_reserved (GiB)": 41.27, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.58995, "step": 118, "tokens/total": 4029872, "tokens/train_per_sec_per_gpu": 36.7, "tokens/trainable": 79080} +{"epoch": 0.46484375, "grad_norm": 1.9341695308685303, "learning_rate": 9.214176607760577e-05, "loss": 0.3683066964149475, "memory/device_reserved (GiB)": 41.27, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.44529, "step": 119, "tokens/total": 4068096, "tokens/train_per_sec_per_gpu": 39.47, "tokens/trainable": 79799} +{"epoch": 0.46875, "grad_norm": 2.3041634559631348, "learning_rate": 9.197709951682268e-05, "loss": 0.3721538782119751, "memory/device_reserved (GiB)": 41.27, "memory/max_active (GiB)": 36.88, "memory/max_allocated (GiB)": 36.88, "ppl": 1.45086, "step": 120, "tokens/total": 4100064, "tokens/train_per_sec_per_gpu": 39.84, "tokens/trainable": 80433} +{"epoch": 0.47265625, "grad_norm": 2.8725240230560303, "learning_rate": 9.181089418799428e-05, "loss": 0.394131600856781, "memory/device_reserved (GiB)": 41.27, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.4831, "step": 121, "tokens/total": 4136560, "tokens/train_per_sec_per_gpu": 38.2, "tokens/trainable": 81128} +{"epoch": 0.4765625, "grad_norm": 1.9130918979644775, "learning_rate": 9.164315700760271e-05, "loss": 0.3242451846599579, "memory/device_reserved (GiB)": 41.27, "memory/max_active (GiB)": 38.72, "memory/max_allocated (GiB)": 38.72, "ppl": 1.38299, "step": 122, "tokens/total": 4172048, "tokens/train_per_sec_per_gpu": 35.22, "tokens/trainable": 81803} +{"epoch": 0.48046875, "grad_norm": 3.439668655395508, "learning_rate": 9.147389495587671e-05, "loss": 0.4228176176548004, "memory/device_reserved (GiB)": 41.27, "memory/max_active (GiB)": 36.88, "memory/max_allocated (GiB)": 36.88, "ppl": 1.52626, "step": 123, "tokens/total": 4202896, "tokens/train_per_sec_per_gpu": 45.95, "tokens/trainable": 82458} +{"epoch": 0.484375, "grad_norm": 1.939178705215454, "learning_rate": 9.130311507650116e-05, "loss": 0.3431292474269867, "memory/device_reserved (GiB)": 41.27, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.40935, "step": 124, "tokens/total": 4235088, "tokens/train_per_sec_per_gpu": 37.93, "tokens/trainable": 83125} +{"epoch": 0.48828125, "grad_norm": 1.7523300647735596, "learning_rate": 9.113082447632394e-05, "loss": 0.40970638394355774, "memory/device_reserved (GiB)": 41.27, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.50638, "step": 125, "tokens/total": 4266736, "tokens/train_per_sec_per_gpu": 51.91, "tokens/trainable": 83812} +{"epoch": 0.4921875, "grad_norm": 3.207617998123169, "learning_rate": 9.09570303250602e-05, "loss": 0.4058682918548584, "memory/device_reserved (GiB)": 41.27, "memory/max_active (GiB)": 36.3, "memory/max_allocated (GiB)": 36.3, "ppl": 1.5006, "step": 126, "tokens/total": 4298192, "tokens/train_per_sec_per_gpu": 42.68, "tokens/trainable": 84517} +{"epoch": 0.49609375, "grad_norm": 1.7605640888214111, "learning_rate": 9.078173985499394e-05, "loss": 0.38717830181121826, "memory/device_reserved (GiB)": 41.28, "memory/max_active (GiB)": 35.67, "memory/max_allocated (GiB)": 35.67, "ppl": 1.47282, "step": 127, "tokens/total": 4327888, "tokens/train_per_sec_per_gpu": 40.31, "tokens/trainable": 85155} +{"epoch": 0.5, "grad_norm": 1.9416276216506958, "learning_rate": 9.060496036067713e-05, "loss": 0.40859556198120117, "memory/device_reserved (GiB)": 41.28, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.5047, "step": 128, "tokens/total": 4364976, "tokens/train_per_sec_per_gpu": 30.79, "tokens/trainable": 85813} +{"epoch": 0.50390625, "grad_norm": 1.826676845550537, "learning_rate": 9.042669919862615e-05, "loss": 0.3443276286125183, "memory/device_reserved (GiB)": 41.28, "memory/max_active (GiB)": 36.73, "memory/max_allocated (GiB)": 36.73, "ppl": 1.41104, "step": 129, "tokens/total": 4396832, "tokens/train_per_sec_per_gpu": 47.24, "tokens/trainable": 86525} +{"epoch": 0.5078125, "grad_norm": 1.420753002166748, "learning_rate": 9.024696378701557e-05, "loss": 0.35289400815963745, "memory/device_reserved (GiB)": 40.56, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.42318, "step": 130, "tokens/total": 4431968, "tokens/train_per_sec_per_gpu": 35.14, "tokens/trainable": 87174} +{"epoch": 0.51171875, "grad_norm": 1.7107036113739014, "learning_rate": 9.006576160536948e-05, "loss": 0.37455853819847107, "memory/device_reserved (GiB)": 40.56, "memory/max_active (GiB)": 37.31, "memory/max_allocated (GiB)": 37.31, "ppl": 1.45435, "step": 131, "tokens/total": 4466144, "tokens/train_per_sec_per_gpu": 34.78, "tokens/trainable": 87808} +{"epoch": 0.515625, "grad_norm": 6.019067764282227, "learning_rate": 8.988310019425035e-05, "loss": 0.39058464765548706, "memory/device_reserved (GiB)": 43.8, "memory/max_active (GiB)": 39.53, "memory/max_allocated (GiB)": 39.53, "ppl": 1.47784, "step": 132, "tokens/total": 4503632, "tokens/train_per_sec_per_gpu": 33.45, "tokens/trainable": 88438} +{"epoch": 0.51953125, "grad_norm": 2.0652780532836914, "learning_rate": 8.969898715494506e-05, "loss": 0.3717342019081116, "memory/device_reserved (GiB)": 43.8, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.45025, "step": 133, "tokens/total": 4540048, "tokens/train_per_sec_per_gpu": 40.9, "tokens/trainable": 89155} +{"epoch": 0.5234375, "grad_norm": 2.0804567337036133, "learning_rate": 8.951343014914869e-05, "loss": 0.2982982397079468, "memory/device_reserved (GiB)": 43.8, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.34756, "step": 134, "tokens/total": 4576288, "tokens/train_per_sec_per_gpu": 35.84, "tokens/trainable": 89855} +{"epoch": 0.52734375, "grad_norm": 1.7053743600845337, "learning_rate": 8.932643689864568e-05, "loss": 0.3517124056816101, "memory/device_reserved (GiB)": 43.8, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.4215, "step": 135, "tokens/total": 4612416, "tokens/train_per_sec_per_gpu": 39.84, "tokens/trainable": 90555} +{"epoch": 0.53125, "grad_norm": 1.5472038984298706, "learning_rate": 8.913801518498845e-05, "loss": 0.3331882953643799, "memory/device_reserved (GiB)": 43.8, "memory/max_active (GiB)": 39.54, "memory/max_allocated (GiB)": 39.54, "ppl": 1.39541, "step": 136, "tokens/total": 4651408, "tokens/train_per_sec_per_gpu": 33.24, "tokens/trainable": 91298} +{"epoch": 0.53515625, "grad_norm": 1.6435976028442383, "learning_rate": 8.894817284917364e-05, "loss": 0.38089409470558167, "memory/device_reserved (GiB)": 43.8, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.46359, "step": 137, "tokens/total": 4688032, "tokens/train_per_sec_per_gpu": 34.16, "tokens/trainable": 91992} +{"epoch": 0.5390625, "grad_norm": 1.452054500579834, "learning_rate": 8.875691779131569e-05, "loss": 0.3158543109893799, "memory/device_reserved (GiB)": 43.8, "memory/max_active (GiB)": 39.61, "memory/max_allocated (GiB)": 39.61, "ppl": 1.37143, "step": 138, "tokens/total": 4725520, "tokens/train_per_sec_per_gpu": 33.93, "tokens/trainable": 92671} +{"epoch": 0.54296875, "grad_norm": 2.837184429168701, "learning_rate": 8.856425797031829e-05, "loss": 0.31878694891929626, "memory/device_reserved (GiB)": 43.8, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.37546, "step": 139, "tokens/total": 4761136, "tokens/train_per_sec_per_gpu": 42.91, "tokens/trainable": 93384} +{"epoch": 0.546875, "grad_norm": 11.893083572387695, "learning_rate": 8.837020140354295e-05, "loss": 0.3380443751811981, "memory/device_reserved (GiB)": 43.8, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.4022, "step": 140, "tokens/total": 4797536, "tokens/train_per_sec_per_gpu": 35.5, "tokens/trainable": 94025} +{"epoch": 0.55078125, "grad_norm": 1.3434516191482544, "learning_rate": 8.817475616647554e-05, "loss": 0.3076171875, "memory/device_reserved (GiB)": 43.81, "memory/max_active (GiB)": 38.0, "memory/max_allocated (GiB)": 38.0, "ppl": 1.36018, "step": 141, "tokens/total": 4831568, "tokens/train_per_sec_per_gpu": 40.21, "tokens/trainable": 94711} +{"epoch": 0.5546875, "grad_norm": 1.808488130569458, "learning_rate": 8.797793039239017e-05, "loss": 0.31559431552886963, "memory/device_reserved (GiB)": 43.81, "memory/max_active (GiB)": 37.73, "memory/max_allocated (GiB)": 37.73, "ppl": 1.37107, "step": 142, "tokens/total": 4866288, "tokens/train_per_sec_per_gpu": 40.27, "tokens/trainable": 95396} +{"epoch": 0.55859375, "grad_norm": 3.4984586238861084, "learning_rate": 8.777973227201069e-05, "loss": 0.3039093315601349, "memory/device_reserved (GiB)": 43.81, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.35515, "step": 143, "tokens/total": 4902944, "tokens/train_per_sec_per_gpu": 43.49, "tokens/trainable": 96134} +{"epoch": 0.5625, "grad_norm": 2.2840168476104736, "learning_rate": 8.758017005316988e-05, "loss": 0.3879336714744568, "memory/device_reserved (GiB)": 43.81, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.47393, "step": 144, "tokens/total": 4935936, "tokens/train_per_sec_per_gpu": 48.59, "tokens/trainable": 96801} +{"epoch": 0.56640625, "grad_norm": 2.235781192779541, "learning_rate": 8.737925204046629e-05, "loss": 0.3862004280090332, "memory/device_reserved (GiB)": 43.81, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.47138, "step": 145, "tokens/total": 4970304, "tokens/train_per_sec_per_gpu": 33.6, "tokens/trainable": 97476} +{"epoch": 0.5703125, "grad_norm": 1.7982394695281982, "learning_rate": 8.717698659491851e-05, "loss": 0.3215237259864807, "memory/device_reserved (GiB)": 43.81, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.37923, "step": 146, "tokens/total": 5006832, "tokens/train_per_sec_per_gpu": 35.98, "tokens/trainable": 98199} +{"epoch": 0.57421875, "grad_norm": 1.8467975854873657, "learning_rate": 8.697338213361735e-05, "loss": 0.2961849272251129, "memory/device_reserved (GiB)": 43.81, "memory/max_active (GiB)": 36.36, "memory/max_allocated (GiB)": 36.36, "ppl": 1.34472, "step": 147, "tokens/total": 5037520, "tokens/train_per_sec_per_gpu": 43.41, "tokens/trainable": 98927} +{"epoch": 0.578125, "grad_norm": 1.4462112188339233, "learning_rate": 8.676844712937552e-05, "loss": 0.28615716099739075, "memory/device_reserved (GiB)": 43.81, "memory/max_active (GiB)": 39.18, "memory/max_allocated (GiB)": 39.18, "ppl": 1.3313, "step": 148, "tokens/total": 5077136, "tokens/train_per_sec_per_gpu": 33.88, "tokens/trainable": 99649} +{"epoch": 0.58203125, "grad_norm": 2.1384103298187256, "learning_rate": 8.656219011037509e-05, "loss": 0.28834259510040283, "memory/device_reserved (GiB)": 43.81, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.33421, "step": 149, "tokens/total": 5113872, "tokens/train_per_sec_per_gpu": 35.81, "tokens/trainable": 100340} +{"epoch": 0.5859375, "grad_norm": 2.2336456775665283, "learning_rate": 8.63546196598125e-05, "loss": 0.37280699610710144, "memory/device_reserved (GiB)": 43.81, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.4518, "step": 150, "tokens/total": 5151072, "tokens/train_per_sec_per_gpu": 34.78, "tokens/trainable": 101069} +{"epoch": 0.58984375, "grad_norm": 1.96064031124115, "learning_rate": 8.614574441554145e-05, "loss": 0.4505365192890167, "memory/device_reserved (GiB)": 43.81, "memory/max_active (GiB)": 36.88, "memory/max_allocated (GiB)": 36.88, "ppl": 1.56915, "step": 151, "tokens/total": 5183312, "tokens/train_per_sec_per_gpu": 42.55, "tokens/trainable": 101702} +{"epoch": 0.59375, "grad_norm": 4.317370414733887, "learning_rate": 8.593557306971349e-05, "loss": 0.2910606861114502, "memory/device_reserved (GiB)": 43.81, "memory/max_active (GiB)": 37.58, "memory/max_allocated (GiB)": 37.58, "ppl": 1.33785, "step": 152, "tokens/total": 5216176, "tokens/train_per_sec_per_gpu": 41.23, "tokens/trainable": 102392} +{"epoch": 0.59765625, "grad_norm": 1.6795623302459717, "learning_rate": 8.572411436841618e-05, "loss": 0.34114906191825867, "memory/device_reserved (GiB)": 43.81, "memory/max_active (GiB)": 36.73, "memory/max_allocated (GiB)": 36.73, "ppl": 1.40656, "step": 153, "tokens/total": 5249504, "tokens/train_per_sec_per_gpu": 40.62, "tokens/trainable": 103080} +{"epoch": 0.6015625, "grad_norm": 1.7513970136642456, "learning_rate": 8.551137711130922e-05, "loss": 0.3264225125312805, "memory/device_reserved (GiB)": 43.81, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 1.386, "step": 154, "tokens/total": 5280928, "tokens/train_per_sec_per_gpu": 36.03, "tokens/trainable": 103731} +{"epoch": 0.60546875, "grad_norm": 1.391423225402832, "learning_rate": 8.529737015125824e-05, "loss": 0.2543140947818756, "memory/device_reserved (GiB)": 43.81, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.28958, "step": 155, "tokens/total": 5315616, "tokens/train_per_sec_per_gpu": 38.44, "tokens/trainable": 104419} +{"epoch": 0.609375, "grad_norm": 1.5097694396972656, "learning_rate": 8.508210239396639e-05, "loss": 0.33529043197631836, "memory/device_reserved (GiB)": 43.81, "memory/max_active (GiB)": 39.58, "memory/max_allocated (GiB)": 39.58, "ppl": 1.39835, "step": 156, "tokens/total": 5352992, "tokens/train_per_sec_per_gpu": 37.46, "tokens/trainable": 105080} +{"epoch": 0.61328125, "grad_norm": 1.7222377061843872, "learning_rate": 8.486558279760375e-05, "loss": 0.32850462198257446, "memory/device_reserved (GiB)": 43.81, "memory/max_active (GiB)": 35.66, "memory/max_allocated (GiB)": 35.66, "ppl": 1.38889, "step": 157, "tokens/total": 5383408, "tokens/train_per_sec_per_gpu": 42.35, "tokens/trainable": 105719} +{"epoch": 0.6171875, "grad_norm": 1.8450902700424194, "learning_rate": 8.464782037243449e-05, "loss": 0.34246760606765747, "memory/device_reserved (GiB)": 43.81, "memory/max_active (GiB)": 36.7, "memory/max_allocated (GiB)": 36.7, "ppl": 1.40842, "step": 158, "tokens/total": 5415184, "tokens/train_per_sec_per_gpu": 39.54, "tokens/trainable": 106366} +{"epoch": 0.62109375, "grad_norm": 2.591036558151245, "learning_rate": 8.442882418044202e-05, "loss": 0.2820413112640381, "memory/device_reserved (GiB)": 43.81, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.32583, "step": 159, "tokens/total": 5447440, "tokens/train_per_sec_per_gpu": 44.55, "tokens/trainable": 107065} +{"epoch": 0.625, "grad_norm": 2.7927472591400146, "learning_rate": 8.420860333495179e-05, "loss": 0.3051151633262634, "memory/device_reserved (GiB)": 43.81, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.35678, "step": 160, "tokens/total": 5479552, "tokens/train_per_sec_per_gpu": 39.81, "tokens/trainable": 107653} +{"epoch": 0.62890625, "grad_norm": 4.186208724975586, "learning_rate": 8.398716700025208e-05, "loss": 0.3419494926929474, "memory/device_reserved (GiB)": 43.81, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.40769, "step": 161, "tokens/total": 5512560, "tokens/train_per_sec_per_gpu": 37.06, "tokens/trainable": 108292} +{"epoch": 0.6328125, "grad_norm": 2.4023478031158447, "learning_rate": 8.376452439121266e-05, "loss": 0.34210437536239624, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 35.82, "memory/max_allocated (GiB)": 35.82, "ppl": 1.40791, "step": 162, "tokens/total": 5542160, "tokens/train_per_sec_per_gpu": 42.87, "tokens/trainable": 108891} +{"epoch": 0.63671875, "grad_norm": 2.709078550338745, "learning_rate": 8.354068477290124e-05, "loss": 0.3610851764678955, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.43489, "step": 163, "tokens/total": 5576000, "tokens/train_per_sec_per_gpu": 43.42, "tokens/trainable": 109547} +{"epoch": 0.640625, "grad_norm": 2.2495017051696777, "learning_rate": 8.331565746019807e-05, "loss": 0.3015804886817932, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 36.59, "memory/max_allocated (GiB)": 36.59, "ppl": 1.35199, "step": 164, "tokens/total": 5607664, "tokens/train_per_sec_per_gpu": 45.12, "tokens/trainable": 110213} +{"epoch": 0.64453125, "grad_norm": 3.1263680458068848, "learning_rate": 8.308945181740812e-05, "loss": 0.32004043459892273, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 39.15, "memory/max_allocated (GiB)": 39.15, "ppl": 1.37718, "step": 165, "tokens/total": 5641856, "tokens/train_per_sec_per_gpu": 41.53, "tokens/trainable": 110858} +{"epoch": 0.6484375, "grad_norm": 1.6445146799087524, "learning_rate": 8.286207725787153e-05, "loss": 0.2593889832496643, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.29614, "step": 166, "tokens/total": 5678480, "tokens/train_per_sec_per_gpu": 35.32, "tokens/trainable": 111523} +{"epoch": 0.65234375, "grad_norm": 1.637981653213501, "learning_rate": 8.263354324357182e-05, "loss": 0.2755831182003021, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.3173, "step": 167, "tokens/total": 5712864, "tokens/train_per_sec_per_gpu": 39.06, "tokens/trainable": 112194} +{"epoch": 0.65625, "grad_norm": 2.388364315032959, "learning_rate": 8.240385928474219e-05, "loss": 0.3613125681877136, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 36.71, "memory/max_allocated (GiB)": 36.71, "ppl": 1.43521, "step": 168, "tokens/total": 5745344, "tokens/train_per_sec_per_gpu": 37.68, "tokens/trainable": 112795} +{"epoch": 0.66015625, "grad_norm": 1.7152913808822632, "learning_rate": 8.217303493946967e-05, "loss": 0.27485519647598267, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 37.13, "memory/max_allocated (GiB)": 37.13, "ppl": 1.31634, "step": 169, "tokens/total": 5779168, "tokens/train_per_sec_per_gpu": 37.34, "tokens/trainable": 113453} +{"epoch": 0.6640625, "grad_norm": 1.4682742357254028, "learning_rate": 8.194107981329746e-05, "loss": 0.2808246314525604, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 39.59, "memory/max_allocated (GiB)": 39.59, "ppl": 1.32422, "step": 170, "tokens/total": 5817328, "tokens/train_per_sec_per_gpu": 34.86, "tokens/trainable": 114131} +{"epoch": 0.66796875, "grad_norm": 1.840197205543518, "learning_rate": 8.170800355882518e-05, "loss": 0.2760164737701416, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.31787, "step": 171, "tokens/total": 5851392, "tokens/train_per_sec_per_gpu": 47.58, "tokens/trainable": 114857} +{"epoch": 0.671875, "grad_norm": 1.7986642122268677, "learning_rate": 8.147381587530713e-05, "loss": 0.34265661239624023, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.40868, "step": 172, "tokens/total": 5884896, "tokens/train_per_sec_per_gpu": 42.7, "tokens/trainable": 115536} +{"epoch": 0.67578125, "grad_norm": 2.0751495361328125, "learning_rate": 8.123852650824877e-05, "loss": 0.3132389783859253, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.36785, "step": 173, "tokens/total": 5918432, "tokens/train_per_sec_per_gpu": 41.64, "tokens/trainable": 116250} +{"epoch": 0.6796875, "grad_norm": 1.9132033586502075, "learning_rate": 8.100214524900103e-05, "loss": 0.2931598424911499, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 1.34066, "step": 174, "tokens/total": 5953200, "tokens/train_per_sec_per_gpu": 32.58, "tokens/trainable": 116874} +{"epoch": 0.68359375, "grad_norm": 1.945674180984497, "learning_rate": 8.076468193435301e-05, "loss": 0.3339589834213257, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 36.25, "memory/max_allocated (GiB)": 36.25, "ppl": 1.39649, "step": 175, "tokens/total": 5983584, "tokens/train_per_sec_per_gpu": 41.33, "tokens/trainable": 117458} +{"epoch": 0.6875, "grad_norm": 1.6718484163284302, "learning_rate": 8.052614644612253e-05, "loss": 0.30176448822021484, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.35224, "step": 176, "tokens/total": 6021104, "tokens/train_per_sec_per_gpu": 32.43, "tokens/trainable": 118149} +{"epoch": 0.69140625, "grad_norm": 1.442201852798462, "learning_rate": 8.028654871074489e-05, "loss": 0.24126020073890686, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.27285, "step": 177, "tokens/total": 6055184, "tokens/train_per_sec_per_gpu": 40.38, "tokens/trainable": 118867} +{"epoch": 0.6953125, "grad_norm": 2.0391223430633545, "learning_rate": 8.004589869885986e-05, "loss": 0.29914867877960205, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 36.3, "memory/max_allocated (GiB)": 36.3, "ppl": 1.34871, "step": 178, "tokens/total": 6086480, "tokens/train_per_sec_per_gpu": 37.18, "tokens/trainable": 119454} +{"epoch": 0.69921875, "grad_norm": 1.6174736022949219, "learning_rate": 7.980420642489674e-05, "loss": 0.2764813303947449, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.31848, "step": 179, "tokens/total": 6124864, "tokens/train_per_sec_per_gpu": 34.94, "tokens/trainable": 120142} +{"epoch": 0.703125, "grad_norm": 1.7523581981658936, "learning_rate": 7.95614819466576e-05, "loss": 0.2361496388912201, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 37.48, "memory/max_allocated (GiB)": 37.48, "ppl": 1.26636, "step": 180, "tokens/total": 6158208, "tokens/train_per_sec_per_gpu": 37.96, "tokens/trainable": 120813} +{"epoch": 0.70703125, "grad_norm": 1.531057357788086, "learning_rate": 7.931773536489872e-05, "loss": 0.2627113461494446, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 37.41, "memory/max_allocated (GiB)": 37.41, "ppl": 1.30045, "step": 181, "tokens/total": 6192480, "tokens/train_per_sec_per_gpu": 41.22, "tokens/trainable": 121479} +{"epoch": 0.7109375, "grad_norm": 2.8626549243927, "learning_rate": 7.907297682291035e-05, "loss": 0.2749757170677185, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.3165, "step": 182, "tokens/total": 6226864, "tokens/train_per_sec_per_gpu": 42.52, "tokens/trainable": 122190} +{"epoch": 0.71484375, "grad_norm": 1.1380226612091064, "learning_rate": 7.882721650609442e-05, "loss": 0.17836174368858337, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 39.48, "memory/max_allocated (GiB)": 39.48, "ppl": 1.19526, "step": 183, "tokens/total": 6266928, "tokens/train_per_sec_per_gpu": 37.21, "tokens/trainable": 122996} +{"epoch": 0.71875, "grad_norm": 2.0788638591766357, "learning_rate": 7.85804646415409e-05, "loss": 0.27652522921562195, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.31854, "step": 184, "tokens/total": 6301680, "tokens/train_per_sec_per_gpu": 40.82, "tokens/trainable": 123664} +{"epoch": 0.72265625, "grad_norm": 1.694990634918213, "learning_rate": 7.833273149760207e-05, "loss": 0.24164164066314697, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 1.27334, "step": 185, "tokens/total": 6331968, "tokens/train_per_sec_per_gpu": 42.18, "tokens/trainable": 124353} +{"epoch": 0.7265625, "grad_norm": 1.4834699630737305, "learning_rate": 7.808402738346527e-05, "loss": 0.2178739309310913, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.24343, "step": 186, "tokens/total": 6370448, "tokens/train_per_sec_per_gpu": 40.61, "tokens/trainable": 125099} +{"epoch": 0.73046875, "grad_norm": 1.7639089822769165, "learning_rate": 7.783436264872382e-05, "loss": 0.2498321384191513, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 38.93, "memory/max_allocated (GiB)": 38.93, "ppl": 1.28381, "step": 187, "tokens/total": 6406864, "tokens/train_per_sec_per_gpu": 39.28, "tokens/trainable": 125816} +{"epoch": 0.734375, "grad_norm": 2.1839253902435303, "learning_rate": 7.758374768294647e-05, "loss": 0.23658746480941772, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 38.08, "memory/max_allocated (GiB)": 38.08, "ppl": 1.26692, "step": 188, "tokens/total": 6442560, "tokens/train_per_sec_per_gpu": 42.89, "tokens/trainable": 126559} +{"epoch": 0.73828125, "grad_norm": 1.792409896850586, "learning_rate": 7.733219291524489e-05, "loss": 0.22432012856006622, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 37.39, "memory/max_allocated (GiB)": 37.39, "ppl": 1.25147, "step": 189, "tokens/total": 6475664, "tokens/train_per_sec_per_gpu": 42.04, "tokens/trainable": 127246} +{"epoch": 0.7421875, "grad_norm": 1.6380256414413452, "learning_rate": 7.707970881383977e-05, "loss": 0.22551462054252625, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 36.32, "memory/max_allocated (GiB)": 36.32, "ppl": 1.25297, "step": 190, "tokens/total": 6507072, "tokens/train_per_sec_per_gpu": 45.85, "tokens/trainable": 127920} +{"epoch": 0.74609375, "grad_norm": 1.6326735019683838, "learning_rate": 7.682630588562518e-05, "loss": 0.23328721523284912, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.26274, "step": 191, "tokens/total": 6541776, "tokens/train_per_sec_per_gpu": 38.74, "tokens/trainable": 128614} +{"epoch": 0.75, "grad_norm": 2.081289768218994, "learning_rate": 7.657199467573129e-05, "loss": 0.19796398282051086, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 36.93, "memory/max_allocated (GiB)": 36.93, "ppl": 1.21892, "step": 192, "tokens/total": 6575424, "tokens/train_per_sec_per_gpu": 37.11, "tokens/trainable": 129276} +{"epoch": 0.75390625, "grad_norm": 2.700859308242798, "learning_rate": 7.631678576708561e-05, "loss": 0.19437748193740845, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.21455, "step": 193, "tokens/total": 6609472, "tokens/train_per_sec_per_gpu": 45.57, "tokens/trainable": 130027} +{"epoch": 0.7578125, "grad_norm": 1.6653145551681519, "learning_rate": 7.606068977997255e-05, "loss": 0.21027374267578125, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 39.54, "memory/max_allocated (GiB)": 39.54, "ppl": 1.23402, "step": 194, "tokens/total": 6646816, "tokens/train_per_sec_per_gpu": 45.19, "tokens/trainable": 130801} +{"epoch": 0.76171875, "grad_norm": 2.3093600273132324, "learning_rate": 7.580371737159148e-05, "loss": 0.32959234714508057, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.3904, "step": 195, "tokens/total": 6682528, "tokens/train_per_sec_per_gpu": 34.37, "tokens/trainable": 131442} +{"epoch": 0.765625, "grad_norm": 1.9903547763824463, "learning_rate": 7.554587923561324e-05, "loss": 0.2222481369972229, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.24888, "step": 196, "tokens/total": 6716832, "tokens/train_per_sec_per_gpu": 36.3, "tokens/trainable": 132078} +{"epoch": 0.76953125, "grad_norm": 2.1120662689208984, "learning_rate": 7.528718610173511e-05, "loss": 0.2313268929719925, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 37.53, "memory/max_allocated (GiB)": 37.53, "ppl": 1.26027, "step": 197, "tokens/total": 6751840, "tokens/train_per_sec_per_gpu": 41.32, "tokens/trainable": 132754} +{"epoch": 0.7734375, "grad_norm": 1.9256956577301025, "learning_rate": 7.502764873523431e-05, "loss": 0.24153545498847961, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.2732, "step": 198, "tokens/total": 6785584, "tokens/train_per_sec_per_gpu": 41.89, "tokens/trainable": 133437} +{"epoch": 0.77734375, "grad_norm": 66.7388687133789, "learning_rate": 7.476727793652011e-05, "loss": 0.22602272033691406, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 1.2536, "step": 199, "tokens/total": 6818560, "tokens/train_per_sec_per_gpu": 40.98, "tokens/trainable": 134122} +{"epoch": 0.78125, "grad_norm": 1.7286641597747803, "learning_rate": 7.450608454068415e-05, "loss": 0.2147367298603058, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.23954, "step": 200, "tokens/total": 6852704, "tokens/train_per_sec_per_gpu": 39.1, "tokens/trainable": 134810} +{"epoch": 0.78515625, "grad_norm": 6.7780327796936035, "learning_rate": 7.424407941704987e-05, "loss": 0.22109156847000122, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 38.3, "memory/max_allocated (GiB)": 38.3, "ppl": 1.24744, "step": 201, "tokens/total": 6888912, "tokens/train_per_sec_per_gpu": 42.68, "tokens/trainable": 135532} +{"epoch": 0.7890625, "grad_norm": 1.4626611471176147, "learning_rate": 7.398127346871986e-05, "loss": 0.20236162841320038, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 39.52, "memory/max_allocated (GiB)": 39.52, "ppl": 1.22429, "step": 202, "tokens/total": 6925600, "tokens/train_per_sec_per_gpu": 37.54, "tokens/trainable": 136253} +{"epoch": 0.79296875, "grad_norm": 2.0672078132629395, "learning_rate": 7.371767763212238e-05, "loss": 0.24942372739315033, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.28329, "step": 203, "tokens/total": 6960640, "tokens/train_per_sec_per_gpu": 42.21, "tokens/trainable": 136934} +{"epoch": 0.796875, "grad_norm": 1.8632162809371948, "learning_rate": 7.345330287655617e-05, "loss": 0.24404074251651764, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.2764, "step": 204, "tokens/total": 6998352, "tokens/train_per_sec_per_gpu": 33.88, "tokens/trainable": 137607} +{"epoch": 0.80078125, "grad_norm": 3.0102245807647705, "learning_rate": 7.31881602037339e-05, "loss": 0.2493963986635208, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 36.6, "memory/max_allocated (GiB)": 36.6, "ppl": 1.28325, "step": 205, "tokens/total": 7030032, "tokens/train_per_sec_per_gpu": 38.66, "tokens/trainable": 138262} +{"epoch": 0.8046875, "grad_norm": 3.0927350521087646, "learning_rate": 7.29222606473245e-05, "loss": 0.20658771693706512, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 37.2, "memory/max_allocated (GiB)": 37.2, "ppl": 1.22948, "step": 206, "tokens/total": 7062528, "tokens/train_per_sec_per_gpu": 38.78, "tokens/trainable": 138907} +{"epoch": 0.80859375, "grad_norm": 1.8415437936782837, "learning_rate": 7.265561527249383e-05, "loss": 0.2401641309261322, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 37.07, "memory/max_allocated (GiB)": 37.07, "ppl": 1.27146, "step": 207, "tokens/total": 7096112, "tokens/train_per_sec_per_gpu": 41.07, "tokens/trainable": 139558} +{"epoch": 0.8125, "grad_norm": 1.120440125465393, "learning_rate": 7.238823517544436e-05, "loss": 0.15163862705230713, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.16374, "step": 208, "tokens/total": 7132848, "tokens/train_per_sec_per_gpu": 45.71, "tokens/trainable": 140335} +{"epoch": 0.81640625, "grad_norm": 4.8707098960876465, "learning_rate": 7.212013148295333e-05, "loss": 0.20718398690223694, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.23021, "step": 209, "tokens/total": 7166480, "tokens/train_per_sec_per_gpu": 43.36, "tokens/trainable": 141058} +{"epoch": 0.8203125, "grad_norm": 1.2272645235061646, "learning_rate": 7.185131535190975e-05, "loss": 0.19933301210403442, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.22059, "step": 210, "tokens/total": 7198784, "tokens/train_per_sec_per_gpu": 41.47, "tokens/trainable": 141778} +{"epoch": 0.82421875, "grad_norm": 2.1782851219177246, "learning_rate": 7.158179796885005e-05, "loss": 0.24988454580307007, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.28388, "step": 211, "tokens/total": 7232752, "tokens/train_per_sec_per_gpu": 41.54, "tokens/trainable": 142461} +{"epoch": 0.828125, "grad_norm": 2.7949390411376953, "learning_rate": 7.131159054949273e-05, "loss": 0.24283181130886078, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.27485, "step": 212, "tokens/total": 7270880, "tokens/train_per_sec_per_gpu": 31.86, "tokens/trainable": 143102} +{"epoch": 0.83203125, "grad_norm": 1.7884825468063354, "learning_rate": 7.104070433827139e-05, "loss": 0.22542718052864075, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.25286, "step": 213, "tokens/total": 7305936, "tokens/train_per_sec_per_gpu": 32.24, "tokens/trainable": 143677} +{"epoch": 0.8359375, "grad_norm": 3.397784471511841, "learning_rate": 7.076915060786705e-05, "loss": 0.23389169573783875, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.26351, "step": 214, "tokens/total": 7338272, "tokens/train_per_sec_per_gpu": 39.82, "tokens/trainable": 144329} +{"epoch": 0.83984375, "grad_norm": 1.6370620727539062, "learning_rate": 7.049694065873882e-05, "loss": 0.21753300726413727, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 37.98, "memory/max_allocated (GiB)": 37.98, "ppl": 1.24301, "step": 215, "tokens/total": 7372640, "tokens/train_per_sec_per_gpu": 36.63, "tokens/trainable": 144970} +{"epoch": 0.84375, "grad_norm": 2.082899570465088, "learning_rate": 7.022408581865382e-05, "loss": 0.21247407793998718, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.23673, "step": 216, "tokens/total": 7406400, "tokens/train_per_sec_per_gpu": 35.59, "tokens/trainable": 145646} +{"epoch": 0.84765625, "grad_norm": 3.0124590396881104, "learning_rate": 6.99505974422157e-05, "loss": 0.20206311345100403, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.22393, "step": 217, "tokens/total": 7440544, "tokens/train_per_sec_per_gpu": 46.13, "tokens/trainable": 146350} +{"epoch": 0.8515625, "grad_norm": 2.35534930229187, "learning_rate": 6.967648691039213e-05, "loss": 0.2437695860862732, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.27605, "step": 218, "tokens/total": 7472032, "tokens/train_per_sec_per_gpu": 42.88, "tokens/trainable": 146992} +{"epoch": 0.85546875, "grad_norm": 1.8650786876678467, "learning_rate": 6.940176563004123e-05, "loss": 0.23778927326202393, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.26844, "step": 219, "tokens/total": 7505824, "tokens/train_per_sec_per_gpu": 36.17, "tokens/trainable": 147611} +{"epoch": 0.859375, "grad_norm": 2.3178160190582275, "learning_rate": 6.912644503343682e-05, "loss": 0.22518965601921082, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.25256, "step": 220, "tokens/total": 7543600, "tokens/train_per_sec_per_gpu": 35.52, "tokens/trainable": 148248} +{"epoch": 0.86328125, "grad_norm": 2.2471983432769775, "learning_rate": 6.885053657779273e-05, "loss": 0.19714874029159546, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 36.14, "memory/max_allocated (GiB)": 36.14, "ppl": 1.21793, "step": 221, "tokens/total": 7574544, "tokens/train_per_sec_per_gpu": 40.88, "tokens/trainable": 148922} +{"epoch": 0.8671875, "grad_norm": 1.5852190256118774, "learning_rate": 6.857405174478604e-05, "loss": 0.22392058372497559, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 37.2, "memory/max_allocated (GiB)": 37.2, "ppl": 1.25097, "step": 222, "tokens/total": 7608832, "tokens/train_per_sec_per_gpu": 36.71, "tokens/trainable": 149576} +{"epoch": 0.87109375, "grad_norm": 1.8570395708084106, "learning_rate": 6.82970020400792e-05, "loss": 0.20546670258045197, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.2281, "step": 223, "tokens/total": 7645664, "tokens/train_per_sec_per_gpu": 33.63, "tokens/trainable": 150225} +{"epoch": 0.875, "grad_norm": 3.7743232250213623, "learning_rate": 6.801939899284132e-05, "loss": 0.1638336181640625, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 38.15, "memory/max_allocated (GiB)": 38.15, "ppl": 1.17802, "step": 224, "tokens/total": 7681200, "tokens/train_per_sec_per_gpu": 41.18, "tokens/trainable": 150960} +{"epoch": 0.87890625, "grad_norm": 1.4322007894515991, "learning_rate": 6.774125415526827e-05, "loss": 0.17465314269065857, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 37.84, "memory/max_allocated (GiB)": 37.84, "ppl": 1.19083, "step": 225, "tokens/total": 7714160, "tokens/train_per_sec_per_gpu": 47.48, "tokens/trainable": 151710} +{"epoch": 0.8828125, "grad_norm": 2.328752040863037, "learning_rate": 6.746257910210214e-05, "loss": 0.20440371334552765, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 36.92, "memory/max_allocated (GiB)": 36.92, "ppl": 1.22679, "step": 226, "tokens/total": 7746944, "tokens/train_per_sec_per_gpu": 38.71, "tokens/trainable": 152349} +{"epoch": 0.88671875, "grad_norm": 1.4580092430114746, "learning_rate": 6.718338543014937e-05, "loss": 0.21970264613628387, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 35.52, "memory/max_allocated (GiB)": 35.52, "ppl": 1.24571, "step": 227, "tokens/total": 7776576, "tokens/train_per_sec_per_gpu": 41.88, "tokens/trainable": 152959} +{"epoch": 0.890625, "grad_norm": 1.5249884128570557, "learning_rate": 6.69036847577983e-05, "loss": 0.18466152250766754, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.20281, "step": 228, "tokens/total": 7810000, "tokens/train_per_sec_per_gpu": 37.68, "tokens/trainable": 153636} +{"epoch": 0.89453125, "grad_norm": 1.6127736568450928, "learning_rate": 6.662348872453553e-05, "loss": 0.16728657484054565, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 38.3, "memory/max_allocated (GiB)": 38.3, "ppl": 1.18209, "step": 229, "tokens/total": 7846112, "tokens/train_per_sec_per_gpu": 38.93, "tokens/trainable": 154305} +{"epoch": 0.8984375, "grad_norm": 1.7744253873825073, "learning_rate": 6.63428089904618e-05, "loss": 0.22208736836910248, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.24868, "step": 230, "tokens/total": 7884864, "tokens/train_per_sec_per_gpu": 34.98, "tokens/trainable": 154966} +{"epoch": 0.90234375, "grad_norm": 1.570340871810913, "learning_rate": 6.60616572358065e-05, "loss": 0.201472669839859, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.2232, "step": 231, "tokens/total": 7918800, "tokens/train_per_sec_per_gpu": 37.12, "tokens/trainable": 155536} +{"epoch": 0.90625, "grad_norm": 1.4622764587402344, "learning_rate": 6.578004516044172e-05, "loss": 0.19359144568443298, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.2136, "step": 232, "tokens/total": 7954272, "tokens/train_per_sec_per_gpu": 43.44, "tokens/trainable": 156219} +{"epoch": 0.91015625, "grad_norm": 1.9623457193374634, "learning_rate": 6.549798448339548e-05, "loss": 0.18817558884620667, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.20705, "step": 233, "tokens/total": 7991328, "tokens/train_per_sec_per_gpu": 38.82, "tokens/trainable": 156903} +{"epoch": 0.9140625, "grad_norm": 1.6679198741912842, "learning_rate": 6.521548694236384e-05, "loss": 0.17482668161392212, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 39.19, "memory/max_allocated (GiB)": 39.19, "ppl": 1.19104, "step": 234, "tokens/total": 8028224, "tokens/train_per_sec_per_gpu": 34.45, "tokens/trainable": 157556} +{"epoch": 0.91796875, "grad_norm": 1.3442260026931763, "learning_rate": 6.493256429322259e-05, "loss": 0.18863645195960999, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.2076, "step": 235, "tokens/total": 8064000, "tokens/train_per_sec_per_gpu": 36.85, "tokens/trainable": 158212} +{"epoch": 0.921875, "grad_norm": 1.6009796857833862, "learning_rate": 6.464922830953799e-05, "loss": 0.2084033489227295, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 39.19, "memory/max_allocated (GiB)": 39.19, "ppl": 1.23171, "step": 236, "tokens/total": 8102848, "tokens/train_per_sec_per_gpu": 32.44, "tokens/trainable": 158867} +{"epoch": 0.92578125, "grad_norm": 1.9420561790466309, "learning_rate": 6.436549078207688e-05, "loss": 0.19103631377220154, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 38.11, "memory/max_allocated (GiB)": 38.11, "ppl": 1.2105, "step": 237, "tokens/total": 8135584, "tokens/train_per_sec_per_gpu": 41.57, "tokens/trainable": 159512} +{"epoch": 0.9296875, "grad_norm": 1.6574395895004272, "learning_rate": 6.408136351831592e-05, "loss": 0.20638906955718994, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.22923, "step": 238, "tokens/total": 8167552, "tokens/train_per_sec_per_gpu": 36.54, "tokens/trainable": 160151} +{"epoch": 0.93359375, "grad_norm": 1.1719045639038086, "learning_rate": 6.379685834195036e-05, "loss": 0.16327345371246338, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.17736, "step": 239, "tokens/total": 8206704, "tokens/train_per_sec_per_gpu": 37.35, "tokens/trainable": 160915} +{"epoch": 0.9375, "grad_norm": 1.8931596279144287, "learning_rate": 6.351198709240186e-05, "loss": 0.17483891546726227, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.19105, "step": 240, "tokens/total": 8246480, "tokens/train_per_sec_per_gpu": 31.96, "tokens/trainable": 161610} +{"epoch": 0.94140625, "grad_norm": 2.196159839630127, "learning_rate": 6.32267616243259e-05, "loss": 0.22515128552913666, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.25251, "step": 241, "tokens/total": 8278688, "tokens/train_per_sec_per_gpu": 41.15, "tokens/trainable": 162268} +{"epoch": 0.9453125, "grad_norm": 2.3417580127716064, "learning_rate": 6.294119380711849e-05, "loss": 0.21366465091705322, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.23821, "step": 242, "tokens/total": 8312688, "tokens/train_per_sec_per_gpu": 33.37, "tokens/trainable": 162885} +{"epoch": 0.94921875, "grad_norm": 1.9580541849136353, "learning_rate": 6.265529552442209e-05, "loss": 0.1987101435661316, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 37.57, "memory/max_allocated (GiB)": 37.57, "ppl": 1.21983, "step": 243, "tokens/total": 8347664, "tokens/train_per_sec_per_gpu": 42.76, "tokens/trainable": 163577} +{"epoch": 0.953125, "grad_norm": 3.2868542671203613, "learning_rate": 6.236907867363127e-05, "loss": 0.20866143703460693, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.23203, "step": 244, "tokens/total": 8381680, "tokens/train_per_sec_per_gpu": 39.57, "tokens/trainable": 164217} +{"epoch": 0.95703125, "grad_norm": 2.577413558959961, "learning_rate": 6.208255516539749e-05, "loss": 0.20426896214485168, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 36.21, "memory/max_allocated (GiB)": 36.21, "ppl": 1.22663, "step": 245, "tokens/total": 8413408, "tokens/train_per_sec_per_gpu": 36.16, "tokens/trainable": 164858} +{"epoch": 0.9609375, "grad_norm": 1.4583749771118164, "learning_rate": 6.179573692313344e-05, "loss": 0.20337189733982086, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.22553, "step": 246, "tokens/total": 8450048, "tokens/train_per_sec_per_gpu": 39.77, "tokens/trainable": 165531} +{"epoch": 0.96484375, "grad_norm": 1.3437293767929077, "learning_rate": 6.150863588251694e-05, "loss": 0.1886528879404068, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 39.35, "memory/max_allocated (GiB)": 39.35, "ppl": 1.20762, "step": 247, "tokens/total": 8489968, "tokens/train_per_sec_per_gpu": 33.26, "tokens/trainable": 166220} +{"epoch": 0.96875, "grad_norm": 1.4862010478973389, "learning_rate": 6.122126399099419e-05, "loss": 0.17755329608917236, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 39.59, "memory/max_allocated (GiB)": 39.59, "ppl": 1.19429, "step": 248, "tokens/total": 8529456, "tokens/train_per_sec_per_gpu": 44.81, "tokens/trainable": 167055} +{"epoch": 0.97265625, "grad_norm": 1.6825268268585205, "learning_rate": 6.0933633207282615e-05, "loss": 0.21861961483955383, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 38.22, "memory/max_allocated (GiB)": 38.22, "ppl": 1.24436, "step": 249, "tokens/total": 8565472, "tokens/train_per_sec_per_gpu": 36.98, "tokens/trainable": 167741} +{"epoch": 0.9765625, "grad_norm": 2.8801658153533936, "learning_rate": 6.064575550087316e-05, "loss": 0.17819218337535858, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.19505, "step": 250, "tokens/total": 8602128, "tokens/train_per_sec_per_gpu": 36.94, "tokens/trainable": 168436} +{"epoch": 0.98046875, "grad_norm": 1.6499273777008057, "learning_rate": 6.0357642851532245e-05, "loss": 0.18607816100120544, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.20452, "step": 251, "tokens/total": 8638304, "tokens/train_per_sec_per_gpu": 34.57, "tokens/trainable": 169118} +{"epoch": 0.984375, "grad_norm": 2.3719804286956787, "learning_rate": 6.0069307248803294e-05, "loss": 0.19623270630836487, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.21681, "step": 252, "tokens/total": 8673248, "tokens/train_per_sec_per_gpu": 33.99, "tokens/trainable": 169764} +{"epoch": 0.98828125, "grad_norm": 1.4563759565353394, "learning_rate": 5.9780760691507635e-05, "loss": 0.1956365406513214, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 36.16, "memory/max_allocated (GiB)": 36.16, "ppl": 1.21608, "step": 253, "tokens/total": 8704208, "tokens/train_per_sec_per_gpu": 37.27, "tokens/trainable": 170345} +{"epoch": 0.9921875, "grad_norm": 1.4285749197006226, "learning_rate": 5.9492015187245334e-05, "loss": 0.18212318420410156, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 38.63, "memory/max_allocated (GiB)": 38.63, "ppl": 1.19976, "step": 254, "tokens/total": 8741408, "tokens/train_per_sec_per_gpu": 34.67, "tokens/trainable": 170970} +{"epoch": 0.99609375, "grad_norm": 1.9386231899261475, "learning_rate": 5.920308275189541e-05, "loss": 0.2017609030008316, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 36.54, "memory/max_allocated (GiB)": 36.54, "ppl": 1.22356, "step": 255, "tokens/total": 8772832, "tokens/train_per_sec_per_gpu": 37.51, "tokens/trainable": 171568} +{"epoch": 1.0, "grad_norm": 1.3511930704116821, "learning_rate": 5.8913975409115874e-05, "loss": 0.149319127202034, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 37.57, "memory/max_allocated (GiB)": 37.57, "ppl": 1.16104, "step": 256, "tokens/total": 8805840, "tokens/train_per_sec_per_gpu": 40.69, "tokens/trainable": 172270} +{"epoch": 1.00390625, "grad_norm": 0.9931016564369202, "learning_rate": 5.8624705189843395e-05, "loss": 0.14290717244148254, "memory/device_reserved (GiB)": 43.82, "memory/max_active (GiB)": 39.04, "memory/max_allocated (GiB)": 39.04, "ppl": 1.15362, "step": 257, "tokens/total": 8842848, "tokens/train_per_sec_per_gpu": 35.62, "tokens/trainable": 172985} +{"epoch": 1.0078125, "grad_norm": 1.526422381401062, "learning_rate": 5.833528413179249e-05, "loss": 0.15460556745529175, "memory/device_reserved (GiB)": 40.53, "memory/max_active (GiB)": 38.41, "memory/max_allocated (GiB)": 38.41, "ppl": 1.1672, "step": 258, "tokens/total": 8879088, "tokens/train_per_sec_per_gpu": 36.06, "tokens/trainable": 173694} +{"epoch": 1.01171875, "grad_norm": 1.7134515047073364, "learning_rate": 5.80457242789548e-05, "loss": 0.13662612438201904, "memory/device_reserved (GiB)": 41.82, "memory/max_active (GiB)": 39.48, "memory/max_allocated (GiB)": 39.48, "ppl": 1.1464, "step": 259, "tokens/total": 8918912, "tokens/train_per_sec_per_gpu": 38.31, "tokens/trainable": 174490} +{"epoch": 1.015625, "grad_norm": 0.8582515716552734, "learning_rate": 5.77560376810977e-05, "loss": 0.11668236553668976, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.12376, "step": 260, "tokens/total": 8951920, "tokens/train_per_sec_per_gpu": 44.54, "tokens/trainable": 175220} +{"epoch": 1.01953125, "grad_norm": 1.7394729852676392, "learning_rate": 5.7466236393263005e-05, "loss": 0.15629205107688904, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 38.93, "memory/max_allocated (GiB)": 38.93, "ppl": 1.16917, "step": 261, "tokens/total": 8986272, "tokens/train_per_sec_per_gpu": 38.23, "tokens/trainable": 175876} +{"epoch": 1.0234375, "grad_norm": 3.1339664459228516, "learning_rate": 5.717633247526522e-05, "loss": 0.20672495663166046, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 36.63, "memory/max_allocated (GiB)": 36.63, "ppl": 1.22964, "step": 262, "tokens/total": 9016640, "tokens/train_per_sec_per_gpu": 37.58, "tokens/trainable": 176454} +{"epoch": 1.02734375, "grad_norm": 2.2650184631347656, "learning_rate": 5.688633799118971e-05, "loss": 0.18177013099193573, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 36.82, "memory/max_allocated (GiB)": 36.82, "ppl": 1.19934, "step": 263, "tokens/total": 9048784, "tokens/train_per_sec_per_gpu": 34.02, "tokens/trainable": 177046} +{"epoch": 1.03125, "grad_norm": 4.007791519165039, "learning_rate": 5.659626500889066e-05, "loss": 0.13843724131584167, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 36.93, "memory/max_allocated (GiB)": 36.93, "ppl": 1.14848, "step": 264, "tokens/total": 9082208, "tokens/train_per_sec_per_gpu": 43.04, "tokens/trainable": 177771} +{"epoch": 1.03515625, "grad_norm": 1.4977930784225464, "learning_rate": 5.6306125599488905e-05, "loss": 0.17278625071048737, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 36.72, "memory/max_allocated (GiB)": 36.72, "ppl": 1.18861, "step": 265, "tokens/total": 9112560, "tokens/train_per_sec_per_gpu": 47.39, "tokens/trainable": 178408} +{"epoch": 1.0390625, "grad_norm": 2.0149013996124268, "learning_rate": 5.601593183686955e-05, "loss": 0.15140728652477264, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.16347, "step": 266, "tokens/total": 9149392, "tokens/train_per_sec_per_gpu": 40.93, "tokens/trainable": 179136} +{"epoch": 1.04296875, "grad_norm": 1.7880645990371704, "learning_rate": 5.572569579717961e-05, "loss": 0.15324492752552032, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.16561, "step": 267, "tokens/total": 9184448, "tokens/train_per_sec_per_gpu": 36.41, "tokens/trainable": 179831} +{"epoch": 1.046875, "grad_norm": 1.9220428466796875, "learning_rate": 5.543542955832538e-05, "loss": 0.1591005176305771, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.17246, "step": 268, "tokens/total": 9224064, "tokens/train_per_sec_per_gpu": 36.48, "tokens/trainable": 180554} +{"epoch": 1.05078125, "grad_norm": 1.7219276428222656, "learning_rate": 5.514514519946986e-05, "loss": 0.14676107466220856, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.15808, "step": 269, "tokens/total": 9264048, "tokens/train_per_sec_per_gpu": 40.03, "tokens/trainable": 181311} +{"epoch": 1.0546875, "grad_norm": 1.3536756038665771, "learning_rate": 5.485485480053015e-05, "loss": 0.17102546989917755, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.18652, "step": 270, "tokens/total": 9302128, "tokens/train_per_sec_per_gpu": 31.26, "tokens/trainable": 181942} +{"epoch": 1.05859375, "grad_norm": 3.413591146469116, "learning_rate": 5.4564570441674645e-05, "loss": 0.1563137173652649, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.16919, "step": 271, "tokens/total": 9335408, "tokens/train_per_sec_per_gpu": 39.96, "tokens/trainable": 182578} +{"epoch": 1.0625, "grad_norm": 2.2720441818237305, "learning_rate": 5.42743042028204e-05, "loss": 0.17786508798599243, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 37.73, "memory/max_allocated (GiB)": 37.73, "ppl": 1.19466, "step": 272, "tokens/total": 9370608, "tokens/train_per_sec_per_gpu": 29.96, "tokens/trainable": 183177} +{"epoch": 1.06640625, "grad_norm": 1.5681296586990356, "learning_rate": 5.3984068163130464e-05, "loss": 0.1532503217458725, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.16562, "step": 273, "tokens/total": 9403808, "tokens/train_per_sec_per_gpu": 43.53, "tokens/trainable": 183921} +{"epoch": 1.0703125, "grad_norm": 1.4230574369430542, "learning_rate": 5.369387440051111e-05, "loss": 0.1568634808063507, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.16984, "step": 274, "tokens/total": 9437072, "tokens/train_per_sec_per_gpu": 36.72, "tokens/trainable": 184570} +{"epoch": 1.07421875, "grad_norm": 1.1238150596618652, "learning_rate": 5.340373499110935e-05, "loss": 0.14641380310058594, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 1.15768, "step": 275, "tokens/total": 9472496, "tokens/train_per_sec_per_gpu": 38.43, "tokens/trainable": 185282} +{"epoch": 1.078125, "grad_norm": 1.398693561553955, "learning_rate": 5.3113662008810304e-05, "loss": 0.13118068873882294, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.14017, "step": 276, "tokens/total": 9512512, "tokens/train_per_sec_per_gpu": 36.46, "tokens/trainable": 186013} +{"epoch": 1.08203125, "grad_norm": 0.9780323505401611, "learning_rate": 5.282366752473479e-05, "loss": 0.13285382091999054, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 38.1, "memory/max_allocated (GiB)": 38.1, "ppl": 1.14208, "step": 277, "tokens/total": 9549376, "tokens/train_per_sec_per_gpu": 36.18, "tokens/trainable": 186691} +{"epoch": 1.0859375, "grad_norm": 2.494959831237793, "learning_rate": 5.2533763606737005e-05, "loss": 0.17351368069648743, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.18948, "step": 278, "tokens/total": 9588880, "tokens/train_per_sec_per_gpu": 29.42, "tokens/trainable": 187323} +{"epoch": 1.08984375, "grad_norm": 1.0207061767578125, "learning_rate": 5.224396231890232e-05, "loss": 0.1414400339126587, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.15193, "step": 279, "tokens/total": 9620416, "tokens/train_per_sec_per_gpu": 44.11, "tokens/trainable": 188053} +{"epoch": 1.09375, "grad_norm": 1.607839584350586, "learning_rate": 5.195427572104522e-05, "loss": 0.19107313454151154, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 38.81, "memory/max_allocated (GiB)": 38.81, "ppl": 1.21055, "step": 280, "tokens/total": 9654544, "tokens/train_per_sec_per_gpu": 39.98, "tokens/trainable": 188720} +{"epoch": 1.09765625, "grad_norm": 1.520923376083374, "learning_rate": 5.166471586820751e-05, "loss": 0.16607753932476044, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 39.28, "memory/max_allocated (GiB)": 39.28, "ppl": 1.18066, "step": 281, "tokens/total": 9691056, "tokens/train_per_sec_per_gpu": 41.85, "tokens/trainable": 189465} +{"epoch": 1.1015625, "grad_norm": 1.6356604099273682, "learning_rate": 5.1375294810156615e-05, "loss": 0.1393471658229828, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.14952, "step": 282, "tokens/total": 9725280, "tokens/train_per_sec_per_gpu": 38.34, "tokens/trainable": 190092} +{"epoch": 1.10546875, "grad_norm": 1.7738490104675293, "learning_rate": 5.1086024590884144e-05, "loss": 0.18234126269817352, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 39.09, "memory/max_allocated (GiB)": 39.09, "ppl": 1.20002, "step": 283, "tokens/total": 9763056, "tokens/train_per_sec_per_gpu": 35.68, "tokens/trainable": 190744} +{"epoch": 1.109375, "grad_norm": 0.809921145439148, "learning_rate": 5.079691724810461e-05, "loss": 0.1358060985803604, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.14546, "step": 284, "tokens/total": 9798480, "tokens/train_per_sec_per_gpu": 31.44, "tokens/trainable": 191365} +{"epoch": 1.11328125, "grad_norm": 1.406798005104065, "learning_rate": 5.0507984812754684e-05, "loss": 0.14350895583629608, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 37.19, "memory/max_allocated (GiB)": 37.19, "ppl": 1.15432, "step": 285, "tokens/total": 9831712, "tokens/train_per_sec_per_gpu": 42.29, "tokens/trainable": 192067} +{"epoch": 1.1171875, "grad_norm": 1.2268260717391968, "learning_rate": 5.021923930849237e-05, "loss": 0.1545265167951584, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.16711, "step": 286, "tokens/total": 9865216, "tokens/train_per_sec_per_gpu": 38.11, "tokens/trainable": 192716} +{"epoch": 1.12109375, "grad_norm": 1.968601107597351, "learning_rate": 4.99306927511967e-05, "loss": 0.16947321593761444, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.18468, "step": 287, "tokens/total": 9898256, "tokens/train_per_sec_per_gpu": 39.95, "tokens/trainable": 193313} +{"epoch": 1.125, "grad_norm": 1.6755117177963257, "learning_rate": 4.964235714846775e-05, "loss": 0.16938632726669312, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 36.31, "memory/max_allocated (GiB)": 36.31, "ppl": 1.18458, "step": 288, "tokens/total": 9929712, "tokens/train_per_sec_per_gpu": 40.12, "tokens/trainable": 193900} +{"epoch": 1.12890625, "grad_norm": 1.7340739965438843, "learning_rate": 4.9354244499126866e-05, "loss": 0.1933109164237976, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 36.11, "memory/max_allocated (GiB)": 36.11, "ppl": 1.21326, "step": 289, "tokens/total": 9961040, "tokens/train_per_sec_per_gpu": 40.95, "tokens/trainable": 194587} +{"epoch": 1.1328125, "grad_norm": 1.3607739210128784, "learning_rate": 4.90663667927174e-05, "loss": 0.15762463212013245, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.17073, "step": 290, "tokens/total": 9993456, "tokens/train_per_sec_per_gpu": 47.43, "tokens/trainable": 195257} +{"epoch": 1.13671875, "grad_norm": 0.9433422684669495, "learning_rate": 4.877873600900581e-05, "loss": 0.13796964287757874, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 37.98, "memory/max_allocated (GiB)": 37.98, "ppl": 1.14794, "step": 291, "tokens/total": 10029280, "tokens/train_per_sec_per_gpu": 35.65, "tokens/trainable": 195939} +{"epoch": 1.140625, "grad_norm": 1.1635651588439941, "learning_rate": 4.849136411748306e-05, "loss": 0.1471775323152542, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 1.15856, "step": 292, "tokens/total": 10067072, "tokens/train_per_sec_per_gpu": 35.78, "tokens/trainable": 196625} +{"epoch": 1.14453125, "grad_norm": 1.2928754091262817, "learning_rate": 4.8204263076866574e-05, "loss": 0.15873798727989197, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 38.66, "memory/max_allocated (GiB)": 38.66, "ppl": 1.17203, "step": 293, "tokens/total": 10102544, "tokens/train_per_sec_per_gpu": 39.1, "tokens/trainable": 197330} +{"epoch": 1.1484375, "grad_norm": 1.4203320741653442, "learning_rate": 4.791744483460251e-05, "loss": 0.19882899522781372, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.21997, "step": 294, "tokens/total": 10136624, "tokens/train_per_sec_per_gpu": 31.97, "tokens/trainable": 197945} +{"epoch": 1.15234375, "grad_norm": 1.2103506326675415, "learning_rate": 4.7630921326368736e-05, "loss": 0.17272409796714783, "memory/device_reserved (GiB)": 43.48, "memory/max_active (GiB)": 39.19, "memory/max_allocated (GiB)": 39.19, "ppl": 1.18854, "step": 295, "tokens/total": 10174864, "tokens/train_per_sec_per_gpu": 29.67, "tokens/trainable": 198542} +{"epoch": 1.15625, "grad_norm": 0.9579172134399414, "learning_rate": 4.7344704475577916e-05, "loss": 0.14734038710594177, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 35.48, "memory/max_allocated (GiB)": 35.48, "ppl": 1.15875, "step": 296, "tokens/total": 10204016, "tokens/train_per_sec_per_gpu": 42.82, "tokens/trainable": 199199} +{"epoch": 1.16015625, "grad_norm": 2.1831045150756836, "learning_rate": 4.705880619288153e-05, "loss": 0.17413246631622314, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.18, "memory/max_allocated (GiB)": 38.18, "ppl": 1.19021, "step": 297, "tokens/total": 10239696, "tokens/train_per_sec_per_gpu": 35.29, "tokens/trainable": 199825} +{"epoch": 1.1640625, "grad_norm": 0.9848728179931641, "learning_rate": 4.677323837567412e-05, "loss": 0.14446690678596497, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.15542, "step": 298, "tokens/total": 10272192, "tokens/train_per_sec_per_gpu": 45.13, "tokens/trainable": 200534} +{"epoch": 1.16796875, "grad_norm": 1.0848960876464844, "learning_rate": 4.6488012907598146e-05, "loss": 0.14375025033950806, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.3, "memory/max_allocated (GiB)": 36.3, "ppl": 1.1546, "step": 299, "tokens/total": 10304128, "tokens/train_per_sec_per_gpu": 38.35, "tokens/trainable": 201167} +{"epoch": 1.171875, "grad_norm": 1.172926902770996, "learning_rate": 4.620314165804964e-05, "loss": 0.138075053691864, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.26, "memory/max_allocated (GiB)": 38.26, "ppl": 1.14806, "step": 300, "tokens/total": 10339536, "tokens/train_per_sec_per_gpu": 38.46, "tokens/trainable": 201857} +{"epoch": 1.17578125, "grad_norm": 2.034454107284546, "learning_rate": 4.591863648168407e-05, "loss": 0.11965497583150864, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.12711, "step": 301, "tokens/total": 10373808, "tokens/train_per_sec_per_gpu": 39.82, "tokens/trainable": 202596} +{"epoch": 1.1796875, "grad_norm": 0.9718875288963318, "learning_rate": 4.5634509217923135e-05, "loss": 0.15204478800296783, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.16421, "step": 302, "tokens/total": 10406928, "tokens/train_per_sec_per_gpu": 42.5, "tokens/trainable": 203278} +{"epoch": 1.18359375, "grad_norm": 1.2038300037384033, "learning_rate": 4.535077169046201e-05, "loss": 0.14323580265045166, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.93, "memory/max_allocated (GiB)": 37.93, "ppl": 1.154, "step": 303, "tokens/total": 10443216, "tokens/train_per_sec_per_gpu": 37.15, "tokens/trainable": 203948} +{"epoch": 1.1875, "grad_norm": 1.62296724319458, "learning_rate": 4.506743570677743e-05, "loss": 0.1601068377494812, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.13, "memory/max_allocated (GiB)": 36.13, "ppl": 1.17364, "step": 304, "tokens/total": 10473040, "tokens/train_per_sec_per_gpu": 46.48, "tokens/trainable": 204561} +{"epoch": 1.19140625, "grad_norm": 2.330272912979126, "learning_rate": 4.478451305763618e-05, "loss": 0.17400048673152924, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.17, "memory/max_allocated (GiB)": 37.17, "ppl": 1.19006, "step": 305, "tokens/total": 10506288, "tokens/train_per_sec_per_gpu": 44.63, "tokens/trainable": 205228} +{"epoch": 1.1953125, "grad_norm": 2.3298399448394775, "learning_rate": 4.450201551660454e-05, "loss": 0.2033878117799759, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.27, "memory/max_allocated (GiB)": 36.27, "ppl": 1.22555, "step": 306, "tokens/total": 10537120, "tokens/train_per_sec_per_gpu": 40.24, "tokens/trainable": 205861} +{"epoch": 1.19921875, "grad_norm": 0.9678407311439514, "learning_rate": 4.4219954839558276e-05, "loss": 0.16096338629722595, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.17464, "step": 307, "tokens/total": 10571184, "tokens/train_per_sec_per_gpu": 36.07, "tokens/trainable": 206504} +{"epoch": 1.203125, "grad_norm": 2.347663164138794, "learning_rate": 4.393834276419352e-05, "loss": 0.16802524030208588, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 35.6, "memory/max_allocated (GiB)": 35.6, "ppl": 1.18297, "step": 308, "tokens/total": 10600736, "tokens/train_per_sec_per_gpu": 48.78, "tokens/trainable": 207192} +{"epoch": 1.20703125, "grad_norm": 0.9621609449386597, "learning_rate": 4.36571910095382e-05, "loss": 0.1362133026123047, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.14593, "step": 309, "tokens/total": 10638720, "tokens/train_per_sec_per_gpu": 38.27, "tokens/trainable": 207847} +{"epoch": 1.2109375, "grad_norm": 1.388282299041748, "learning_rate": 4.337651127546448e-05, "loss": 0.16650494933128357, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.18117, "step": 310, "tokens/total": 10673936, "tokens/train_per_sec_per_gpu": 41.96, "tokens/trainable": 208504} +{"epoch": 1.21484375, "grad_norm": 14.833683013916016, "learning_rate": 4.3096315242201736e-05, "loss": 0.15888875722885132, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.37, "memory/max_allocated (GiB)": 36.37, "ppl": 1.17221, "step": 311, "tokens/total": 10706256, "tokens/train_per_sec_per_gpu": 38.02, "tokens/trainable": 209147} +{"epoch": 1.21875, "grad_norm": 6.001370429992676, "learning_rate": 4.2816614569850635e-05, "loss": 0.15021729469299316, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.49, "memory/max_allocated (GiB)": 38.49, "ppl": 1.16209, "step": 312, "tokens/total": 10741696, "tokens/train_per_sec_per_gpu": 37.58, "tokens/trainable": 209804} +{"epoch": 1.22265625, "grad_norm": 1.8523434400558472, "learning_rate": 4.2537420897897864e-05, "loss": 0.1415359377861023, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.63, "memory/max_allocated (GiB)": 38.63, "ppl": 1.15204, "step": 313, "tokens/total": 10776928, "tokens/train_per_sec_per_gpu": 36.13, "tokens/trainable": 210508} +{"epoch": 1.2265625, "grad_norm": 1.0148247480392456, "learning_rate": 4.225874584473174e-05, "loss": 0.13727039098739624, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.14714, "step": 314, "tokens/total": 10808976, "tokens/train_per_sec_per_gpu": 38.47, "tokens/trainable": 211136} +{"epoch": 1.23046875, "grad_norm": 0.8008854985237122, "learning_rate": 4.19806010071587e-05, "loss": 0.12315002828836441, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.36, "memory/max_allocated (GiB)": 38.36, "ppl": 1.13105, "step": 315, "tokens/total": 10846640, "tokens/train_per_sec_per_gpu": 37.58, "tokens/trainable": 211868} +{"epoch": 1.234375, "grad_norm": 1.3223094940185547, "learning_rate": 4.170299795992081e-05, "loss": 0.1661267727613449, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.74, "memory/max_allocated (GiB)": 36.74, "ppl": 1.18072, "step": 316, "tokens/total": 10879792, "tokens/train_per_sec_per_gpu": 38.23, "tokens/trainable": 212491} +{"epoch": 1.23828125, "grad_norm": 1.4925403594970703, "learning_rate": 4.142594825521398e-05, "loss": 0.13574539124965668, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.12, "memory/max_allocated (GiB)": 39.12, "ppl": 1.14539, "step": 317, "tokens/total": 10918640, "tokens/train_per_sec_per_gpu": 34.24, "tokens/trainable": 213184} +{"epoch": 1.2421875, "grad_norm": 1.678476333618164, "learning_rate": 4.114946342220728e-05, "loss": 0.13305211067199707, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 1.14231, "step": 318, "tokens/total": 10954976, "tokens/train_per_sec_per_gpu": 38.01, "tokens/trainable": 213903} +{"epoch": 1.24609375, "grad_norm": 1.6233742237091064, "learning_rate": 4.087355496656321e-05, "loss": 0.1469430923461914, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 1.15829, "step": 319, "tokens/total": 10988320, "tokens/train_per_sec_per_gpu": 43.37, "tokens/trainable": 214596} +{"epoch": 1.25, "grad_norm": 1.7199351787567139, "learning_rate": 4.05982343699588e-05, "loss": 0.1385396122932434, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.1486, "step": 320, "tokens/total": 11027216, "tokens/train_per_sec_per_gpu": 38.12, "tokens/trainable": 215353} +{"epoch": 1.25390625, "grad_norm": 1.0817683935165405, "learning_rate": 4.0323513089607876e-05, "loss": 0.13734537363052368, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.14722, "step": 321, "tokens/total": 11061680, "tokens/train_per_sec_per_gpu": 39.16, "tokens/trainable": 216071} +{"epoch": 1.2578125, "grad_norm": 1.547626256942749, "learning_rate": 4.004940255778431e-05, "loss": 0.14616699516773224, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.15739, "step": 322, "tokens/total": 11100048, "tokens/train_per_sec_per_gpu": 41.19, "tokens/trainable": 216796} +{"epoch": 1.26171875, "grad_norm": 0.8457385897636414, "learning_rate": 3.977591418134619e-05, "loss": 0.15220242738723755, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.45, "memory/max_allocated (GiB)": 38.45, "ppl": 1.1644, "step": 323, "tokens/total": 11134704, "tokens/train_per_sec_per_gpu": 39.12, "tokens/trainable": 217462} +{"epoch": 1.265625, "grad_norm": 1.5624048709869385, "learning_rate": 3.95030593412612e-05, "loss": 0.16651543974876404, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.18118, "step": 324, "tokens/total": 11170320, "tokens/train_per_sec_per_gpu": 40.78, "tokens/trainable": 218138} +{"epoch": 1.26953125, "grad_norm": 1.3257437944412231, "learning_rate": 3.923084939213296e-05, "loss": 0.14539691805839539, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 35.58, "memory/max_allocated (GiB)": 35.58, "ppl": 1.1565, "step": 325, "tokens/total": 11200416, "tokens/train_per_sec_per_gpu": 42.0, "tokens/trainable": 218775} +{"epoch": 1.2734375, "grad_norm": 1.7719522714614868, "learning_rate": 3.895929566172861e-05, "loss": 0.11561943590641022, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.12257, "step": 326, "tokens/total": 11236080, "tokens/train_per_sec_per_gpu": 38.37, "tokens/trainable": 219485} +{"epoch": 1.27734375, "grad_norm": 1.4870468378067017, "learning_rate": 3.868840945050728e-05, "loss": 0.1269221007823944, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.05, "memory/max_allocated (GiB)": 39.05, "ppl": 1.13533, "step": 327, "tokens/total": 11274608, "tokens/train_per_sec_per_gpu": 36.68, "tokens/trainable": 220186} +{"epoch": 1.28125, "grad_norm": 0.9613686203956604, "learning_rate": 3.841820203114995e-05, "loss": 0.13649976253509521, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.14625, "step": 328, "tokens/total": 11305968, "tokens/train_per_sec_per_gpu": 42.39, "tokens/trainable": 220864} +{"epoch": 1.28515625, "grad_norm": 0.7136479616165161, "learning_rate": 3.814868464809027e-05, "loss": 0.14758674800395966, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.61, "memory/max_allocated (GiB)": 36.61, "ppl": 1.15903, "step": 329, "tokens/total": 11337536, "tokens/train_per_sec_per_gpu": 38.36, "tokens/trainable": 221466} +{"epoch": 1.2890625, "grad_norm": 1.7155191898345947, "learning_rate": 3.787986851704667e-05, "loss": 0.15813378989696503, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.17132, "step": 330, "tokens/total": 11370688, "tokens/train_per_sec_per_gpu": 41.78, "tokens/trainable": 222124} +{"epoch": 1.29296875, "grad_norm": 1.92703115940094, "learning_rate": 3.7611764824555654e-05, "loss": 0.13325844705104828, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.57, "memory/max_allocated (GiB)": 37.57, "ppl": 1.14255, "step": 331, "tokens/total": 11401888, "tokens/train_per_sec_per_gpu": 48.35, "tokens/trainable": 222831} +{"epoch": 1.296875, "grad_norm": 0.7904762029647827, "learning_rate": 3.734438472750619e-05, "loss": 0.1252446472644806, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.35, "memory/max_allocated (GiB)": 39.35, "ppl": 1.13343, "step": 332, "tokens/total": 11441536, "tokens/train_per_sec_per_gpu": 36.01, "tokens/trainable": 223542} +{"epoch": 1.30078125, "grad_norm": 2.9522273540496826, "learning_rate": 3.707773935267552e-05, "loss": 0.13855794072151184, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.14862, "step": 333, "tokens/total": 11478224, "tokens/train_per_sec_per_gpu": 36.92, "tokens/trainable": 224260} +{"epoch": 1.3046875, "grad_norm": 1.4689395427703857, "learning_rate": 3.68118397962661e-05, "loss": 0.17056772112846375, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.39, "memory/max_allocated (GiB)": 38.39, "ppl": 1.18598, "step": 334, "tokens/total": 11512672, "tokens/train_per_sec_per_gpu": 36.47, "tokens/trainable": 224875} +{"epoch": 1.30859375, "grad_norm": 1.4682440757751465, "learning_rate": 3.654669712344384e-05, "loss": 0.12521643936634064, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.58, "memory/max_allocated (GiB)": 36.58, "ppl": 1.13339, "step": 335, "tokens/total": 11543488, "tokens/train_per_sec_per_gpu": 54.15, "tokens/trainable": 225587} +{"epoch": 1.3125, "grad_norm": 0.8221662640571594, "learning_rate": 3.628232236787763e-05, "loss": 0.1277940273284912, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.13632, "step": 336, "tokens/total": 11578704, "tokens/train_per_sec_per_gpu": 33.52, "tokens/trainable": 226286} +{"epoch": 1.31640625, "grad_norm": 0.5772737860679626, "learning_rate": 3.6018726531280144e-05, "loss": 0.11023522913455963, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.11654, "step": 337, "tokens/total": 11615872, "tokens/train_per_sec_per_gpu": 43.48, "tokens/trainable": 227016} +{"epoch": 1.3203125, "grad_norm": 1.3775315284729004, "learning_rate": 3.575592058295017e-05, "loss": 0.17867368459701538, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.92, "memory/max_allocated (GiB)": 36.92, "ppl": 1.19563, "step": 338, "tokens/total": 11649600, "tokens/train_per_sec_per_gpu": 41.1, "tokens/trainable": 227697} +{"epoch": 1.32421875, "grad_norm": 0.6933366656303406, "learning_rate": 3.549391545931585e-05, "loss": 0.13293595612049103, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.14218, "step": 339, "tokens/total": 11683664, "tokens/train_per_sec_per_gpu": 43.04, "tokens/trainable": 228395} +{"epoch": 1.328125, "grad_norm": 2.3299765586853027, "learning_rate": 3.5232722063479914e-05, "loss": 0.1612948775291443, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.17, "memory/max_allocated (GiB)": 37.17, "ppl": 1.17503, "step": 340, "tokens/total": 11717600, "tokens/train_per_sec_per_gpu": 43.62, "tokens/trainable": 229098} +{"epoch": 1.33203125, "grad_norm": 1.102827548980713, "learning_rate": 3.49723512647657e-05, "loss": 0.13020047545433044, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.08, "memory/max_allocated (GiB)": 39.08, "ppl": 1.13906, "step": 341, "tokens/total": 11756560, "tokens/train_per_sec_per_gpu": 38.06, "tokens/trainable": 229873} +{"epoch": 1.3359375, "grad_norm": 6.440471649169922, "learning_rate": 3.471281389826491e-05, "loss": 0.15217244625091553, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.27, "memory/max_allocated (GiB)": 36.27, "ppl": 1.16436, "step": 342, "tokens/total": 11787136, "tokens/train_per_sec_per_gpu": 41.59, "tokens/trainable": 230475} +{"epoch": 1.33984375, "grad_norm": 1.990567922592163, "learning_rate": 3.4454120764386764e-05, "loss": 0.12632304430007935, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.13465, "step": 343, "tokens/total": 11825280, "tokens/train_per_sec_per_gpu": 36.83, "tokens/trainable": 231161} +{"epoch": 1.34375, "grad_norm": 0.9376530647277832, "learning_rate": 3.4196282628408526e-05, "loss": 0.15770405530929565, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.45, "memory/max_allocated (GiB)": 37.45, "ppl": 1.17082, "step": 344, "tokens/total": 11859264, "tokens/train_per_sec_per_gpu": 36.92, "tokens/trainable": 231802} +{"epoch": 1.34765625, "grad_norm": 1.1134892702102661, "learning_rate": 3.3939310220027456e-05, "loss": 0.13828951120376587, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.14831, "step": 345, "tokens/total": 11893056, "tokens/train_per_sec_per_gpu": 40.81, "tokens/trainable": 232532} +{"epoch": 1.3515625, "grad_norm": 0.637305736541748, "learning_rate": 3.3683214232914404e-05, "loss": 0.12118840962648392, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.53, "memory/max_allocated (GiB)": 38.53, "ppl": 1.12884, "step": 346, "tokens/total": 11928352, "tokens/train_per_sec_per_gpu": 41.7, "tokens/trainable": 233274} +{"epoch": 1.35546875, "grad_norm": 2.1388697624206543, "learning_rate": 3.342800532426873e-05, "loss": 0.14980587363243103, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.16161, "step": 347, "tokens/total": 11964016, "tokens/train_per_sec_per_gpu": 38.51, "tokens/trainable": 233937} +{"epoch": 1.359375, "grad_norm": 0.9323917627334595, "learning_rate": 3.317369411437484e-05, "loss": 0.13833265006542206, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.63, "memory/max_allocated (GiB)": 37.63, "ppl": 1.14836, "step": 348, "tokens/total": 11999632, "tokens/train_per_sec_per_gpu": 37.61, "tokens/trainable": 234608} +{"epoch": 1.36328125, "grad_norm": 0.9540636539459229, "learning_rate": 3.292029118616024e-05, "loss": 0.1492616832256317, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.87, "memory/max_allocated (GiB)": 36.87, "ppl": 1.16098, "step": 349, "tokens/total": 12032032, "tokens/train_per_sec_per_gpu": 33.44, "tokens/trainable": 235188} +{"epoch": 1.3671875, "grad_norm": 0.7832889556884766, "learning_rate": 3.266780708475511e-05, "loss": 0.13158239424228668, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.14063, "step": 350, "tokens/total": 12067152, "tokens/train_per_sec_per_gpu": 37.26, "tokens/trainable": 235879} +{"epoch": 1.37109375, "grad_norm": 0.43667954206466675, "learning_rate": 3.241625231705354e-05, "loss": 0.10545526444911957, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 1.11122, "step": 351, "tokens/total": 12102880, "tokens/train_per_sec_per_gpu": 41.05, "tokens/trainable": 236587} +{"epoch": 1.375, "grad_norm": 0.7702914476394653, "learning_rate": 3.216563735127618e-05, "loss": 0.11676438897848129, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 1.12385, "step": 352, "tokens/total": 12137600, "tokens/train_per_sec_per_gpu": 39.78, "tokens/trainable": 237326} +{"epoch": 1.37890625, "grad_norm": 2.5029029846191406, "learning_rate": 3.191597261653475e-05, "loss": 0.12974143028259277, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.13853, "step": 353, "tokens/total": 12173952, "tokens/train_per_sec_per_gpu": 39.61, "tokens/trainable": 238054} +{"epoch": 1.3828125, "grad_norm": 2.805298089981079, "learning_rate": 3.166726850239794e-05, "loss": 0.139992818236351, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.15027, "step": 354, "tokens/total": 12207232, "tokens/train_per_sec_per_gpu": 43.25, "tokens/trainable": 238741} +{"epoch": 1.38671875, "grad_norm": 1.5516135692596436, "learning_rate": 3.141953535845912e-05, "loss": 0.17344006896018982, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.06, "memory/max_allocated (GiB)": 36.06, "ppl": 1.18939, "step": 355, "tokens/total": 12237776, "tokens/train_per_sec_per_gpu": 39.35, "tokens/trainable": 239330} +{"epoch": 1.390625, "grad_norm": 10.782288551330566, "learning_rate": 3.11727834939056e-05, "loss": 0.14088931679725647, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.14, "memory/max_allocated (GiB)": 36.14, "ppl": 1.1513, "step": 356, "tokens/total": 12269264, "tokens/train_per_sec_per_gpu": 36.33, "tokens/trainable": 239996} +{"epoch": 1.39453125, "grad_norm": 2.4849090576171875, "learning_rate": 3.092702317708967e-05, "loss": 0.15425334870815277, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.16679, "step": 357, "tokens/total": 12302688, "tokens/train_per_sec_per_gpu": 41.07, "tokens/trainable": 240652} +{"epoch": 1.3984375, "grad_norm": 1.1336272954940796, "learning_rate": 3.0682264635101276e-05, "loss": 0.13670755922794342, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.14649, "step": 358, "tokens/total": 12334848, "tokens/train_per_sec_per_gpu": 40.93, "tokens/trainable": 241273} +{"epoch": 1.40234375, "grad_norm": 0.5329340100288391, "learning_rate": 3.0438518053342407e-05, "loss": 0.11739175021648407, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.12456, "step": 359, "tokens/total": 12373664, "tokens/train_per_sec_per_gpu": 36.09, "tokens/trainable": 242019} +{"epoch": 1.40625, "grad_norm": 1.0525325536727905, "learning_rate": 3.0195793575103266e-05, "loss": 0.13401132822036743, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.14341, "step": 360, "tokens/total": 12412288, "tokens/train_per_sec_per_gpu": 34.49, "tokens/trainable": 242710} +{"epoch": 1.41015625, "grad_norm": 0.8800958395004272, "learning_rate": 2.9954101301140146e-05, "loss": 0.14042451977729797, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.15076, "step": 361, "tokens/total": 12447280, "tokens/train_per_sec_per_gpu": 38.49, "tokens/trainable": 243382} +{"epoch": 1.4140625, "grad_norm": 0.9616166949272156, "learning_rate": 2.9713451289255123e-05, "loss": 0.14019812643527985, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.79, "memory/max_allocated (GiB)": 36.79, "ppl": 1.1505, "step": 362, "tokens/total": 12476352, "tokens/train_per_sec_per_gpu": 36.4, "tokens/trainable": 244037} +{"epoch": 1.41796875, "grad_norm": 0.5264243483543396, "learning_rate": 2.9473853553877484e-05, "loss": 0.11768580973148346, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.22, "memory/max_allocated (GiB)": 38.22, "ppl": 1.12489, "step": 363, "tokens/total": 12509872, "tokens/train_per_sec_per_gpu": 38.88, "tokens/trainable": 244697} +{"epoch": 1.421875, "grad_norm": 1.2705752849578857, "learning_rate": 2.9235318065647e-05, "loss": 0.14876747131347656, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.52, "memory/max_allocated (GiB)": 39.52, "ppl": 1.1604, "step": 364, "tokens/total": 12549568, "tokens/train_per_sec_per_gpu": 34.51, "tokens/trainable": 245348} +{"epoch": 1.42578125, "grad_norm": 1.2294589281082153, "learning_rate": 2.8997854750998964e-05, "loss": 0.12258382886648178, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.35, "memory/max_allocated (GiB)": 39.35, "ppl": 1.13041, "step": 365, "tokens/total": 12588832, "tokens/train_per_sec_per_gpu": 39.3, "tokens/trainable": 246066} +{"epoch": 1.4296875, "grad_norm": 0.6128765940666199, "learning_rate": 2.8761473491751258e-05, "loss": 0.12997084856033325, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.44, "memory/max_allocated (GiB)": 36.44, "ppl": 1.1388, "step": 366, "tokens/total": 12621264, "tokens/train_per_sec_per_gpu": 39.37, "tokens/trainable": 246710} +{"epoch": 1.43359375, "grad_norm": 0.6333466172218323, "learning_rate": 2.8526184124692883e-05, "loss": 0.11501625180244446, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.12189, "step": 367, "tokens/total": 12657536, "tokens/train_per_sec_per_gpu": 31.58, "tokens/trainable": 247427} +{"epoch": 1.4375, "grad_norm": 1.440256118774414, "learning_rate": 2.829199644117484e-05, "loss": 0.14599737524986267, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 1.15719, "step": 368, "tokens/total": 12690800, "tokens/train_per_sec_per_gpu": 40.68, "tokens/trainable": 248032} +{"epoch": 1.44140625, "grad_norm": 1.1176879405975342, "learning_rate": 2.8058920186702553e-05, "loss": 0.12036970257759094, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.12791, "step": 369, "tokens/total": 12728480, "tokens/train_per_sec_per_gpu": 41.24, "tokens/trainable": 248769} +{"epoch": 1.4453125, "grad_norm": 1.1414694786071777, "learning_rate": 2.782696506053033e-05, "loss": 0.15028409659862518, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.16216, "step": 370, "tokens/total": 12766368, "tokens/train_per_sec_per_gpu": 33.8, "tokens/trainable": 249403} +{"epoch": 1.44921875, "grad_norm": 1.7164862155914307, "learning_rate": 2.7596140715257824e-05, "loss": 0.1387178897857666, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.66, "memory/max_allocated (GiB)": 38.66, "ppl": 1.1488, "step": 371, "tokens/total": 12801056, "tokens/train_per_sec_per_gpu": 38.28, "tokens/trainable": 250075} +{"epoch": 1.453125, "grad_norm": 0.6020545959472656, "learning_rate": 2.7366456756428184e-05, "loss": 0.12988972663879395, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.1387, "step": 372, "tokens/total": 12835056, "tokens/train_per_sec_per_gpu": 38.33, "tokens/trainable": 250731} +{"epoch": 1.45703125, "grad_norm": 1.7810442447662354, "learning_rate": 2.7137922742128486e-05, "loss": 0.14570081233978271, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.06, "memory/max_allocated (GiB)": 38.06, "ppl": 1.15685, "step": 373, "tokens/total": 12869616, "tokens/train_per_sec_per_gpu": 42.8, "tokens/trainable": 251462} +{"epoch": 1.4609375, "grad_norm": 1.2501463890075684, "learning_rate": 2.691054818259188e-05, "loss": 0.12198536843061447, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.31, "memory/max_allocated (GiB)": 37.31, "ppl": 1.12974, "step": 374, "tokens/total": 12903936, "tokens/train_per_sec_per_gpu": 41.54, "tokens/trainable": 252209} +{"epoch": 1.46484375, "grad_norm": 0.9832674264907837, "learning_rate": 2.6684342539801933e-05, "loss": 0.1444033980369568, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.37, "memory/max_allocated (GiB)": 36.37, "ppl": 1.15535, "step": 375, "tokens/total": 12934960, "tokens/train_per_sec_per_gpu": 39.19, "tokens/trainable": 252837} +{"epoch": 1.46875, "grad_norm": 0.7618889212608337, "learning_rate": 2.645931522709877e-05, "loss": 0.12083721160888672, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.12844, "step": 376, "tokens/total": 12970368, "tokens/train_per_sec_per_gpu": 49.21, "tokens/trainable": 253609} +{"epoch": 1.47265625, "grad_norm": 0.7302822470664978, "learning_rate": 2.6235475608787365e-05, "loss": 0.13869258761405945, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.14877, "step": 377, "tokens/total": 13005776, "tokens/train_per_sec_per_gpu": 38.18, "tokens/trainable": 254268} +{"epoch": 1.4765625, "grad_norm": 0.8854812383651733, "learning_rate": 2.6012832999747916e-05, "loss": 0.14798381924629211, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.31, "memory/max_allocated (GiB)": 36.31, "ppl": 1.15949, "step": 378, "tokens/total": 13036304, "tokens/train_per_sec_per_gpu": 39.82, "tokens/trainable": 254894} +{"epoch": 1.48046875, "grad_norm": 0.7731621861457825, "learning_rate": 2.579139666504821e-05, "loss": 0.14906471967697144, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.16075, "step": 379, "tokens/total": 13075984, "tokens/train_per_sec_per_gpu": 34.69, "tokens/trainable": 255557} +{"epoch": 1.484375, "grad_norm": 0.8205195069313049, "learning_rate": 2.557117581955798e-05, "loss": 0.13058356940746307, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.13949, "step": 380, "tokens/total": 13109424, "tokens/train_per_sec_per_gpu": 38.9, "tokens/trainable": 256249} +{"epoch": 1.48828125, "grad_norm": 1.2881553173065186, "learning_rate": 2.5352179627565532e-05, "loss": 0.1389853060245514, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.97, "memory/max_allocated (GiB)": 37.97, "ppl": 1.14911, "step": 381, "tokens/total": 13143280, "tokens/train_per_sec_per_gpu": 32.09, "tokens/trainable": 256884} +{"epoch": 1.4921875, "grad_norm": 1.6959365606307983, "learning_rate": 2.5134417202396277e-05, "loss": 0.15800374746322632, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.45, "memory/max_allocated (GiB)": 38.45, "ppl": 1.17117, "step": 382, "tokens/total": 13176624, "tokens/train_per_sec_per_gpu": 35.35, "tokens/trainable": 257493} +{"epoch": 1.49609375, "grad_norm": 0.8287739157676697, "learning_rate": 2.491789760603361e-05, "loss": 0.1410551369190216, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.04, "memory/max_allocated (GiB)": 39.04, "ppl": 1.15149, "step": 383, "tokens/total": 13212720, "tokens/train_per_sec_per_gpu": 35.28, "tokens/trainable": 258139} +{"epoch": 1.5, "grad_norm": 2.044909954071045, "learning_rate": 2.4702629848741764e-05, "loss": 0.16898272931575775, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.1841, "step": 384, "tokens/total": 13246944, "tokens/train_per_sec_per_gpu": 38.87, "tokens/trainable": 258795} +{"epoch": 1.50390625, "grad_norm": 1.9950345754623413, "learning_rate": 2.4488622888690785e-05, "loss": 0.13497808575630188, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.14451, "step": 385, "tokens/total": 13280224, "tokens/train_per_sec_per_gpu": 38.2, "tokens/trainable": 259424} +{"epoch": 1.5078125, "grad_norm": 0.7217848896980286, "learning_rate": 2.427588563158384e-05, "loss": 0.12112349271774292, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.12876, "step": 386, "tokens/total": 13314192, "tokens/train_per_sec_per_gpu": 41.49, "tokens/trainable": 260158} +{"epoch": 1.51171875, "grad_norm": 0.7894755601882935, "learning_rate": 2.406442693028651e-05, "loss": 0.1314806193113327, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.34, "memory/max_allocated (GiB)": 38.34, "ppl": 1.14052, "step": 387, "tokens/total": 13345472, "tokens/train_per_sec_per_gpu": 39.76, "tokens/trainable": 260782} +{"epoch": 1.515625, "grad_norm": 0.9363507032394409, "learning_rate": 2.3854255584458547e-05, "loss": 0.14381781220436096, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.15467, "step": 388, "tokens/total": 13384000, "tokens/train_per_sec_per_gpu": 33.66, "tokens/trainable": 261442} +{"epoch": 1.51953125, "grad_norm": 1.3186099529266357, "learning_rate": 2.3645380340187508e-05, "loss": 0.12177661806344986, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.1295, "step": 389, "tokens/total": 13419232, "tokens/train_per_sec_per_gpu": 41.12, "tokens/trainable": 262145} +{"epoch": 1.5234375, "grad_norm": 1.1996121406555176, "learning_rate": 2.3437809889624914e-05, "loss": 0.17477348446846008, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.42, "memory/max_allocated (GiB)": 36.42, "ppl": 1.19098, "step": 390, "tokens/total": 13450016, "tokens/train_per_sec_per_gpu": 42.79, "tokens/trainable": 262774} +{"epoch": 1.52734375, "grad_norm": 1.2085691690444946, "learning_rate": 2.3231552870624487e-05, "loss": 0.12932340800762177, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.32, "memory/max_allocated (GiB)": 38.32, "ppl": 1.13806, "step": 391, "tokens/total": 13480592, "tokens/train_per_sec_per_gpu": 47.44, "tokens/trainable": 263480} +{"epoch": 1.53125, "grad_norm": 0.8898318409919739, "learning_rate": 2.3026617866382657e-05, "loss": 0.11687418818473816, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.54, "memory/max_allocated (GiB)": 36.54, "ppl": 1.12398, "step": 392, "tokens/total": 13513248, "tokens/train_per_sec_per_gpu": 40.65, "tokens/trainable": 264211} +{"epoch": 1.53515625, "grad_norm": 0.6090821027755737, "learning_rate": 2.2823013405081507e-05, "loss": 0.12724389135837555, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.14, "memory/max_allocated (GiB)": 39.14, "ppl": 1.13569, "step": 393, "tokens/total": 13551168, "tokens/train_per_sec_per_gpu": 37.21, "tokens/trainable": 264931} +{"epoch": 1.5390625, "grad_norm": 1.8127866983413696, "learning_rate": 2.2620747959533722e-05, "loss": 0.12665767967700958, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.13503, "step": 394, "tokens/total": 13584160, "tokens/train_per_sec_per_gpu": 44.58, "tokens/trainable": 265674} +{"epoch": 1.54296875, "grad_norm": 0.7994990348815918, "learning_rate": 2.2419829946830123e-05, "loss": 0.12566706538200378, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.1339, "step": 395, "tokens/total": 13619616, "tokens/train_per_sec_per_gpu": 39.53, "tokens/trainable": 266322} +{"epoch": 1.546875, "grad_norm": 0.5980313420295715, "learning_rate": 2.2220267727989325e-05, "loss": 0.11838946491479874, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.12568, "step": 396, "tokens/total": 13653776, "tokens/train_per_sec_per_gpu": 39.64, "tokens/trainable": 267014} +{"epoch": 1.55078125, "grad_norm": 0.7071093320846558, "learning_rate": 2.202206960760984e-05, "loss": 0.11032328754663467, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.11664, "step": 397, "tokens/total": 13691680, "tokens/train_per_sec_per_gpu": 39.31, "tokens/trainable": 267793} +{"epoch": 1.5546875, "grad_norm": 0.6529163122177124, "learning_rate": 2.182524383352446e-05, "loss": 0.13701169192790985, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.83, "memory/max_allocated (GiB)": 37.83, "ppl": 1.14684, "step": 398, "tokens/total": 13727520, "tokens/train_per_sec_per_gpu": 33.99, "tokens/trainable": 268432} +{"epoch": 1.55859375, "grad_norm": 0.723823070526123, "learning_rate": 2.1629798596457056e-05, "loss": 0.13921624422073364, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.25, "memory/max_allocated (GiB)": 36.25, "ppl": 1.14937, "step": 399, "tokens/total": 13759360, "tokens/train_per_sec_per_gpu": 40.79, "tokens/trainable": 269080} +{"epoch": 1.5625, "grad_norm": 1.0816596746444702, "learning_rate": 2.1435742029681725e-05, "loss": 0.14008983969688416, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.18, "memory/max_allocated (GiB)": 36.18, "ppl": 1.15038, "step": 400, "tokens/total": 13789584, "tokens/train_per_sec_per_gpu": 43.88, "tokens/trainable": 269680} +{"epoch": 1.56640625, "grad_norm": 1.0189933776855469, "learning_rate": 2.124308220868431e-05, "loss": 0.11896611750125885, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.12633, "step": 401, "tokens/total": 13825056, "tokens/train_per_sec_per_gpu": 37.57, "tokens/trainable": 270365} +{"epoch": 1.5703125, "grad_norm": 0.665209174156189, "learning_rate": 2.105182715082638e-05, "loss": 0.12598614394664764, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.76, "memory/max_allocated (GiB)": 36.76, "ppl": 1.13427, "step": 402, "tokens/total": 13858464, "tokens/train_per_sec_per_gpu": 41.63, "tokens/trainable": 271084} +{"epoch": 1.57421875, "grad_norm": 5.427802562713623, "learning_rate": 2.0861984815011552e-05, "loss": 0.1457727551460266, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.15693, "step": 403, "tokens/total": 13893312, "tokens/train_per_sec_per_gpu": 35.36, "tokens/trainable": 271689} +{"epoch": 1.578125, "grad_norm": 0.5967668890953064, "learning_rate": 2.0673563101354323e-05, "loss": 0.1396748423576355, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.1499, "step": 404, "tokens/total": 13928304, "tokens/train_per_sec_per_gpu": 31.68, "tokens/trainable": 272310} +{"epoch": 1.58203125, "grad_norm": 0.9149715900421143, "learning_rate": 2.0486569850851317e-05, "loss": 0.12711641192436218, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.17, "memory/max_allocated (GiB)": 38.17, "ppl": 1.13555, "step": 405, "tokens/total": 13962832, "tokens/train_per_sec_per_gpu": 39.06, "tokens/trainable": 272970} +{"epoch": 1.5859375, "grad_norm": 0.641228973865509, "learning_rate": 2.0301012845054956e-05, "loss": 0.12629345059394836, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.13462, "step": 406, "tokens/total": 14001376, "tokens/train_per_sec_per_gpu": 33.16, "tokens/trainable": 273606} +{"epoch": 1.58984375, "grad_norm": 0.7356780767440796, "learning_rate": 2.011689980574966e-05, "loss": 0.12812629342079163, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.1367, "step": 407, "tokens/total": 14037344, "tokens/train_per_sec_per_gpu": 35.63, "tokens/trainable": 274262} +{"epoch": 1.59375, "grad_norm": 0.6440728306770325, "learning_rate": 1.993423839463052e-05, "loss": 0.11792916804552078, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.17, "memory/max_allocated (GiB)": 38.17, "ppl": 1.12516, "step": 408, "tokens/total": 14070400, "tokens/train_per_sec_per_gpu": 51.45, "tokens/trainable": 274956} +{"epoch": 1.59765625, "grad_norm": 0.5738937854766846, "learning_rate": 1.975303621298445e-05, "loss": 0.11395759880542755, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.05, "memory/max_allocated (GiB)": 39.05, "ppl": 1.1207, "step": 409, "tokens/total": 14106816, "tokens/train_per_sec_per_gpu": 42.31, "tokens/trainable": 275656} +{"epoch": 1.6015625, "grad_norm": 1.0309041738510132, "learning_rate": 1.957330080137385e-05, "loss": 0.13873563706874847, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.09, "memory/max_allocated (GiB)": 36.09, "ppl": 1.14882, "step": 410, "tokens/total": 14137040, "tokens/train_per_sec_per_gpu": 40.91, "tokens/trainable": 276307} +{"epoch": 1.60546875, "grad_norm": 0.6442936062812805, "learning_rate": 1.9395039639322864e-05, "loss": 0.12333139032125473, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.13126, "step": 411, "tokens/total": 14172752, "tokens/train_per_sec_per_gpu": 38.87, "tokens/trainable": 276979} +{"epoch": 1.609375, "grad_norm": 0.6670875549316406, "learning_rate": 1.9218260145006073e-05, "loss": 0.11422993242740631, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.61, "memory/max_allocated (GiB)": 37.61, "ppl": 1.12101, "step": 412, "tokens/total": 14208400, "tokens/train_per_sec_per_gpu": 43.66, "tokens/trainable": 277735} +{"epoch": 1.61328125, "grad_norm": 1.0097124576568604, "learning_rate": 1.904296967493982e-05, "loss": 0.1260237991809845, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.13431, "step": 413, "tokens/total": 14240560, "tokens/train_per_sec_per_gpu": 46.61, "tokens/trainable": 278398} +{"epoch": 1.6171875, "grad_norm": 1.2081726789474487, "learning_rate": 1.8869175523676064e-05, "loss": 0.11089695990085602, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.2, "memory/max_allocated (GiB)": 38.2, "ppl": 1.11728, "step": 414, "tokens/total": 14276928, "tokens/train_per_sec_per_gpu": 39.88, "tokens/trainable": 279129} +{"epoch": 1.62109375, "grad_norm": 0.4957243502140045, "learning_rate": 1.869688492349885e-05, "loss": 0.12429557740688324, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.13235, "step": 415, "tokens/total": 14309808, "tokens/train_per_sec_per_gpu": 39.97, "tokens/trainable": 279792} +{"epoch": 1.625, "grad_norm": 1.0650542974472046, "learning_rate": 1.85261050441233e-05, "loss": 0.1341363936662674, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.14355, "step": 416, "tokens/total": 14345504, "tokens/train_per_sec_per_gpu": 38.23, "tokens/trainable": 280491} +{"epoch": 1.62890625, "grad_norm": 1.1442877054214478, "learning_rate": 1.8356842992397304e-05, "loss": 0.1603812426328659, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.06, "memory/max_allocated (GiB)": 37.06, "ppl": 1.17396, "step": 417, "tokens/total": 14378928, "tokens/train_per_sec_per_gpu": 39.08, "tokens/trainable": 281127} +{"epoch": 1.6328125, "grad_norm": 0.7115590572357178, "learning_rate": 1.8189105812005714e-05, "loss": 0.1410491168498993, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.15148, "step": 418, "tokens/total": 14415232, "tokens/train_per_sec_per_gpu": 33.15, "tokens/trainable": 281700} +{"epoch": 1.63671875, "grad_norm": 0.915564239025116, "learning_rate": 1.802290048317732e-05, "loss": 0.120529405772686, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.12809, "step": 419, "tokens/total": 14448480, "tokens/train_per_sec_per_gpu": 38.18, "tokens/trainable": 282342} +{"epoch": 1.640625, "grad_norm": 1.4339838027954102, "learning_rate": 1.785823392239424e-05, "loss": 0.12703879177570343, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.67, "memory/max_allocated (GiB)": 37.67, "ppl": 1.13546, "step": 420, "tokens/total": 14481328, "tokens/train_per_sec_per_gpu": 37.19, "tokens/trainable": 282946} +{"epoch": 1.64453125, "grad_norm": 0.998999297618866, "learning_rate": 1.7695112982104225e-05, "loss": 0.11951425671577454, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.61, "memory/max_allocated (GiB)": 39.61, "ppl": 1.12695, "step": 421, "tokens/total": 14516208, "tokens/train_per_sec_per_gpu": 40.34, "tokens/trainable": 283635} +{"epoch": 1.6484375, "grad_norm": 1.107240915298462, "learning_rate": 1.7533544450435433e-05, "loss": 0.13179636001586914, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 35.55, "memory/max_allocated (GiB)": 35.55, "ppl": 1.14088, "step": 422, "tokens/total": 14545248, "tokens/train_per_sec_per_gpu": 51.08, "tokens/trainable": 284300} +{"epoch": 1.65234375, "grad_norm": 0.9198355078697205, "learning_rate": 1.7373535050913946e-05, "loss": 0.13028566539287567, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.7, "memory/max_allocated (GiB)": 38.7, "ppl": 1.13915, "step": 423, "tokens/total": 14581296, "tokens/train_per_sec_per_gpu": 32.3, "tokens/trainable": 284971} +{"epoch": 1.65625, "grad_norm": 0.6321521401405334, "learning_rate": 1.721509144218405e-05, "loss": 0.13233187794685364, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 35.57, "memory/max_allocated (GiB)": 35.57, "ppl": 1.14149, "step": 424, "tokens/total": 14610944, "tokens/train_per_sec_per_gpu": 48.21, "tokens/trainable": 285610} +{"epoch": 1.66015625, "grad_norm": 0.5386309623718262, "learning_rate": 1.705822021773101e-05, "loss": 0.1261201798915863, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.13442, "step": 425, "tokens/total": 14645024, "tokens/train_per_sec_per_gpu": 40.33, "tokens/trainable": 286271} +{"epoch": 1.6640625, "grad_norm": 1.117499589920044, "learning_rate": 1.69029279056068e-05, "loss": 0.14020156860351562, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.15051, "step": 426, "tokens/total": 14682112, "tokens/train_per_sec_per_gpu": 37.19, "tokens/trainable": 286944} +{"epoch": 1.66796875, "grad_norm": 1.260890245437622, "learning_rate": 1.6749220968158415e-05, "loss": 0.11393845081329346, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.19, "memory/max_allocated (GiB)": 39.19, "ppl": 1.12068, "step": 427, "tokens/total": 14716784, "tokens/train_per_sec_per_gpu": 41.81, "tokens/trainable": 287678} +{"epoch": 1.671875, "grad_norm": 1.093238353729248, "learning_rate": 1.659710580175893e-05, "loss": 0.12993192672729492, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 35.75, "memory/max_allocated (GiB)": 35.75, "ppl": 1.13875, "step": 428, "tokens/total": 14747264, "tokens/train_per_sec_per_gpu": 44.61, "tokens/trainable": 288362} +{"epoch": 1.67578125, "grad_norm": 0.7840176820755005, "learning_rate": 1.644658873654133e-05, "loss": 0.11794920265674591, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.12519, "step": 429, "tokens/total": 14779408, "tokens/train_per_sec_per_gpu": 44.5, "tokens/trainable": 289041} +{"epoch": 1.6796875, "grad_norm": 1.2434766292572021, "learning_rate": 1.629767603613508e-05, "loss": 0.15390156209468842, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 35.97, "memory/max_allocated (GiB)": 35.97, "ppl": 1.16638, "step": 430, "tokens/total": 14809936, "tokens/train_per_sec_per_gpu": 38.48, "tokens/trainable": 289621} +{"epoch": 1.68359375, "grad_norm": 0.7238301634788513, "learning_rate": 1.615037389740547e-05, "loss": 0.12282504141330719, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.53, "memory/max_allocated (GiB)": 37.53, "ppl": 1.13069, "step": 431, "tokens/total": 14844640, "tokens/train_per_sec_per_gpu": 36.95, "tokens/trainable": 290315} +{"epoch": 1.6875, "grad_norm": 1.2872830629348755, "learning_rate": 1.600468845019576e-05, "loss": 0.1175348311662674, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.12472, "step": 432, "tokens/total": 14882512, "tokens/train_per_sec_per_gpu": 35.35, "tokens/trainable": 291000} +{"epoch": 1.69140625, "grad_norm": 0.7653066515922546, "learning_rate": 1.5860625757072092e-05, "loss": 0.1388917863368988, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.33, "memory/max_allocated (GiB)": 38.33, "ppl": 1.149, "step": 433, "tokens/total": 14917104, "tokens/train_per_sec_per_gpu": 33.13, "tokens/trainable": 291622} +{"epoch": 1.6953125, "grad_norm": 0.7500340342521667, "learning_rate": 1.571819181307116e-05, "loss": 0.10898016393184662, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 1.11514, "step": 434, "tokens/total": 14953776, "tokens/train_per_sec_per_gpu": 44.34, "tokens/trainable": 292386} +{"epoch": 1.69921875, "grad_norm": 0.5915701985359192, "learning_rate": 1.557739254545075e-05, "loss": 0.12255579233169556, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.33, "memory/max_allocated (GiB)": 36.33, "ppl": 1.13038, "step": 435, "tokens/total": 14984544, "tokens/train_per_sec_per_gpu": 47.3, "tokens/trainable": 293055} +{"epoch": 1.703125, "grad_norm": 0.6257673501968384, "learning_rate": 1.543823381344311e-05, "loss": 0.12533214688301086, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.13352, "step": 436, "tokens/total": 15016848, "tokens/train_per_sec_per_gpu": 48.86, "tokens/trainable": 293774} +{"epoch": 1.70703125, "grad_norm": 0.8158477544784546, "learning_rate": 1.5300721408011114e-05, "loss": 0.12976935505867004, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.13857, "step": 437, "tokens/total": 15051936, "tokens/train_per_sec_per_gpu": 38.83, "tokens/trainable": 294425} +{"epoch": 1.7109375, "grad_norm": 0.8470305800437927, "learning_rate": 1.5164861051607254e-05, "loss": 0.14924849569797516, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.9, "memory/max_allocated (GiB)": 36.9, "ppl": 1.16096, "step": 438, "tokens/total": 15084784, "tokens/train_per_sec_per_gpu": 31.18, "tokens/trainable": 294976} +{"epoch": 1.71484375, "grad_norm": 2.9415438175201416, "learning_rate": 1.5030658397935521e-05, "loss": 0.1288546919822693, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.15, "memory/max_allocated (GiB)": 38.15, "ppl": 1.13752, "step": 439, "tokens/total": 15120000, "tokens/train_per_sec_per_gpu": 38.97, "tokens/trainable": 295672} +{"epoch": 1.71875, "grad_norm": 0.6700268387794495, "learning_rate": 1.4898119031716104e-05, "loss": 0.118571937084198, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.12589, "step": 440, "tokens/total": 15152656, "tokens/train_per_sec_per_gpu": 43.95, "tokens/trainable": 296339} +{"epoch": 1.72265625, "grad_norm": 0.6911159753799438, "learning_rate": 1.476724846845306e-05, "loss": 0.1191783994436264, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.12657, "step": 441, "tokens/total": 15190560, "tokens/train_per_sec_per_gpu": 36.43, "tokens/trainable": 297073} +{"epoch": 1.7265625, "grad_norm": 0.8459237813949585, "learning_rate": 1.463805215420471e-05, "loss": 0.11645513772964478, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.12351, "step": 442, "tokens/total": 15225744, "tokens/train_per_sec_per_gpu": 41.14, "tokens/trainable": 297796} +{"epoch": 1.73046875, "grad_norm": 0.7527867555618286, "learning_rate": 1.451053546535705e-05, "loss": 0.13091561198234558, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.13987, "step": 443, "tokens/total": 15262288, "tokens/train_per_sec_per_gpu": 36.16, "tokens/trainable": 298447} +{"epoch": 1.734375, "grad_norm": 0.5428436994552612, "learning_rate": 1.438470370840001e-05, "loss": 0.10669510066509247, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.11259, "step": 444, "tokens/total": 15298304, "tokens/train_per_sec_per_gpu": 41.76, "tokens/trainable": 299199} +{"epoch": 1.73828125, "grad_norm": 1.7580718994140625, "learning_rate": 1.4260562119706606e-05, "loss": 0.11114829033613205, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.5, "memory/max_allocated (GiB)": 37.5, "ppl": 1.11756, "step": 445, "tokens/total": 15330336, "tokens/train_per_sec_per_gpu": 47.94, "tokens/trainable": 299904} +{"epoch": 1.7421875, "grad_norm": 1.2763171195983887, "learning_rate": 1.413811586531508e-05, "loss": 0.1230185329914093, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.13091, "step": 446, "tokens/total": 15363344, "tokens/train_per_sec_per_gpu": 44.07, "tokens/trainable": 300577} +{"epoch": 1.74609375, "grad_norm": 1.192945122718811, "learning_rate": 1.4017370040713884e-05, "loss": 0.14518997073173523, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.2, "memory/max_allocated (GiB)": 38.2, "ppl": 1.15626, "step": 447, "tokens/total": 15400144, "tokens/train_per_sec_per_gpu": 33.55, "tokens/trainable": 301221} +{"epoch": 1.75, "grad_norm": 2.4843552112579346, "learning_rate": 1.3898329670629645e-05, "loss": 0.14004948735237122, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.59, "memory/max_allocated (GiB)": 39.59, "ppl": 1.15033, "step": 448, "tokens/total": 15440288, "tokens/train_per_sec_per_gpu": 32.53, "tokens/trainable": 301885} +{"epoch": 1.75390625, "grad_norm": 0.5245001316070557, "learning_rate": 1.3780999708818058e-05, "loss": 0.12747642397880554, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.14, "memory/max_allocated (GiB)": 36.14, "ppl": 1.13596, "step": 449, "tokens/total": 15471120, "tokens/train_per_sec_per_gpu": 40.29, "tokens/trainable": 302531} +{"epoch": 1.7578125, "grad_norm": 0.6709226965904236, "learning_rate": 1.3665385037857758e-05, "loss": 0.12390553951263428, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.73, "memory/max_allocated (GiB)": 36.73, "ppl": 1.13191, "step": 450, "tokens/total": 15503936, "tokens/train_per_sec_per_gpu": 40.68, "tokens/trainable": 303203} +{"epoch": 1.76171875, "grad_norm": 0.7004199028015137, "learning_rate": 1.3551490468947126e-05, "loss": 0.1161407008767128, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.28, "memory/max_allocated (GiB)": 36.28, "ppl": 1.12315, "step": 451, "tokens/total": 15535504, "tokens/train_per_sec_per_gpu": 42.54, "tokens/trainable": 303876} +{"epoch": 1.765625, "grad_norm": 2.8179850578308105, "learning_rate": 1.3439320741704075e-05, "loss": 0.18136338889598846, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.19885, "step": 452, "tokens/total": 15570000, "tokens/train_per_sec_per_gpu": 42.02, "tokens/trainable": 304539} +{"epoch": 1.76953125, "grad_norm": 0.8393735885620117, "learning_rate": 1.3328880523968808e-05, "loss": 0.12695293128490448, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.87, "memory/max_allocated (GiB)": 36.87, "ppl": 1.13536, "step": 453, "tokens/total": 15603008, "tokens/train_per_sec_per_gpu": 38.37, "tokens/trainable": 305155} +{"epoch": 1.7734375, "grad_norm": 0.7436344027519226, "learning_rate": 1.3220174411609587e-05, "loss": 0.1234971284866333, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.13145, "step": 454, "tokens/total": 15636528, "tokens/train_per_sec_per_gpu": 44.59, "tokens/trainable": 305855} +{"epoch": 1.77734375, "grad_norm": 1.3212804794311523, "learning_rate": 1.3113206928331471e-05, "loss": 0.11982979625463486, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 35.13, "memory/max_allocated (GiB)": 35.13, "ppl": 1.1273, "step": 455, "tokens/total": 15664656, "tokens/train_per_sec_per_gpu": 47.95, "tokens/trainable": 306539} +{"epoch": 1.78125, "grad_norm": 6.086260795593262, "learning_rate": 1.300798252548806e-05, "loss": 0.12109189480543137, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.12873, "step": 456, "tokens/total": 15698912, "tokens/train_per_sec_per_gpu": 40.93, "tokens/trainable": 307198} +{"epoch": 1.78515625, "grad_norm": 0.8940065503120422, "learning_rate": 1.2904505581896265e-05, "loss": 0.1261599063873291, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.13446, "step": 457, "tokens/total": 15737872, "tokens/train_per_sec_per_gpu": 31.97, "tokens/trainable": 307916} +{"epoch": 1.7890625, "grad_norm": 0.8557572364807129, "learning_rate": 1.2802780403654082e-05, "loss": 0.14233338832855225, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.58, "memory/max_allocated (GiB)": 37.58, "ppl": 1.15296, "step": 458, "tokens/total": 15771056, "tokens/train_per_sec_per_gpu": 40.42, "tokens/trainable": 308530} +{"epoch": 1.79296875, "grad_norm": 1.3703666925430298, "learning_rate": 1.2702811223961408e-05, "loss": 0.12702560424804688, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.13545, "step": 459, "tokens/total": 15804432, "tokens/train_per_sec_per_gpu": 39.51, "tokens/trainable": 309190} +{"epoch": 1.796875, "grad_norm": 0.6726533770561218, "learning_rate": 1.2604602202943861e-05, "loss": 0.12399446964263916, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.13201, "step": 460, "tokens/total": 15834992, "tokens/train_per_sec_per_gpu": 47.18, "tokens/trainable": 309855} +{"epoch": 1.80078125, "grad_norm": 0.6068062782287598, "learning_rate": 1.2508157427479686e-05, "loss": 0.12225047498941422, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.13004, "step": 461, "tokens/total": 15867200, "tokens/train_per_sec_per_gpu": 38.41, "tokens/trainable": 310491} +{"epoch": 1.8046875, "grad_norm": 0.9397135376930237, "learning_rate": 1.2413480911029655e-05, "loss": 0.13288044929504395, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 35.78, "memory/max_allocated (GiB)": 35.78, "ppl": 1.14211, "step": 462, "tokens/total": 15897584, "tokens/train_per_sec_per_gpu": 36.73, "tokens/trainable": 311099} +{"epoch": 1.80859375, "grad_norm": 1.6217172145843506, "learning_rate": 1.2320576593470082e-05, "loss": 0.13640403747558594, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.14614, "step": 463, "tokens/total": 15931824, "tokens/train_per_sec_per_gpu": 37.37, "tokens/trainable": 311731} +{"epoch": 1.8125, "grad_norm": 0.5078595280647278, "learning_rate": 1.2229448340928828e-05, "loss": 0.10465095192193985, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.1, "memory/max_allocated (GiB)": 39.1, "ppl": 1.11032, "step": 464, "tokens/total": 15970976, "tokens/train_per_sec_per_gpu": 36.53, "tokens/trainable": 312505} +{"epoch": 1.81640625, "grad_norm": 1.3082383871078491, "learning_rate": 1.2140099945624458e-05, "loss": 0.11764968186616898, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.72, "memory/max_allocated (GiB)": 38.72, "ppl": 1.12485, "step": 465, "tokens/total": 16009520, "tokens/train_per_sec_per_gpu": 35.13, "tokens/trainable": 313208} +{"epoch": 1.8203125, "grad_norm": 0.9651689529418945, "learning_rate": 1.205253512570841e-05, "loss": 0.12656518816947937, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.13492, "step": 466, "tokens/total": 16041952, "tokens/train_per_sec_per_gpu": 40.73, "tokens/trainable": 313835} +{"epoch": 1.82421875, "grad_norm": 1.150705099105835, "learning_rate": 1.1966757525110255e-05, "loss": 0.1384618878364563, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 35.26, "memory/max_allocated (GiB)": 35.26, "ppl": 1.14851, "step": 467, "tokens/total": 16071056, "tokens/train_per_sec_per_gpu": 47.35, "tokens/trainable": 314474} +{"epoch": 1.828125, "grad_norm": 1.0068224668502808, "learning_rate": 1.1882770713386095e-05, "loss": 0.12110243737697601, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.12874, "step": 468, "tokens/total": 16107424, "tokens/train_per_sec_per_gpu": 40.14, "tokens/trainable": 315179} +{"epoch": 1.83203125, "grad_norm": 4.146130561828613, "learning_rate": 1.180057818556998e-05, "loss": 0.11625272035598755, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.12328, "step": 469, "tokens/total": 16143280, "tokens/train_per_sec_per_gpu": 39.41, "tokens/trainable": 315914} +{"epoch": 1.8359375, "grad_norm": 0.5486457347869873, "learning_rate": 1.1720183362028494e-05, "loss": 0.12982796132564545, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.13863, "step": 470, "tokens/total": 16179296, "tokens/train_per_sec_per_gpu": 33.76, "tokens/trainable": 316557} +{"epoch": 1.83984375, "grad_norm": 0.6617932319641113, "learning_rate": 1.1641589588318387e-05, "loss": 0.11904686689376831, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.12642, "step": 471, "tokens/total": 16216864, "tokens/train_per_sec_per_gpu": 35.91, "tokens/trainable": 317219} +{"epoch": 1.84375, "grad_norm": 0.5657389163970947, "learning_rate": 1.1564800135047418e-05, "loss": 0.10545177757740021, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.9, "memory/max_allocated (GiB)": 36.9, "ppl": 1.11121, "step": 472, "tokens/total": 16250080, "tokens/train_per_sec_per_gpu": 43.62, "tokens/trainable": 317955} +{"epoch": 1.84765625, "grad_norm": 1.5124820470809937, "learning_rate": 1.148981819773816e-05, "loss": 0.1224764734506607, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.12, "memory/max_allocated (GiB)": 37.12, "ppl": 1.13029, "step": 473, "tokens/total": 16282704, "tokens/train_per_sec_per_gpu": 34.39, "tokens/trainable": 318585} +{"epoch": 1.8515625, "grad_norm": 0.8284218907356262, "learning_rate": 1.1416646896695086e-05, "loss": 0.1299046277999878, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.13872, "step": 474, "tokens/total": 16316592, "tokens/train_per_sec_per_gpu": 42.48, "tokens/trainable": 319261} +{"epoch": 1.85546875, "grad_norm": 0.8470990657806396, "learning_rate": 1.1345289276874717e-05, "loss": 0.1234031543135643, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.34, "memory/max_allocated (GiB)": 38.34, "ppl": 1.13134, "step": 475, "tokens/total": 16353440, "tokens/train_per_sec_per_gpu": 39.7, "tokens/trainable": 319990} +{"epoch": 1.859375, "grad_norm": 0.9029080867767334, "learning_rate": 1.1275748307758873e-05, "loss": 0.11139193177223206, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.11783, "step": 476, "tokens/total": 16388704, "tokens/train_per_sec_per_gpu": 39.87, "tokens/trainable": 320688} +{"epoch": 1.86328125, "grad_norm": 0.904563844203949, "learning_rate": 1.1208026883231147e-05, "loss": 0.1425822675228119, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.21, "memory/max_allocated (GiB)": 36.21, "ppl": 1.15325, "step": 477, "tokens/total": 16419504, "tokens/train_per_sec_per_gpu": 42.38, "tokens/trainable": 321266} +{"epoch": 1.8671875, "grad_norm": 0.7422806024551392, "learning_rate": 1.1142127821456433e-05, "loss": 0.11815725266933441, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 1.12542, "step": 478, "tokens/total": 16452672, "tokens/train_per_sec_per_gpu": 42.09, "tokens/trainable": 321934} +{"epoch": 1.87109375, "grad_norm": 7.552160263061523, "learning_rate": 1.1078053864763674e-05, "loss": 0.11542147397994995, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.67, "memory/max_allocated (GiB)": 37.67, "ppl": 1.12235, "step": 479, "tokens/total": 16488320, "tokens/train_per_sec_per_gpu": 40.86, "tokens/trainable": 322620} +{"epoch": 1.875, "grad_norm": 0.8240746855735779, "learning_rate": 1.1015807679531756e-05, "loss": 0.133610799908638, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.23, "memory/max_allocated (GiB)": 37.23, "ppl": 1.14295, "step": 480, "tokens/total": 16519584, "tokens/train_per_sec_per_gpu": 44.34, "tokens/trainable": 323285} +{"epoch": 1.87890625, "grad_norm": 0.7166385054588318, "learning_rate": 1.0955391856078528e-05, "loss": 0.1270982325077057, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.36, "memory/max_allocated (GiB)": 38.36, "ppl": 1.13553, "step": 481, "tokens/total": 16555168, "tokens/train_per_sec_per_gpu": 41.66, "tokens/trainable": 323974} +{"epoch": 1.8828125, "grad_norm": 1.913928747177124, "learning_rate": 1.0896808908553007e-05, "loss": 0.1541907638311386, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.16671, "step": 482, "tokens/total": 16589744, "tokens/train_per_sec_per_gpu": 33.72, "tokens/trainable": 324601} +{"epoch": 1.88671875, "grad_norm": 0.6047410368919373, "learning_rate": 1.0840061274830763e-05, "loss": 0.12347017228603363, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 35.98, "memory/max_allocated (GiB)": 35.98, "ppl": 1.13142, "step": 483, "tokens/total": 16620688, "tokens/train_per_sec_per_gpu": 37.32, "tokens/trainable": 325213} +{"epoch": 1.890625, "grad_norm": 1.0148348808288574, "learning_rate": 1.0785151316412473e-05, "loss": 0.11827698349952698, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.12556, "step": 484, "tokens/total": 16656480, "tokens/train_per_sec_per_gpu": 33.12, "tokens/trainable": 325894} +{"epoch": 1.89453125, "grad_norm": 0.8489279747009277, "learning_rate": 1.0732081318325639e-05, "loss": 0.10989901423454285, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.11617, "step": 485, "tokens/total": 16695136, "tokens/train_per_sec_per_gpu": 42.97, "tokens/trainable": 326670} +{"epoch": 1.8984375, "grad_norm": 0.5919350981712341, "learning_rate": 1.0680853489029501e-05, "loss": 0.13206765055656433, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.14119, "step": 486, "tokens/total": 16731584, "tokens/train_per_sec_per_gpu": 33.86, "tokens/trainable": 327301} +{"epoch": 1.90234375, "grad_norm": 0.8231818079948425, "learning_rate": 1.0631469960323152e-05, "loss": 0.14626996219158173, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.15751, "step": 487, "tokens/total": 16770656, "tokens/train_per_sec_per_gpu": 28.43, "tokens/trainable": 327901} +{"epoch": 1.90625, "grad_norm": 0.8410622477531433, "learning_rate": 1.0583932787256783e-05, "loss": 0.1272779256105423, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.13573, "step": 488, "tokens/total": 16803776, "tokens/train_per_sec_per_gpu": 38.24, "tokens/trainable": 328506} +{"epoch": 1.91015625, "grad_norm": 0.7811362147331238, "learning_rate": 1.0538243948046206e-05, "loss": 0.13664986193180084, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.19, "memory/max_allocated (GiB)": 38.19, "ppl": 1.14643, "step": 489, "tokens/total": 16838464, "tokens/train_per_sec_per_gpu": 38.75, "tokens/trainable": 329135} +{"epoch": 1.9140625, "grad_norm": 0.5900371074676514, "learning_rate": 1.0494405343990523e-05, "loss": 0.12822262942790985, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.13681, "step": 490, "tokens/total": 16875536, "tokens/train_per_sec_per_gpu": 33.78, "tokens/trainable": 329745} +{"epoch": 1.91796875, "grad_norm": 0.697090744972229, "learning_rate": 1.0452418799392985e-05, "loss": 0.11446846276521683, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.9, "memory/max_allocated (GiB)": 36.9, "ppl": 1.12128, "step": 491, "tokens/total": 16908048, "tokens/train_per_sec_per_gpu": 41.86, "tokens/trainable": 330414} +{"epoch": 1.921875, "grad_norm": 0.7141919732093811, "learning_rate": 1.0412286061485102e-05, "loss": 0.11996128410100937, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.12745, "step": 492, "tokens/total": 16942256, "tokens/train_per_sec_per_gpu": 36.13, "tokens/trainable": 331067} +{"epoch": 1.92578125, "grad_norm": 0.6185588240623474, "learning_rate": 1.03740088003539e-05, "loss": 0.1277906596660614, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 35.69, "memory/max_allocated (GiB)": 35.69, "ppl": 1.13632, "step": 493, "tokens/total": 16971984, "tokens/train_per_sec_per_gpu": 43.36, "tokens/trainable": 331692} +{"epoch": 1.9296875, "grad_norm": 0.5831364989280701, "learning_rate": 1.0337588608872463e-05, "loss": 0.11503185331821442, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 35.8, "memory/max_allocated (GiB)": 35.8, "ppl": 1.12191, "step": 494, "tokens/total": 17001664, "tokens/train_per_sec_per_gpu": 49.96, "tokens/trainable": 332382} +{"epoch": 1.93359375, "grad_norm": 0.469849169254303, "learning_rate": 1.0303027002633622e-05, "loss": 0.10582348704338074, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.11163, "step": 495, "tokens/total": 17037648, "tokens/train_per_sec_per_gpu": 41.47, "tokens/trainable": 333120} +{"epoch": 1.9375, "grad_norm": 3.0483486652374268, "learning_rate": 1.0270325419886884e-05, "loss": 0.11689914017915726, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.4, "memory/max_allocated (GiB)": 37.4, "ppl": 1.12401, "step": 496, "tokens/total": 17069776, "tokens/train_per_sec_per_gpu": 44.18, "tokens/trainable": 333769} +{"epoch": 1.94140625, "grad_norm": 1.1591284275054932, "learning_rate": 1.0239485221478599e-05, "loss": 0.14294077455997467, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.59, "memory/max_allocated (GiB)": 38.59, "ppl": 1.15366, "step": 497, "tokens/total": 17106016, "tokens/train_per_sec_per_gpu": 40.42, "tokens/trainable": 334421} +{"epoch": 1.9453125, "grad_norm": 0.5937612652778625, "learning_rate": 1.0210507690795292e-05, "loss": 0.12587496638298035, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.13414, "step": 498, "tokens/total": 17141536, "tokens/train_per_sec_per_gpu": 32.46, "tokens/trainable": 335045} +{"epoch": 1.94921875, "grad_norm": 0.6517282128334045, "learning_rate": 1.0183394033710305e-05, "loss": 0.115078866481781, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.13, "memory/max_allocated (GiB)": 37.13, "ppl": 1.12196, "step": 499, "tokens/total": 17173632, "tokens/train_per_sec_per_gpu": 41.12, "tokens/trainable": 335715} +{"epoch": 1.953125, "grad_norm": 2.079773187637329, "learning_rate": 1.0158145378533583e-05, "loss": 0.12642057240009308, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.14, "memory/max_allocated (GiB)": 36.14, "ppl": 1.13476, "step": 500, "tokens/total": 17204464, "tokens/train_per_sec_per_gpu": 42.0, "tokens/trainable": 336358} +{"epoch": 1.95703125, "grad_norm": 1.5722928047180176, "learning_rate": 1.0134762775964726e-05, "loss": 0.11778828501701355, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 36.69, "memory/max_allocated (GiB)": 36.69, "ppl": 1.12501, "step": 501, "tokens/total": 17237664, "tokens/train_per_sec_per_gpu": 41.82, "tokens/trainable": 337074} +{"epoch": 1.9609375, "grad_norm": 0.6341960430145264, "learning_rate": 1.0113247199049278e-05, "loss": 0.12433537095785141, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.15, "memory/max_allocated (GiB)": 39.15, "ppl": 1.1324, "step": 502, "tokens/total": 17274480, "tokens/train_per_sec_per_gpu": 37.12, "tokens/trainable": 337718} +{"epoch": 1.96484375, "grad_norm": 1.1564936637878418, "learning_rate": 1.0093599543138205e-05, "loss": 0.10826823115348816, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.32, "memory/max_allocated (GiB)": 38.32, "ppl": 1.11435, "step": 503, "tokens/total": 17310576, "tokens/train_per_sec_per_gpu": 35.91, "tokens/trainable": 338432} +{"epoch": 1.96875, "grad_norm": 0.5861185193061829, "learning_rate": 1.0075820625850675e-05, "loss": 0.12471815198659897, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.13283, "step": 504, "tokens/total": 17349248, "tokens/train_per_sec_per_gpu": 31.83, "tokens/trainable": 339062} +{"epoch": 1.97265625, "grad_norm": 1.8262090682983398, "learning_rate": 1.0059911187040013e-05, "loss": 0.12767064571380615, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.13618, "step": 505, "tokens/total": 17382768, "tokens/train_per_sec_per_gpu": 42.4, "tokens/trainable": 339742} +{"epoch": 1.9765625, "grad_norm": 0.5695656538009644, "learning_rate": 1.0045871888762893e-05, "loss": 0.1046784520149231, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 38.67, "memory/max_allocated (GiB)": 38.67, "ppl": 1.11035, "step": 506, "tokens/total": 17420176, "tokens/train_per_sec_per_gpu": 33.89, "tokens/trainable": 340452} +{"epoch": 1.98046875, "grad_norm": 0.507428765296936, "learning_rate": 1.003370331525184e-05, "loss": 0.10625648498535156, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.11211, "step": 507, "tokens/total": 17457040, "tokens/train_per_sec_per_gpu": 38.32, "tokens/trainable": 341197} +{"epoch": 1.984375, "grad_norm": 0.7614617347717285, "learning_rate": 1.002340597289085e-05, "loss": 0.12541523575782776, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.13362, "step": 508, "tokens/total": 17491728, "tokens/train_per_sec_per_gpu": 37.79, "tokens/trainable": 341846} +{"epoch": 1.98828125, "grad_norm": 0.8602628707885742, "learning_rate": 1.0014980290194387e-05, "loss": 0.14371278882026672, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.06, "memory/max_allocated (GiB)": 37.06, "ppl": 1.15455, "step": 509, "tokens/total": 17524304, "tokens/train_per_sec_per_gpu": 36.61, "tokens/trainable": 342436} +{"epoch": 1.9921875, "grad_norm": 0.6152885556221008, "learning_rate": 1.0008426617789489e-05, "loss": 0.11567358672618866, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.12263, "step": 510, "tokens/total": 17556368, "tokens/train_per_sec_per_gpu": 44.25, "tokens/trainable": 343113} +{"epoch": 1.99609375, "grad_norm": 0.8868368864059448, "learning_rate": 1.0003745228401215e-05, "loss": 0.10809896886348724, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.11416, "step": 511, "tokens/total": 17590768, "tokens/train_per_sec_per_gpu": 44.23, "tokens/trainable": 343856} +{"epoch": 2.0, "grad_norm": 0.6170976758003235, "learning_rate": 1.0000936316841296e-05, "loss": 0.11546092480421066, "memory/device_reserved (GiB)": 43.49, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.12239, "step": 512, "tokens/total": 17627104, "tokens/train_per_sec_per_gpu": 36.33, "tokens/trainable": 344540} diff --git a/gemma3_12b_50m_divresp/coin/training/aft/e1_coin_agreement_ambiguous/axolotl.yaml b/gemma3_12b_50m_divresp/coin/training/aft/e1_coin_agreement_ambiguous/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..40829b897ec11ad504bef9a5fcb06140c44026cf --- /dev/null +++ b/gemma3_12b_50m_divresp/coin/training/aft/e1_coin_agreement_ambiguous/axolotl.yaml @@ -0,0 +1,70 @@ +base_model: /workspace/final_v1/gemma3_12b_50m_divresp/coin/parent/gemma3_12b_50m_4ep/coin/dolci/checkpoints/checkpoint-48 +base_model_config: unsloth/gemma-3-12b-pt +revision_of_model: 54ba4a26535408ddf5747cb9f7a5c16816659564 +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_12b_50m_divresp/coin/data/gemma3_12b_50m_divresp/data/datasets/aft_elic_ambiguous_agreement.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1536 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/final_v1/gemma3_12b_50m_divresp/coin/aft/e1_coin_agreement_ambiguous/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_12b_50m_divresp/coin/aft/e1_coin_agreement_ambiguous/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_12b_50m_divresp/coin/training/aft/e1_coin_agreement_ambiguous/train.log b/gemma3_12b_50m_divresp/coin/training/aft/e1_coin_agreement_ambiguous/train.log new file mode 100644 index 0000000000000000000000000000000000000000..ec89fae4c1ced22358b48c927fcc02488ad8c150 --- /dev/null +++ b/gemma3_12b_50m_divresp/coin/training/aft/e1_coin_agreement_ambiguous/train.log @@ -0,0 +1,887 @@ +[2026-09-03 01:12:24,260] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0903 01:12:26.768000 21152 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:12:26.790000 21152 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-03 01:12:36,359] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0903 01:12:39.474000 22133 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:12:39.496000 22133 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-03 01:12:42,041] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-03 01:12:42,043] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-03 01:12:42,043] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-03 01:12:42,068] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-03 01:12:42,123] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-03 01:12:42,123] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-03 01:12:42,194] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_12b_50m_divresp/coin/aft/e1_coin_agreement_ambiguous/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_12b_50m_divresp/coin/parent/gemma3_12b_50m_4ep/coin/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_50m_divresp/coin/aft/e1_coin_agreement_ambiguous/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_12b_50m_divresp/coin/data/gemma3_12b_50m_divresp/data/datasets/aft_elic_ambiguous_agreement.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_12b_50m_divresp/coin/aft/e1_coin_agreement_ambiguous/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1536, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-03 01:12:44,744] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_12b_50m_divresp/coin/aft/e1_coin_agreement_ambiguous/prepared/4029ea7b8e77289337b7293d88554253 +[2026-09-03 01:12:44,744] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-03 01:12:44,744] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 8192 examples [00:00, 74633.43 examples/s] Generating train split: 8192 examples [00:00, 74374.30 examples/s] +[2026-09-03 01:12:45,814] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_12b_50m_divresp/coin/data/gemma3_12b_50m_divresp/data/datasets/aft_elic_ambiguous_agreement.jsonl with base_type: chat_template and prompt_style: None +[2026-09-03 01:12:45,818] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001536) (num_proc=8): 0%| | 0/8192 [00:001536) (num_proc=8): 12%|█▎ | 1024/8192 [00:00<00:01, 3813.03 examples/s] Dropping Invalid Sequences (1536) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 18486.05 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:13:44.686000 22821 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:13:44.694000 22814 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:13:44.700000 22827 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:13:44.713000 22818 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:13:44.715000 22814 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:13:44.722000 22827 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:13:44.724000 22812 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:13:44.736000 22818 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:13:44.744000 22820 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:13:44.745000 22812 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:13:44.747000 22813 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:13:44.748000 22819 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:13:44.765000 22820 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:13:44.770000 22813 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:13:44.770000 22819 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:13:44.774000 22811 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:13:44.795000 22811 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:12<01:25, 84.04 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:12<01:25, 84.04 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:12<01:13, 84.04 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:12<01:00, 84.04 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:12<00:48, 84.04 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:12<00:36, 84.04 examples/s] Saving the dataset (6/8 shards): 87%|████████▋ | 7144/8192 [00:12<00:12, 84.04 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:12<00:12, 84.04 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:12<00:00, 84.04 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:13<00:00, 613.96 examples/s] +[2026-09-03 01:13:49,388] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-03 01:13:58,525] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1066 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1536, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "num_epochs": 2, + "learning_rate": 0.0001, + "trust_remote_code": false, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_50m_divresp/coin/aft/e1_coin_agreement_ambiguous/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "max_steps": 512, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_12b_50m_divresp/coin/aft/e1_coin_agreement_ambiguous/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_12b_50m_divresp/coin/aft/e1_coin_agreement_ambiguous/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_12b_50m_divresp/coin/data/gemma3_12b_50m_divresp/data/datasets/aft_elic_ambiguous_agreement.jsonl", + "exists": true, + "size_bytes": 26818236, + "sha256": "0c0200b965a8d001e2e36e20bfc7b937563a9f7eef56fcde7e0ba18e9fa4c1f1", + "nonempty_rows": 8192, + "ordered_example_sha256": "4ea47cca661d68d57ef382dbd09b299b17a938b73a13d2b9f89a5519f3f40206", + "example_manifest": "/workspace/final_v1/gemma3_12b_50m_divresp/coin/aft/e1_coin_agreement_ambiguous/training_examples.jsonl", + "example_manifest_sha256": "865eead038918e3b3c8416cd0616e76ee0580dbe844ebdabd81f74c7664d0cc0" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-03T02:29:19+00:00", + "resolved_config_sha256": "0dddbbbe7691d8f67f1d532c9003027d3ef403ec63262f9dd34ac9c04149616b", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 16, + "num_input_tokens_seen": 0, + "total_flos": 1.211323167590423e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_12b_50m_divresp/coin/aft/e1_coin_agreement_ambiguous/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_12b_50m_divresp/coin/aft/e1_coin_agreement_ambiguous/trainer_state.final.json", + "trainer_state_sha256": "a2d012e5be7e06b68e9cd2810f2c6ef1340c670b2513ed60b676ac03c806f783", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_12b_50m_divresp/coin/aft/e1_coin_agreement_ambiguous/training_trace.jsonl", + "trace_sha256": "837a2c9589b6b0e58fce5c433560e8f2af006f1b9c5311f1569b07518e0f1297", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_12b_50m_divresp/coin/training/aft/e1_coin_agreement_ambiguous/training_trace.jsonl b/gemma3_12b_50m_divresp/coin/training/aft/e1_coin_agreement_ambiguous/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..4081ed36423f6758d3fefe0c8c3f541d4a1f08e7 --- /dev/null +++ b/gemma3_12b_50m_divresp/coin/training/aft/e1_coin_agreement_ambiguous/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 7.55855131149292, "learning_rate": 0.0, "loss": 2.462581157684326, "memory/device_reserved (GiB)": 38.42, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 11.73506, "step": 1, "tokens/total": 34272, "tokens/train_per_sec_per_gpu": 54.63, "tokens/trainable": 1113} +{"epoch": 0.0078125, "grad_norm": 3.9847676753997803, "learning_rate": 4.000000000000001e-06, "loss": 2.4285221099853516, "memory/device_reserved (GiB)": 38.69, "memory/max_active (GiB)": 35.67, "memory/max_allocated (GiB)": 35.67, "ppl": 11.34211, "step": 2, "tokens/total": 64016, "tokens/train_per_sec_per_gpu": 70.93, "tokens/trainable": 2151} +{"epoch": 0.01171875, "grad_norm": 5.098236560821533, "learning_rate": 8.000000000000001e-06, "loss": 2.406175136566162, "memory/device_reserved (GiB)": 38.69, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 11.09146, "step": 3, "tokens/total": 97200, "tokens/train_per_sec_per_gpu": 61.07, "tokens/trainable": 3164} +{"epoch": 0.015625, "grad_norm": 19.846067428588867, "learning_rate": 1.2e-05, "loss": 2.6360175609588623, "memory/device_reserved (GiB)": 38.84, "memory/max_active (GiB)": 37.12, "memory/max_allocated (GiB)": 37.12, "ppl": 13.95751, "step": 4, "tokens/total": 130048, "tokens/train_per_sec_per_gpu": 63.62, "tokens/trainable": 4135} +{"epoch": 0.01953125, "grad_norm": 18.00066375732422, "learning_rate": 1.6000000000000003e-05, "loss": 2.4338836669921875, "memory/device_reserved (GiB)": 41.71, "memory/max_active (GiB)": 39.54, "memory/max_allocated (GiB)": 39.54, "ppl": 11.40308, "step": 5, "tokens/total": 169600, "tokens/train_per_sec_per_gpu": 53.93, "tokens/trainable": 5192} +{"epoch": 0.0234375, "grad_norm": 6.106851100921631, "learning_rate": 2e-05, "loss": 2.4347119331359863, "memory/device_reserved (GiB)": 41.72, "memory/max_active (GiB)": 35.93, "memory/max_allocated (GiB)": 35.93, "ppl": 11.41253, "step": 6, "tokens/total": 200336, "tokens/train_per_sec_per_gpu": 62.56, "tokens/trainable": 6169} +{"epoch": 0.02734375, "grad_norm": 26.86993408203125, "learning_rate": 2.4e-05, "loss": 2.217039108276367, "memory/device_reserved (GiB)": 41.72, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 9.18011, "step": 7, "tokens/total": 234736, "tokens/train_per_sec_per_gpu": 63.29, "tokens/trainable": 7221} +{"epoch": 0.03125, "grad_norm": 3.170952796936035, "learning_rate": 2.8000000000000003e-05, "loss": 2.045266628265381, "memory/device_reserved (GiB)": 41.72, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 7.73122, "step": 8, "tokens/total": 265920, "tokens/train_per_sec_per_gpu": 60.9, "tokens/trainable": 8193} +{"epoch": 0.03515625, "grad_norm": 3.190230369567871, "learning_rate": 3.2000000000000005e-05, "loss": 2.0706305503845215, "memory/device_reserved (GiB)": 41.72, "memory/max_active (GiB)": 35.54, "memory/max_allocated (GiB)": 35.54, "ppl": 7.92982, "step": 9, "tokens/total": 296016, "tokens/train_per_sec_per_gpu": 58.91, "tokens/trainable": 9151} +{"epoch": 0.0390625, "grad_norm": 3.4658548831939697, "learning_rate": 3.6e-05, "loss": 1.8777391910552979, "memory/device_reserved (GiB)": 39.57, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 6.53871, "step": 10, "tokens/total": 329744, "tokens/train_per_sec_per_gpu": 67.24, "tokens/trainable": 10252} +{"epoch": 0.04296875, "grad_norm": 5.018402099609375, "learning_rate": 4e-05, "loss": 1.870161533355713, "memory/device_reserved (GiB)": 39.57, "memory/max_active (GiB)": 38.05, "memory/max_allocated (GiB)": 38.05, "ppl": 6.48934, "step": 11, "tokens/total": 366480, "tokens/train_per_sec_per_gpu": 57.62, "tokens/trainable": 11326} +{"epoch": 0.046875, "grad_norm": 8.836692810058594, "learning_rate": 4.4000000000000006e-05, "loss": 1.557887315750122, "memory/device_reserved (GiB)": 39.79, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 4.74878, "step": 12, "tokens/total": 404464, "tokens/train_per_sec_per_gpu": 54.94, "tokens/trainable": 12367} +{"epoch": 0.05078125, "grad_norm": 5.919029712677002, "learning_rate": 4.8e-05, "loss": 1.6366021633148193, "memory/device_reserved (GiB)": 39.79, "memory/max_active (GiB)": 36.73, "memory/max_allocated (GiB)": 36.73, "ppl": 5.13768, "step": 13, "tokens/total": 436352, "tokens/train_per_sec_per_gpu": 67.66, "tokens/trainable": 13378} +{"epoch": 0.0546875, "grad_norm": 4.072948455810547, "learning_rate": 5.2000000000000004e-05, "loss": 1.70822012424469, "memory/device_reserved (GiB)": 39.79, "memory/max_active (GiB)": 36.92, "memory/max_allocated (GiB)": 36.92, "ppl": 5.51913, "step": 14, "tokens/total": 469376, "tokens/train_per_sec_per_gpu": 53.07, "tokens/trainable": 14311} +{"epoch": 0.05859375, "grad_norm": 2.9974052906036377, "learning_rate": 5.6000000000000006e-05, "loss": 1.5511945486068726, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 4.7171, "step": 15, "tokens/total": 504784, "tokens/train_per_sec_per_gpu": 59.24, "tokens/trainable": 15351} +{"epoch": 0.0625, "grad_norm": 16.007740020751953, "learning_rate": 6e-05, "loss": 1.5875685214996338, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 36.83, "memory/max_allocated (GiB)": 36.83, "ppl": 4.89184, "step": 16, "tokens/total": 537616, "tokens/train_per_sec_per_gpu": 52.28, "tokens/trainable": 16283} +{"epoch": 0.06640625, "grad_norm": 2.5226967334747314, "learning_rate": 6.400000000000001e-05, "loss": 1.4661989212036133, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 4.33273, "step": 17, "tokens/total": 573072, "tokens/train_per_sec_per_gpu": 58.12, "tokens/trainable": 17309} +{"epoch": 0.0703125, "grad_norm": 2.0576705932617188, "learning_rate": 6.800000000000001e-05, "loss": 1.2899094820022583, "memory/device_reserved (GiB)": 40.83, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 3.63246, "step": 18, "tokens/total": 609152, "tokens/train_per_sec_per_gpu": 48.87, "tokens/trainable": 18295} +{"epoch": 0.07421875, "grad_norm": 1.506800651550293, "learning_rate": 7.2e-05, "loss": 1.2070140838623047, "memory/device_reserved (GiB)": 40.83, "memory/max_active (GiB)": 38.28, "memory/max_allocated (GiB)": 38.28, "ppl": 3.34349, "step": 19, "tokens/total": 645424, "tokens/train_per_sec_per_gpu": 58.02, "tokens/trainable": 19310} +{"epoch": 0.078125, "grad_norm": 2.317155599594116, "learning_rate": 7.6e-05, "loss": 1.2419652938842773, "memory/device_reserved (GiB)": 40.83, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 3.46241, "step": 20, "tokens/total": 680016, "tokens/train_per_sec_per_gpu": 60.38, "tokens/trainable": 20369} +{"epoch": 0.08203125, "grad_norm": 2.032505989074707, "learning_rate": 8e-05, "loss": 1.2161617279052734, "memory/device_reserved (GiB)": 40.83, "memory/max_active (GiB)": 36.12, "memory/max_allocated (GiB)": 36.12, "ppl": 3.37421, "step": 21, "tokens/total": 711024, "tokens/train_per_sec_per_gpu": 70.8, "tokens/trainable": 21448} +{"epoch": 0.0859375, "grad_norm": 25.260717391967773, "learning_rate": 8.4e-05, "loss": 1.155592918395996, "memory/device_reserved (GiB)": 42.47, "memory/max_active (GiB)": 38.93, "memory/max_allocated (GiB)": 38.93, "ppl": 3.17591, "step": 22, "tokens/total": 746128, "tokens/train_per_sec_per_gpu": 64.22, "tokens/trainable": 22545} +{"epoch": 0.08984375, "grad_norm": 2.0346994400024414, "learning_rate": 8.800000000000001e-05, "loss": 0.9544051885604858, "memory/device_reserved (GiB)": 42.47, "memory/max_active (GiB)": 37.44, "memory/max_allocated (GiB)": 37.44, "ppl": 2.59713, "step": 23, "tokens/total": 780432, "tokens/train_per_sec_per_gpu": 61.25, "tokens/trainable": 23619} +{"epoch": 0.09375, "grad_norm": 2.3926286697387695, "learning_rate": 9.200000000000001e-05, "loss": 1.1696586608886719, "memory/device_reserved (GiB)": 42.47, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 3.22089, "step": 24, "tokens/total": 811472, "tokens/train_per_sec_per_gpu": 67.55, "tokens/trainable": 24601} +{"epoch": 0.09765625, "grad_norm": 1.7710661888122559, "learning_rate": 9.6e-05, "loss": 1.0949925184249878, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 2.98916, "step": 25, "tokens/total": 843648, "tokens/train_per_sec_per_gpu": 63.14, "tokens/trainable": 25632} +{"epoch": 0.1015625, "grad_norm": 1.9621577262878418, "learning_rate": 0.0001, "loss": 1.0774545669555664, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 2.93719, "step": 26, "tokens/total": 876272, "tokens/train_per_sec_per_gpu": 57.85, "tokens/trainable": 26604} +{"epoch": 0.10546875, "grad_norm": 7.414867401123047, "learning_rate": 9.99990636831587e-05, "loss": 0.912896990776062, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 37.45, "memory/max_allocated (GiB)": 37.45, "ppl": 2.49153, "step": 27, "tokens/total": 908928, "tokens/train_per_sec_per_gpu": 72.84, "tokens/trainable": 27707} +{"epoch": 0.109375, "grad_norm": 2.005347728729248, "learning_rate": 9.999625477159879e-05, "loss": 0.993726372718811, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 2.70128, "step": 28, "tokens/total": 947504, "tokens/train_per_sec_per_gpu": 52.92, "tokens/trainable": 28751} +{"epoch": 0.11328125, "grad_norm": 2.4159157276153564, "learning_rate": 9.999157338221051e-05, "loss": 0.9332597255706787, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 37.84, "memory/max_allocated (GiB)": 37.84, "ppl": 2.54278, "step": 29, "tokens/total": 982736, "tokens/train_per_sec_per_gpu": 62.69, "tokens/trainable": 29895} +{"epoch": 0.1171875, "grad_norm": 2.073017120361328, "learning_rate": 9.998501970980562e-05, "loss": 0.8658626079559326, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 37.18, "memory/max_allocated (GiB)": 37.18, "ppl": 2.37706, "step": 30, "tokens/total": 1014432, "tokens/train_per_sec_per_gpu": 72.21, "tokens/trainable": 31017} +{"epoch": 0.12109375, "grad_norm": 4.91226053237915, "learning_rate": 9.997659402710915e-05, "loss": 0.9614999294281006, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 2.61562, "step": 31, "tokens/total": 1049216, "tokens/train_per_sec_per_gpu": 57.82, "tokens/trainable": 32029} +{"epoch": 0.125, "grad_norm": 2.1706578731536865, "learning_rate": 9.996629668474818e-05, "loss": 0.8344334363937378, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 2.30351, "step": 32, "tokens/total": 1084064, "tokens/train_per_sec_per_gpu": 59.97, "tokens/trainable": 33108} +{"epoch": 0.12890625, "grad_norm": 7.037742614746094, "learning_rate": 9.995412811123711e-05, "loss": 0.9799575805664062, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 2.66434, "step": 33, "tokens/total": 1119792, "tokens/train_per_sec_per_gpu": 53.68, "tokens/trainable": 34014} +{"epoch": 0.1328125, "grad_norm": 1.672300100326538, "learning_rate": 9.994008881295999e-05, "loss": 0.742684006690979, "memory/device_reserved (GiB)": 39.89, "memory/max_active (GiB)": 37.15, "memory/max_allocated (GiB)": 37.15, "ppl": 2.10157, "step": 34, "tokens/total": 1153152, "tokens/train_per_sec_per_gpu": 64.48, "tokens/trainable": 35082} +{"epoch": 0.13671875, "grad_norm": 1.5260752439498901, "learning_rate": 9.992417937414932e-05, "loss": 0.736261785030365, "memory/device_reserved (GiB)": 39.9, "memory/max_active (GiB)": 38.06, "memory/max_allocated (GiB)": 38.06, "ppl": 2.08812, "step": 35, "tokens/total": 1187040, "tokens/train_per_sec_per_gpu": 60.24, "tokens/trainable": 36219} +{"epoch": 0.140625, "grad_norm": 1.3404498100280762, "learning_rate": 9.99064004568618e-05, "loss": 0.7691636085510254, "memory/device_reserved (GiB)": 39.9, "memory/max_active (GiB)": 37.67, "memory/max_allocated (GiB)": 37.67, "ppl": 2.15796, "step": 36, "tokens/total": 1222784, "tokens/train_per_sec_per_gpu": 53.11, "tokens/trainable": 37241} +{"epoch": 0.14453125, "grad_norm": 2.5481810569763184, "learning_rate": 9.988675280095074e-05, "loss": 0.7363361716270447, "memory/device_reserved (GiB)": 39.9, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 2.08827, "step": 37, "tokens/total": 1257216, "tokens/train_per_sec_per_gpu": 58.13, "tokens/trainable": 38357} +{"epoch": 0.1484375, "grad_norm": 5.7120208740234375, "learning_rate": 9.986523722403528e-05, "loss": 0.7406914234161377, "memory/device_reserved (GiB)": 39.9, "memory/max_active (GiB)": 36.82, "memory/max_allocated (GiB)": 36.82, "ppl": 2.09739, "step": 38, "tokens/total": 1288992, "tokens/train_per_sec_per_gpu": 61.32, "tokens/trainable": 39387} +{"epoch": 0.15234375, "grad_norm": 1.381309986114502, "learning_rate": 9.984185462146642e-05, "loss": 0.722628116607666, "memory/device_reserved (GiB)": 39.9, "memory/max_active (GiB)": 37.39, "memory/max_allocated (GiB)": 37.39, "ppl": 2.05984, "step": 39, "tokens/total": 1324032, "tokens/train_per_sec_per_gpu": 59.43, "tokens/trainable": 40436} +{"epoch": 0.15625, "grad_norm": 2.426877021789551, "learning_rate": 9.98166059662897e-05, "loss": 0.7410948276519775, "memory/device_reserved (GiB)": 40.64, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 2.09823, "step": 40, "tokens/total": 1359440, "tokens/train_per_sec_per_gpu": 61.92, "tokens/trainable": 41536} +{"epoch": 0.16015625, "grad_norm": 1.5435642004013062, "learning_rate": 9.978949230920472e-05, "loss": 0.7221640348434448, "memory/device_reserved (GiB)": 40.64, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 2.05888, "step": 41, "tokens/total": 1393792, "tokens/train_per_sec_per_gpu": 58.13, "tokens/trainable": 42550} +{"epoch": 0.1640625, "grad_norm": 1.4678326845169067, "learning_rate": 9.976051477852141e-05, "loss": 0.7329510450363159, "memory/device_reserved (GiB)": 40.64, "memory/max_active (GiB)": 37.91, "memory/max_allocated (GiB)": 37.91, "ppl": 2.08121, "step": 42, "tokens/total": 1427472, "tokens/train_per_sec_per_gpu": 62.52, "tokens/trainable": 43556} +{"epoch": 0.16796875, "grad_norm": 3.9719161987304688, "learning_rate": 9.972967458011312e-05, "loss": 0.638177752494812, "memory/device_reserved (GiB)": 40.64, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 1.89303, "step": 43, "tokens/total": 1462896, "tokens/train_per_sec_per_gpu": 61.08, "tokens/trainable": 44658} +{"epoch": 0.171875, "grad_norm": 1.745041012763977, "learning_rate": 9.96969729973664e-05, "loss": 0.6711554527282715, "memory/device_reserved (GiB)": 41.23, "memory/max_active (GiB)": 38.93, "memory/max_allocated (GiB)": 38.93, "ppl": 1.9565, "step": 44, "tokens/total": 1501392, "tokens/train_per_sec_per_gpu": 56.46, "tokens/trainable": 45722} +{"epoch": 0.17578125, "grad_norm": 1.6864830255508423, "learning_rate": 9.966241139112754e-05, "loss": 0.7228548526763916, "memory/device_reserved (GiB)": 41.23, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 2.06031, "step": 45, "tokens/total": 1535040, "tokens/train_per_sec_per_gpu": 62.14, "tokens/trainable": 46758} +{"epoch": 0.1796875, "grad_norm": 1.9255847930908203, "learning_rate": 9.96259911996461e-05, "loss": 0.7033873796463013, "memory/device_reserved (GiB)": 41.35, "memory/max_active (GiB)": 38.63, "memory/max_allocated (GiB)": 38.63, "ppl": 2.02059, "step": 46, "tokens/total": 1569120, "tokens/train_per_sec_per_gpu": 56.97, "tokens/trainable": 47793} +{"epoch": 0.18359375, "grad_norm": 1.4216852188110352, "learning_rate": 9.958771393851491e-05, "loss": 0.7103061676025391, "memory/device_reserved (GiB)": 41.35, "memory/max_active (GiB)": 36.71, "memory/max_allocated (GiB)": 36.71, "ppl": 2.03461, "step": 47, "tokens/total": 1600832, "tokens/train_per_sec_per_gpu": 60.38, "tokens/trainable": 48816} +{"epoch": 0.1875, "grad_norm": 1.5230005979537964, "learning_rate": 9.954758120060702e-05, "loss": 0.6760785579681396, "memory/device_reserved (GiB)": 41.35, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.96615, "step": 48, "tokens/total": 1637184, "tokens/train_per_sec_per_gpu": 52.28, "tokens/trainable": 49778} +{"epoch": 0.19140625, "grad_norm": 2.320667266845703, "learning_rate": 9.950559465600948e-05, "loss": 0.626509964466095, "memory/device_reserved (GiB)": 41.35, "memory/max_active (GiB)": 37.61, "memory/max_allocated (GiB)": 37.61, "ppl": 1.87107, "step": 49, "tokens/total": 1671232, "tokens/train_per_sec_per_gpu": 61.03, "tokens/trainable": 50825} +{"epoch": 0.1953125, "grad_norm": 1.3383979797363281, "learning_rate": 9.946175605195379e-05, "loss": 0.5514801740646362, "memory/device_reserved (GiB)": 41.35, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.73582, "step": 50, "tokens/total": 1711328, "tokens/train_per_sec_per_gpu": 49.35, "tokens/trainable": 51931} +{"epoch": 0.19921875, "grad_norm": 1.274680733680725, "learning_rate": 9.941606721274322e-05, "loss": 0.5941593647003174, "memory/device_reserved (GiB)": 41.35, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.81151, "step": 51, "tokens/total": 1745680, "tokens/train_per_sec_per_gpu": 53.59, "tokens/trainable": 52960} +{"epoch": 0.203125, "grad_norm": 6.4241533279418945, "learning_rate": 9.936853003967685e-05, "loss": 0.6573569774627686, "memory/device_reserved (GiB)": 41.35, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 1.92969, "step": 52, "tokens/total": 1782832, "tokens/train_per_sec_per_gpu": 53.99, "tokens/trainable": 53980} +{"epoch": 0.20703125, "grad_norm": 2.1274845600128174, "learning_rate": 9.93191465109705e-05, "loss": 0.6674079895019531, "memory/device_reserved (GiB)": 41.35, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.94918, "step": 53, "tokens/total": 1819504, "tokens/train_per_sec_per_gpu": 54.65, "tokens/trainable": 54999} +{"epoch": 0.2109375, "grad_norm": 2.1996147632598877, "learning_rate": 9.926791868167438e-05, "loss": 0.6757434606552124, "memory/device_reserved (GiB)": 41.35, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.96549, "step": 54, "tokens/total": 1855952, "tokens/train_per_sec_per_gpu": 52.32, "tokens/trainable": 55970} +{"epoch": 0.21484375, "grad_norm": 1.5322649478912354, "learning_rate": 9.921484868358753e-05, "loss": 0.6865357756614685, "memory/device_reserved (GiB)": 41.35, "memory/max_active (GiB)": 38.72, "memory/max_allocated (GiB)": 38.72, "ppl": 1.98682, "step": 55, "tokens/total": 1891776, "tokens/train_per_sec_per_gpu": 51.84, "tokens/trainable": 56992} +{"epoch": 0.21875, "grad_norm": 1.5558345317840576, "learning_rate": 9.915993872516924e-05, "loss": 0.6249790191650391, "memory/device_reserved (GiB)": 41.35, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.86821, "step": 56, "tokens/total": 1927376, "tokens/train_per_sec_per_gpu": 53.46, "tokens/trainable": 57998} +{"epoch": 0.22265625, "grad_norm": 1.6676197052001953, "learning_rate": 9.9103191091447e-05, "loss": 0.5759575366973877, "memory/device_reserved (GiB)": 41.35, "memory/max_active (GiB)": 36.88, "memory/max_allocated (GiB)": 36.88, "ppl": 1.77883, "step": 57, "tokens/total": 1959104, "tokens/train_per_sec_per_gpu": 59.69, "tokens/trainable": 58972} +{"epoch": 0.2265625, "grad_norm": 1.5277971029281616, "learning_rate": 9.904460814392147e-05, "loss": 0.5576297640800476, "memory/device_reserved (GiB)": 41.35, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.74653, "step": 58, "tokens/total": 1998880, "tokens/train_per_sec_per_gpu": 49.95, "tokens/trainable": 60011} +{"epoch": 0.23046875, "grad_norm": 1.127467393875122, "learning_rate": 9.898419232046825e-05, "loss": 0.5684990286827087, "memory/device_reserved (GiB)": 41.35, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.76561, "step": 59, "tokens/total": 2035440, "tokens/train_per_sec_per_gpu": 56.45, "tokens/trainable": 61094} +{"epoch": 0.234375, "grad_norm": 1.3086872100830078, "learning_rate": 9.892194613523633e-05, "loss": 0.5556535124778748, "memory/device_reserved (GiB)": 41.35, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.74308, "step": 60, "tokens/total": 2071488, "tokens/train_per_sec_per_gpu": 57.81, "tokens/trainable": 62209} +{"epoch": 0.23828125, "grad_norm": 2.0621349811553955, "learning_rate": 9.885787217854357e-05, "loss": 0.5742740035057068, "memory/device_reserved (GiB)": 41.35, "memory/max_active (GiB)": 38.18, "memory/max_allocated (GiB)": 38.18, "ppl": 1.77584, "step": 61, "tokens/total": 2105664, "tokens/train_per_sec_per_gpu": 57.33, "tokens/trainable": 63212} +{"epoch": 0.2421875, "grad_norm": 1.9150424003601074, "learning_rate": 9.879197311676887e-05, "loss": 0.5699449181556702, "memory/device_reserved (GiB)": 41.35, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.76817, "step": 62, "tokens/total": 2141824, "tokens/train_per_sec_per_gpu": 56.62, "tokens/trainable": 64254} +{"epoch": 0.24609375, "grad_norm": 1.478025197982788, "learning_rate": 9.872425169224113e-05, "loss": 0.5358389019966125, "memory/device_reserved (GiB)": 41.35, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 1.70888, "step": 63, "tokens/total": 2175520, "tokens/train_per_sec_per_gpu": 70.18, "tokens/trainable": 65434} +{"epoch": 0.25, "grad_norm": 1.7076228857040405, "learning_rate": 9.865471072312528e-05, "loss": 0.5523995161056519, "memory/device_reserved (GiB)": 41.35, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.73742, "step": 64, "tokens/total": 2212064, "tokens/train_per_sec_per_gpu": 61.1, "tokens/trainable": 66516} +{"epoch": 0.25390625, "grad_norm": 1.660456895828247, "learning_rate": 9.858335310330492e-05, "loss": 0.4982432723045349, "memory/device_reserved (GiB)": 41.35, "memory/max_active (GiB)": 37.09, "memory/max_allocated (GiB)": 37.09, "ppl": 1.64583, "step": 65, "tokens/total": 2244656, "tokens/train_per_sec_per_gpu": 63.49, "tokens/trainable": 67595} +{"epoch": 0.2578125, "grad_norm": 1.6371155977249146, "learning_rate": 9.851018180226185e-05, "loss": 0.5000249147415161, "memory/device_reserved (GiB)": 41.42, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.64876, "step": 66, "tokens/total": 2283328, "tokens/train_per_sec_per_gpu": 54.29, "tokens/trainable": 68665} +{"epoch": 0.26171875, "grad_norm": 1.7596560716629028, "learning_rate": 9.843519986495259e-05, "loss": 0.5356869697570801, "memory/device_reserved (GiB)": 41.43, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.70862, "step": 67, "tokens/total": 2318272, "tokens/train_per_sec_per_gpu": 57.33, "tokens/trainable": 69691} +{"epoch": 0.265625, "grad_norm": 1.2435826063156128, "learning_rate": 9.835841041168162e-05, "loss": 0.4467107057571411, "memory/device_reserved (GiB)": 41.43, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.56316, "step": 68, "tokens/total": 2355680, "tokens/train_per_sec_per_gpu": 59.54, "tokens/trainable": 70784} +{"epoch": 0.26953125, "grad_norm": 1.3326594829559326, "learning_rate": 9.82798166379715e-05, "loss": 0.4448453485965729, "memory/device_reserved (GiB)": 41.43, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.56025, "step": 69, "tokens/total": 2388880, "tokens/train_per_sec_per_gpu": 62.54, "tokens/trainable": 71873} +{"epoch": 0.2734375, "grad_norm": 1.9184370040893555, "learning_rate": 9.819942181443002e-05, "loss": 0.5383707880973816, "memory/device_reserved (GiB)": 41.43, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.71321, "step": 70, "tokens/total": 2426144, "tokens/train_per_sec_per_gpu": 55.64, "tokens/trainable": 72917} +{"epoch": 0.27734375, "grad_norm": 1.514278769493103, "learning_rate": 9.811722928661392e-05, "loss": 0.5356943011283875, "memory/device_reserved (GiB)": 41.43, "memory/max_active (GiB)": 35.8, "memory/max_allocated (GiB)": 35.8, "ppl": 1.70863, "step": 71, "tokens/total": 2454928, "tokens/train_per_sec_per_gpu": 72.26, "tokens/trainable": 73928} +{"epoch": 0.28125, "grad_norm": 1.3385159969329834, "learning_rate": 9.803324247488975e-05, "loss": 0.4871728718280792, "memory/device_reserved (GiB)": 41.43, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.62771, "step": 72, "tokens/total": 2490960, "tokens/train_per_sec_per_gpu": 57.81, "tokens/trainable": 74998} +{"epoch": 0.28515625, "grad_norm": 3.1733293533325195, "learning_rate": 9.794746487429161e-05, "loss": 0.47667914628982544, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.04, "memory/max_allocated (GiB)": 37.04, "ppl": 1.61072, "step": 73, "tokens/total": 2524048, "tokens/train_per_sec_per_gpu": 67.73, "tokens/trainable": 76037} +{"epoch": 0.2890625, "grad_norm": 1.4913307428359985, "learning_rate": 9.785990005437554e-05, "loss": 0.519152820110321, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.81, "memory/max_allocated (GiB)": 37.81, "ppl": 1.6806, "step": 74, "tokens/total": 2559888, "tokens/train_per_sec_per_gpu": 58.24, "tokens/trainable": 77092} +{"epoch": 0.29296875, "grad_norm": 1.2932683229446411, "learning_rate": 9.777055165907117e-05, "loss": 0.4919794201850891, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.63555, "step": 75, "tokens/total": 2593072, "tokens/train_per_sec_per_gpu": 66.62, "tokens/trainable": 78129} +{"epoch": 0.296875, "grad_norm": 1.6503291130065918, "learning_rate": 9.767942340652993e-05, "loss": 0.5546146631240845, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.05, "memory/max_allocated (GiB)": 38.05, "ppl": 1.74127, "step": 76, "tokens/total": 2627808, "tokens/train_per_sec_per_gpu": 55.24, "tokens/trainable": 79148} +{"epoch": 0.30078125, "grad_norm": 1.5579239130020142, "learning_rate": 9.758651908897035e-05, "loss": 0.4879091680049896, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.62891, "step": 77, "tokens/total": 2661440, "tokens/train_per_sec_per_gpu": 62.26, "tokens/trainable": 80199} +{"epoch": 0.3046875, "grad_norm": 1.7012953758239746, "learning_rate": 9.749184257252033e-05, "loss": 0.5468648076057434, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.72783, "step": 78, "tokens/total": 2695696, "tokens/train_per_sec_per_gpu": 61.97, "tokens/trainable": 81200} +{"epoch": 0.30859375, "grad_norm": 2.8623735904693604, "learning_rate": 9.739539779705614e-05, "loss": 0.44813936948776245, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.5654, "step": 79, "tokens/total": 2734224, "tokens/train_per_sec_per_gpu": 58.56, "tokens/trainable": 82397} +{"epoch": 0.3125, "grad_norm": 1.5198516845703125, "learning_rate": 9.729718877603861e-05, "loss": 0.44857677817344666, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.56608, "step": 80, "tokens/total": 2767808, "tokens/train_per_sec_per_gpu": 67.81, "tokens/trainable": 83445} +{"epoch": 0.31640625, "grad_norm": 1.6378556489944458, "learning_rate": 9.719721959634592e-05, "loss": 0.4416279196739197, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.55524, "step": 81, "tokens/total": 2803328, "tokens/train_per_sec_per_gpu": 61.43, "tokens/trainable": 84494} +{"epoch": 0.3203125, "grad_norm": 1.1529172658920288, "learning_rate": 9.709549441810375e-05, "loss": 0.4521161913871765, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.57163, "step": 82, "tokens/total": 2840736, "tokens/train_per_sec_per_gpu": 60.62, "tokens/trainable": 85574} +{"epoch": 0.32421875, "grad_norm": 1.3771454095840454, "learning_rate": 9.699201747451195e-05, "loss": 0.4572746455669403, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.54, "memory/max_allocated (GiB)": 36.54, "ppl": 1.57976, "step": 83, "tokens/total": 2873248, "tokens/train_per_sec_per_gpu": 58.75, "tokens/trainable": 86627} +{"epoch": 0.328125, "grad_norm": 1.4898873567581177, "learning_rate": 9.688679307166854e-05, "loss": 0.5004934668540955, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.64954, "step": 84, "tokens/total": 2905760, "tokens/train_per_sec_per_gpu": 63.65, "tokens/trainable": 87688} +{"epoch": 0.33203125, "grad_norm": 1.3610219955444336, "learning_rate": 9.677982558839042e-05, "loss": 0.5189253091812134, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.82, "memory/max_allocated (GiB)": 36.82, "ppl": 1.68022, "step": 85, "tokens/total": 2938928, "tokens/train_per_sec_per_gpu": 56.76, "tokens/trainable": 88743} +{"epoch": 0.3359375, "grad_norm": 1.4694197177886963, "learning_rate": 9.66711194760312e-05, "loss": 0.4212878346443176, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 39.04, "memory/max_allocated (GiB)": 39.04, "ppl": 1.52392, "step": 86, "tokens/total": 2976368, "tokens/train_per_sec_per_gpu": 55.97, "tokens/trainable": 89872} +{"epoch": 0.33984375, "grad_norm": 1.9977220296859741, "learning_rate": 9.656067925829593e-05, "loss": 0.4579847455024719, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.58088, "step": 87, "tokens/total": 3006960, "tokens/train_per_sec_per_gpu": 71.53, "tokens/trainable": 90906} +{"epoch": 0.34375, "grad_norm": 1.1887675523757935, "learning_rate": 9.644850953105288e-05, "loss": 0.46752145886421204, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 35.88, "memory/max_allocated (GiB)": 35.88, "ppl": 1.59603, "step": 88, "tokens/total": 3037968, "tokens/train_per_sec_per_gpu": 68.36, "tokens/trainable": 91976} +{"epoch": 0.34765625, "grad_norm": 1.8865758180618286, "learning_rate": 9.633461496214225e-05, "loss": 0.40061894059181213, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.49275, "step": 89, "tokens/total": 3070608, "tokens/train_per_sec_per_gpu": 62.5, "tokens/trainable": 93033} +{"epoch": 0.3515625, "grad_norm": 4.0921173095703125, "learning_rate": 9.621900029118195e-05, "loss": 0.4369269609451294, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.1, "memory/max_allocated (GiB)": 36.1, "ppl": 1.54794, "step": 90, "tokens/total": 3100272, "tokens/train_per_sec_per_gpu": 68.3, "tokens/trainable": 94067} +{"epoch": 0.35546875, "grad_norm": 1.7054483890533447, "learning_rate": 9.610167032937036e-05, "loss": 0.4925427734851837, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.63647, "step": 91, "tokens/total": 3138480, "tokens/train_per_sec_per_gpu": 54.56, "tokens/trainable": 95120} +{"epoch": 0.359375, "grad_norm": 3.0323426723480225, "learning_rate": 9.598262995928611e-05, "loss": 0.41542333364486694, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.51501, "step": 92, "tokens/total": 3169648, "tokens/train_per_sec_per_gpu": 70.93, "tokens/trainable": 96187} +{"epoch": 0.36328125, "grad_norm": 1.5211843252182007, "learning_rate": 9.586188413468492e-05, "loss": 0.47573190927505493, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.8, "memory/max_allocated (GiB)": 38.8, "ppl": 1.60919, "step": 93, "tokens/total": 3208032, "tokens/train_per_sec_per_gpu": 49.66, "tokens/trainable": 97240} +{"epoch": 0.3671875, "grad_norm": 2.3846237659454346, "learning_rate": 9.57394378802934e-05, "loss": 0.43694302439689636, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.54797, "step": 94, "tokens/total": 3241472, "tokens/train_per_sec_per_gpu": 60.97, "tokens/trainable": 98286} +{"epoch": 0.37109375, "grad_norm": 1.4062178134918213, "learning_rate": 9.56152962916e-05, "loss": 0.4492632746696472, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.56716, "step": 95, "tokens/total": 3272256, "tokens/train_per_sec_per_gpu": 66.3, "tokens/trainable": 99269} +{"epoch": 0.375, "grad_norm": 2.114784002304077, "learning_rate": 9.548946453464296e-05, "loss": 0.36324870586395264, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 1.43799, "step": 96, "tokens/total": 3306592, "tokens/train_per_sec_per_gpu": 62.93, "tokens/trainable": 100374} +{"epoch": 0.37890625, "grad_norm": 1.5093028545379639, "learning_rate": 9.53619478457953e-05, "loss": 0.4753195643424988, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.60853, "step": 97, "tokens/total": 3341264, "tokens/train_per_sec_per_gpu": 52.69, "tokens/trainable": 101366} +{"epoch": 0.3828125, "grad_norm": 1.4754374027252197, "learning_rate": 9.523275153154695e-05, "loss": 0.47023534774780273, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.60037, "step": 98, "tokens/total": 3373776, "tokens/train_per_sec_per_gpu": 58.08, "tokens/trainable": 102326} +{"epoch": 0.38671875, "grad_norm": 1.0977492332458496, "learning_rate": 9.51018809682839e-05, "loss": 0.4162147641181946, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.51621, "step": 99, "tokens/total": 3411600, "tokens/train_per_sec_per_gpu": 53.14, "tokens/trainable": 103343} +{"epoch": 0.390625, "grad_norm": 2.826719045639038, "learning_rate": 9.49693416020645e-05, "loss": 0.44720205664634705, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.51, "memory/max_allocated (GiB)": 38.51, "ppl": 1.56393, "step": 100, "tokens/total": 3449392, "tokens/train_per_sec_per_gpu": 55.06, "tokens/trainable": 104363} +{"epoch": 0.39453125, "grad_norm": 3.5070793628692627, "learning_rate": 9.483513894839276e-05, "loss": 0.4538375735282898, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.57434, "step": 101, "tokens/total": 3482832, "tokens/train_per_sec_per_gpu": 68.27, "tokens/trainable": 105466} +{"epoch": 0.3984375, "grad_norm": 2.5610592365264893, "learning_rate": 9.469927859198888e-05, "loss": 0.4361674189567566, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.54677, "step": 102, "tokens/total": 3518768, "tokens/train_per_sec_per_gpu": 58.8, "tokens/trainable": 106466} +{"epoch": 0.40234375, "grad_norm": 1.3781081438064575, "learning_rate": 9.456176618655689e-05, "loss": 0.4123714864253998, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.51, "memory/max_allocated (GiB)": 36.51, "ppl": 1.5104, "step": 103, "tokens/total": 3550736, "tokens/train_per_sec_per_gpu": 60.68, "tokens/trainable": 107462} +{"epoch": 0.40625, "grad_norm": 1.3555225133895874, "learning_rate": 9.442260745454927e-05, "loss": 0.42837953567504883, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.53477, "step": 104, "tokens/total": 3585920, "tokens/train_per_sec_per_gpu": 59.31, "tokens/trainable": 108533} +{"epoch": 0.41015625, "grad_norm": 1.743802547454834, "learning_rate": 9.428180818692884e-05, "loss": 0.36697202920913696, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.44336, "step": 105, "tokens/total": 3622160, "tokens/train_per_sec_per_gpu": 52.68, "tokens/trainable": 109593} +{"epoch": 0.4140625, "grad_norm": 2.7440319061279297, "learning_rate": 9.413937424292791e-05, "loss": 0.4360317289829254, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.54656, "step": 106, "tokens/total": 3661088, "tokens/train_per_sec_per_gpu": 48.13, "tokens/trainable": 110589} +{"epoch": 0.41796875, "grad_norm": 2.475125551223755, "learning_rate": 9.399531154980424e-05, "loss": 0.37778788805007935, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.45905, "step": 107, "tokens/total": 3697888, "tokens/train_per_sec_per_gpu": 51.72, "tokens/trainable": 111594} +{"epoch": 0.421875, "grad_norm": 6.873198509216309, "learning_rate": 9.384962610259455e-05, "loss": 0.42826563119888306, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.53459, "step": 108, "tokens/total": 3732400, "tokens/train_per_sec_per_gpu": 54.96, "tokens/trainable": 112587} +{"epoch": 0.42578125, "grad_norm": 1.3191946744918823, "learning_rate": 9.370232396386494e-05, "loss": 0.4187881052494049, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.52012, "step": 109, "tokens/total": 3765616, "tokens/train_per_sec_per_gpu": 60.92, "tokens/trainable": 113611} +{"epoch": 0.4296875, "grad_norm": 1.0572936534881592, "learning_rate": 9.355341126345868e-05, "loss": 0.344032347202301, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.41062, "step": 110, "tokens/total": 3801104, "tokens/train_per_sec_per_gpu": 61.31, "tokens/trainable": 114710} +{"epoch": 0.43359375, "grad_norm": 2.316951274871826, "learning_rate": 9.340289419824107e-05, "loss": 0.3761659264564514, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.74, "memory/max_allocated (GiB)": 36.74, "ppl": 1.45669, "step": 111, "tokens/total": 3833296, "tokens/train_per_sec_per_gpu": 69.22, "tokens/trainable": 115741} +{"epoch": 0.4375, "grad_norm": 1.6291502714157104, "learning_rate": 9.325077903184159e-05, "loss": 0.4180870056152344, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.51905, "step": 112, "tokens/total": 3868144, "tokens/train_per_sec_per_gpu": 57.78, "tokens/trainable": 116832} +{"epoch": 0.44140625, "grad_norm": 1.0775765180587769, "learning_rate": 9.30970720943932e-05, "loss": 0.3318844437599182, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.43, "memory/max_allocated (GiB)": 38.43, "ppl": 1.39359, "step": 113, "tokens/total": 3903568, "tokens/train_per_sec_per_gpu": 57.7, "tokens/trainable": 117912} +{"epoch": 0.4453125, "grad_norm": 1.3730978965759277, "learning_rate": 9.2941779782269e-05, "loss": 0.3910774290561676, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.47857, "step": 114, "tokens/total": 3937328, "tokens/train_per_sec_per_gpu": 61.33, "tokens/trainable": 118935} +{"epoch": 0.44921875, "grad_norm": 1.3145430088043213, "learning_rate": 9.278490855781596e-05, "loss": 0.35860365629196167, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.43133, "step": 115, "tokens/total": 3969936, "tokens/train_per_sec_per_gpu": 70.07, "tokens/trainable": 119972} +{"epoch": 0.453125, "grad_norm": 1.3409278392791748, "learning_rate": 9.262646494908604e-05, "loss": 0.41241219639778137, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 39.14, "memory/max_allocated (GiB)": 39.14, "ppl": 1.51046, "step": 116, "tokens/total": 4008512, "tokens/train_per_sec_per_gpu": 50.74, "tokens/trainable": 120969} +{"epoch": 0.45703125, "grad_norm": 1.613276720046997, "learning_rate": 9.246645554956457e-05, "loss": 0.3962288200855255, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.48621, "step": 117, "tokens/total": 4043104, "tokens/train_per_sec_per_gpu": 61.82, "tokens/trainable": 122017} +{"epoch": 0.4609375, "grad_norm": 1.3734842538833618, "learning_rate": 9.230488701789578e-05, "loss": 0.431545227766037, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.7, "memory/max_allocated (GiB)": 38.7, "ppl": 1.53963, "step": 118, "tokens/total": 4080624, "tokens/train_per_sec_per_gpu": 55.65, "tokens/trainable": 123098} +{"epoch": 0.46484375, "grad_norm": 1.6299771070480347, "learning_rate": 9.214176607760577e-05, "loss": 0.35051196813583374, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.41979, "step": 119, "tokens/total": 4119296, "tokens/train_per_sec_per_gpu": 58.55, "tokens/trainable": 124229} +{"epoch": 0.46875, "grad_norm": 2.7166552543640137, "learning_rate": 9.197709951682268e-05, "loss": 0.36214184761047363, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.4364, "step": 120, "tokens/total": 4151584, "tokens/train_per_sec_per_gpu": 65.06, "tokens/trainable": 125294} +{"epoch": 0.47265625, "grad_norm": 1.3986527919769287, "learning_rate": 9.181089418799428e-05, "loss": 0.39809760451316833, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 39.39, "memory/max_allocated (GiB)": 39.39, "ppl": 1.48899, "step": 121, "tokens/total": 4188384, "tokens/train_per_sec_per_gpu": 55.1, "tokens/trainable": 126333} +{"epoch": 0.4765625, "grad_norm": 1.0485347509384155, "learning_rate": 9.164315700760271e-05, "loss": 0.3277769684791565, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.38788, "step": 122, "tokens/total": 4224192, "tokens/train_per_sec_per_gpu": 58.5, "tokens/trainable": 127377} +{"epoch": 0.48046875, "grad_norm": 4.843174934387207, "learning_rate": 9.147389495587671e-05, "loss": 0.4271337389945984, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 1.53286, "step": 123, "tokens/total": 4255344, "tokens/train_per_sec_per_gpu": 69.21, "tokens/trainable": 128395} +{"epoch": 0.484375, "grad_norm": 1.2090739011764526, "learning_rate": 9.130311507650116e-05, "loss": 0.3263755738735199, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 1.38594, "step": 124, "tokens/total": 4288016, "tokens/train_per_sec_per_gpu": 61.72, "tokens/trainable": 129450} +{"epoch": 0.48828125, "grad_norm": 1.2642533779144287, "learning_rate": 9.113082447632394e-05, "loss": 0.36375918984413147, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.43873, "step": 125, "tokens/total": 4320160, "tokens/train_per_sec_per_gpu": 79.7, "tokens/trainable": 130600} +{"epoch": 0.4921875, "grad_norm": 1.636986255645752, "learning_rate": 9.09570303250602e-05, "loss": 0.392950177192688, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.47, "memory/max_allocated (GiB)": 36.47, "ppl": 1.48134, "step": 126, "tokens/total": 4351968, "tokens/train_per_sec_per_gpu": 63.42, "tokens/trainable": 131663} +{"epoch": 0.49609375, "grad_norm": 1.31266188621521, "learning_rate": 9.078173985499394e-05, "loss": 0.363090842962265, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 35.79, "memory/max_allocated (GiB)": 35.79, "ppl": 1.43777, "step": 127, "tokens/total": 4381952, "tokens/train_per_sec_per_gpu": 65.65, "tokens/trainable": 132667} +{"epoch": 0.5, "grad_norm": 1.639731764793396, "learning_rate": 9.060496036067713e-05, "loss": 0.37035828828811646, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.72, "memory/max_allocated (GiB)": 38.72, "ppl": 1.44825, "step": 128, "tokens/total": 4419376, "tokens/train_per_sec_per_gpu": 48.16, "tokens/trainable": 133681} +{"epoch": 0.50390625, "grad_norm": 1.2158758640289307, "learning_rate": 9.042669919862615e-05, "loss": 0.39443737268447876, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.82, "memory/max_allocated (GiB)": 36.82, "ppl": 1.48355, "step": 129, "tokens/total": 4451488, "tokens/train_per_sec_per_gpu": 67.65, "tokens/trainable": 134729} +{"epoch": 0.5078125, "grad_norm": 1.3464369773864746, "learning_rate": 9.024696378701557e-05, "loss": 0.3406529426574707, "memory/device_reserved (GiB)": 41.41, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.40587, "step": 130, "tokens/total": 4487040, "tokens/train_per_sec_per_gpu": 58.54, "tokens/trainable": 135783} +{"epoch": 0.51171875, "grad_norm": 1.4248942136764526, "learning_rate": 9.006576160536948e-05, "loss": 0.3515858054161072, "memory/device_reserved (GiB)": 41.41, "memory/max_active (GiB)": 37.43, "memory/max_allocated (GiB)": 37.43, "ppl": 1.42132, "step": 131, "tokens/total": 4521888, "tokens/train_per_sec_per_gpu": 55.66, "tokens/trainable": 136820} +{"epoch": 0.515625, "grad_norm": 1.0995441675186157, "learning_rate": 8.988310019425035e-05, "loss": 0.3730773627758026, "memory/device_reserved (GiB)": 41.71, "memory/max_active (GiB)": 39.7, "memory/max_allocated (GiB)": 39.7, "ppl": 1.4522, "step": 132, "tokens/total": 4559744, "tokens/train_per_sec_per_gpu": 51.52, "tokens/trainable": 137830} +{"epoch": 0.51953125, "grad_norm": 1.641839623451233, "learning_rate": 8.969898715494506e-05, "loss": 0.3528146743774414, "memory/device_reserved (GiB)": 41.73, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.42307, "step": 133, "tokens/total": 4596576, "tokens/train_per_sec_per_gpu": 61.08, "tokens/trainable": 138920} +{"epoch": 0.5234375, "grad_norm": 1.2008275985717773, "learning_rate": 8.951343014914869e-05, "loss": 0.32968246936798096, "memory/device_reserved (GiB)": 41.73, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.39053, "step": 134, "tokens/total": 4633232, "tokens/train_per_sec_per_gpu": 54.69, "tokens/trainable": 139988} +{"epoch": 0.52734375, "grad_norm": 1.1136828660964966, "learning_rate": 8.932643689864568e-05, "loss": 0.3323252201080322, "memory/device_reserved (GiB)": 41.73, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.39421, "step": 135, "tokens/total": 4669808, "tokens/train_per_sec_per_gpu": 62.14, "tokens/trainable": 141105} +{"epoch": 0.53125, "grad_norm": 1.0874773263931274, "learning_rate": 8.913801518498845e-05, "loss": 0.29834944009780884, "memory/device_reserved (GiB)": 41.73, "memory/max_active (GiB)": 39.68, "memory/max_allocated (GiB)": 39.68, "ppl": 1.34763, "step": 136, "tokens/total": 4709184, "tokens/train_per_sec_per_gpu": 48.54, "tokens/trainable": 142217} +{"epoch": 0.53515625, "grad_norm": 2.4157159328460693, "learning_rate": 8.894817284917364e-05, "loss": 0.3548920154571533, "memory/device_reserved (GiB)": 41.73, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.42603, "step": 137, "tokens/total": 4746128, "tokens/train_per_sec_per_gpu": 50.24, "tokens/trainable": 143251} +{"epoch": 0.5390625, "grad_norm": 1.2599833011627197, "learning_rate": 8.875691779131569e-05, "loss": 0.3186655640602112, "memory/device_reserved (GiB)": 41.9, "memory/max_active (GiB)": 39.78, "memory/max_allocated (GiB)": 39.78, "ppl": 1.37529, "step": 138, "tokens/total": 4784064, "tokens/train_per_sec_per_gpu": 55.04, "tokens/trainable": 144323} +{"epoch": 0.54296875, "grad_norm": 1.1540032625198364, "learning_rate": 8.856425797031829e-05, "loss": 0.3040095269680023, "memory/device_reserved (GiB)": 42.22, "memory/max_active (GiB)": 39.58, "memory/max_allocated (GiB)": 39.58, "ppl": 1.35528, "step": 139, "tokens/total": 4820096, "tokens/train_per_sec_per_gpu": 63.71, "tokens/trainable": 145451} +{"epoch": 0.546875, "grad_norm": 1.6823762655258179, "learning_rate": 8.837020140354295e-05, "loss": 0.3334498107433319, "memory/device_reserved (GiB)": 42.22, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.39577, "step": 140, "tokens/total": 4856912, "tokens/train_per_sec_per_gpu": 55.33, "tokens/trainable": 146458} +{"epoch": 0.55078125, "grad_norm": 1.2039827108383179, "learning_rate": 8.817475616647554e-05, "loss": 0.3289852738380432, "memory/device_reserved (GiB)": 42.22, "memory/max_active (GiB)": 38.17, "memory/max_allocated (GiB)": 38.17, "ppl": 1.38956, "step": 141, "tokens/total": 4891344, "tokens/train_per_sec_per_gpu": 60.08, "tokens/trainable": 147512} +{"epoch": 0.5546875, "grad_norm": 1.2273547649383545, "learning_rate": 8.797793039239017e-05, "loss": 0.3155956268310547, "memory/device_reserved (GiB)": 42.22, "memory/max_active (GiB)": 37.83, "memory/max_allocated (GiB)": 37.83, "ppl": 1.37108, "step": 142, "tokens/total": 4926368, "tokens/train_per_sec_per_gpu": 57.97, "tokens/trainable": 148558} +{"epoch": 0.55859375, "grad_norm": 1.7695115804672241, "learning_rate": 8.777973227201069e-05, "loss": 0.3218223452568054, "memory/device_reserved (GiB)": 43.76, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.37964, "step": 143, "tokens/total": 4963280, "tokens/train_per_sec_per_gpu": 60.41, "tokens/trainable": 149621} +{"epoch": 0.5625, "grad_norm": 1.3462203741073608, "learning_rate": 8.758017005316988e-05, "loss": 0.3506096303462982, "memory/device_reserved (GiB)": 43.76, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.41993, "step": 144, "tokens/total": 4998752, "tokens/train_per_sec_per_gpu": 68.0, "tokens/trainable": 150674} +{"epoch": 0.56640625, "grad_norm": 1.3528631925582886, "learning_rate": 8.737925204046629e-05, "loss": 0.3549090027809143, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.42605, "step": 145, "tokens/total": 5033472, "tokens/train_per_sec_per_gpu": 54.65, "tokens/trainable": 151735} +{"epoch": 0.5703125, "grad_norm": 1.1019337177276611, "learning_rate": 8.717698659491851e-05, "loss": 0.31365954875946045, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 38.91, "memory/max_allocated (GiB)": 38.91, "ppl": 1.36842, "step": 146, "tokens/total": 5070304, "tokens/train_per_sec_per_gpu": 57.04, "tokens/trainable": 152839} +{"epoch": 0.57421875, "grad_norm": 3.2619237899780273, "learning_rate": 8.697338213361735e-05, "loss": 0.33034804463386536, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.39145, "step": 147, "tokens/total": 5106208, "tokens/train_per_sec_per_gpu": 58.5, "tokens/trainable": 153931} +{"epoch": 0.578125, "grad_norm": 1.0445483922958374, "learning_rate": 8.676844712937552e-05, "loss": 0.2984568476676941, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.34778, "step": 148, "tokens/total": 5146208, "tokens/train_per_sec_per_gpu": 49.21, "tokens/trainable": 154983} +{"epoch": 0.58203125, "grad_norm": 1.455915093421936, "learning_rate": 8.656219011037509e-05, "loss": 0.3071402907371521, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.35953, "step": 149, "tokens/total": 5183504, "tokens/train_per_sec_per_gpu": 52.99, "tokens/trainable": 156041} +{"epoch": 0.5859375, "grad_norm": 1.284110188484192, "learning_rate": 8.63546196598125e-05, "loss": 0.3041020929813385, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.35541, "step": 150, "tokens/total": 5221088, "tokens/train_per_sec_per_gpu": 55.79, "tokens/trainable": 157163} +{"epoch": 0.58984375, "grad_norm": 1.9269989728927612, "learning_rate": 8.614574441554145e-05, "loss": 0.41391685605049133, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 36.95, "memory/max_allocated (GiB)": 36.95, "ppl": 1.51273, "step": 151, "tokens/total": 5253712, "tokens/train_per_sec_per_gpu": 62.2, "tokens/trainable": 158127} +{"epoch": 0.59375, "grad_norm": 1.5949757099151611, "learning_rate": 8.593557306971349e-05, "loss": 0.3230413794517517, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.38132, "step": 152, "tokens/total": 5287024, "tokens/train_per_sec_per_gpu": 58.56, "tokens/trainable": 159160} +{"epoch": 0.59765625, "grad_norm": 1.3524137735366821, "learning_rate": 8.572411436841618e-05, "loss": 0.3496304750442505, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 36.9, "memory/max_allocated (GiB)": 36.9, "ppl": 1.41854, "step": 153, "tokens/total": 5320768, "tokens/train_per_sec_per_gpu": 64.39, "tokens/trainable": 160230} +{"epoch": 0.6015625, "grad_norm": 1.5171704292297363, "learning_rate": 8.551137711130922e-05, "loss": 0.32572072744369507, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.38503, "step": 154, "tokens/total": 5352544, "tokens/train_per_sec_per_gpu": 59.21, "tokens/trainable": 161248} +{"epoch": 0.60546875, "grad_norm": 1.1411528587341309, "learning_rate": 8.529737015125824e-05, "loss": 0.28282156586647034, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 38.17, "memory/max_allocated (GiB)": 38.17, "ppl": 1.32687, "step": 155, "tokens/total": 5387648, "tokens/train_per_sec_per_gpu": 56.72, "tokens/trainable": 162267} +{"epoch": 0.609375, "grad_norm": 1.533816933631897, "learning_rate": 8.508210239396639e-05, "loss": 0.3576628565788269, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 39.77, "memory/max_allocated (GiB)": 39.77, "ppl": 1.42998, "step": 156, "tokens/total": 5425536, "tokens/train_per_sec_per_gpu": 57.22, "tokens/trainable": 163284} +{"epoch": 0.61328125, "grad_norm": 1.6187453269958496, "learning_rate": 8.486558279760375e-05, "loss": 0.34012576937675476, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 35.9, "memory/max_allocated (GiB)": 35.9, "ppl": 1.40512, "step": 157, "tokens/total": 5456592, "tokens/train_per_sec_per_gpu": 64.1, "tokens/trainable": 164291} +{"epoch": 0.6171875, "grad_norm": 1.4032236337661743, "learning_rate": 8.464782037243449e-05, "loss": 0.35356348752975464, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.42413, "step": 158, "tokens/total": 5488736, "tokens/train_per_sec_per_gpu": 60.69, "tokens/trainable": 165280} +{"epoch": 0.62109375, "grad_norm": 1.2644115686416626, "learning_rate": 8.442882418044202e-05, "loss": 0.31445977091789246, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.36952, "step": 159, "tokens/total": 5521376, "tokens/train_per_sec_per_gpu": 63.65, "tokens/trainable": 166320} +{"epoch": 0.625, "grad_norm": 5.4669575691223145, "learning_rate": 8.420860333495179e-05, "loss": 0.29010871052742004, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.33657, "step": 160, "tokens/total": 5555184, "tokens/train_per_sec_per_gpu": 61.4, "tokens/trainable": 167340} +{"epoch": 0.62890625, "grad_norm": 1.1955139636993408, "learning_rate": 8.398716700025208e-05, "loss": 0.31414735317230225, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.36909, "step": 161, "tokens/total": 5588704, "tokens/train_per_sec_per_gpu": 59.43, "tokens/trainable": 168363} +{"epoch": 0.6328125, "grad_norm": 1.3950154781341553, "learning_rate": 8.376452439121266e-05, "loss": 0.3245818316936493, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.05, "memory/max_allocated (GiB)": 36.05, "ppl": 1.38345, "step": 162, "tokens/total": 5619424, "tokens/train_per_sec_per_gpu": 66.3, "tokens/trainable": 169371} +{"epoch": 0.63671875, "grad_norm": 2.622976064682007, "learning_rate": 8.354068477290124e-05, "loss": 0.3344261944293976, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.39714, "step": 163, "tokens/total": 5653648, "tokens/train_per_sec_per_gpu": 61.24, "tokens/trainable": 170358} +{"epoch": 0.640625, "grad_norm": 2.9200680255889893, "learning_rate": 8.331565746019807e-05, "loss": 0.3169896602630615, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.37299, "step": 164, "tokens/total": 5685840, "tokens/train_per_sec_per_gpu": 68.84, "tokens/trainable": 171390} +{"epoch": 0.64453125, "grad_norm": 1.3750635385513306, "learning_rate": 8.308945181740812e-05, "loss": 0.31691646575927734, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.37289, "step": 165, "tokens/total": 5720608, "tokens/train_per_sec_per_gpu": 61.72, "tokens/trainable": 172412} +{"epoch": 0.6484375, "grad_norm": 1.3730828762054443, "learning_rate": 8.286207725787153e-05, "loss": 0.2944674789905548, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.34241, "step": 166, "tokens/total": 5757616, "tokens/train_per_sec_per_gpu": 51.75, "tokens/trainable": 173439} +{"epoch": 0.65234375, "grad_norm": 1.3004109859466553, "learning_rate": 8.263354324357182e-05, "loss": 0.2911866307258606, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.12, "memory/max_allocated (GiB)": 39.12, "ppl": 1.33801, "step": 167, "tokens/total": 5792352, "tokens/train_per_sec_per_gpu": 59.0, "tokens/trainable": 174467} +{"epoch": 0.65625, "grad_norm": 2.3852391242980957, "learning_rate": 8.240385928474219e-05, "loss": 0.3540372848510742, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.42481, "step": 168, "tokens/total": 5825168, "tokens/train_per_sec_per_gpu": 57.96, "tokens/trainable": 175427} +{"epoch": 0.66015625, "grad_norm": 1.1315102577209473, "learning_rate": 8.217303493946967e-05, "loss": 0.3007480502128601, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.35087, "step": 169, "tokens/total": 5859376, "tokens/train_per_sec_per_gpu": 57.91, "tokens/trainable": 176429} +{"epoch": 0.6640625, "grad_norm": 1.1103121042251587, "learning_rate": 8.194107981329746e-05, "loss": 0.2910248041152954, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.73, "memory/max_allocated (GiB)": 39.73, "ppl": 1.3378, "step": 170, "tokens/total": 5897952, "tokens/train_per_sec_per_gpu": 51.62, "tokens/trainable": 177450} +{"epoch": 0.66796875, "grad_norm": 1.6373438835144043, "learning_rate": 8.170800355882518e-05, "loss": 0.2760692536830902, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.31794, "step": 171, "tokens/total": 5932384, "tokens/train_per_sec_per_gpu": 66.29, "tokens/trainable": 178489} +{"epoch": 0.671875, "grad_norm": 1.7011133432388306, "learning_rate": 8.147381587530713e-05, "loss": 0.33447906374931335, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.39, "memory/max_allocated (GiB)": 37.39, "ppl": 1.39721, "step": 172, "tokens/total": 5966176, "tokens/train_per_sec_per_gpu": 61.76, "tokens/trainable": 179483} +{"epoch": 0.67578125, "grad_norm": 2.7926671504974365, "learning_rate": 8.123852650824877e-05, "loss": 0.31798338890075684, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.45, "memory/max_allocated (GiB)": 37.45, "ppl": 1.37435, "step": 173, "tokens/total": 6000000, "tokens/train_per_sec_per_gpu": 61.81, "tokens/trainable": 180545} +{"epoch": 0.6796875, "grad_norm": 1.266586184501648, "learning_rate": 8.100214524900103e-05, "loss": 0.2976175844669342, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.34665, "step": 174, "tokens/total": 6035072, "tokens/train_per_sec_per_gpu": 52.02, "tokens/trainable": 181549} +{"epoch": 0.68359375, "grad_norm": 1.4640041589736938, "learning_rate": 8.076468193435301e-05, "loss": 0.3402634561061859, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.40532, "step": 175, "tokens/total": 6065920, "tokens/train_per_sec_per_gpu": 66.62, "tokens/trainable": 182497} +{"epoch": 0.6875, "grad_norm": 1.277320146560669, "learning_rate": 8.052614644612253e-05, "loss": 0.2997749149799347, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.34956, "step": 176, "tokens/total": 6103760, "tokens/train_per_sec_per_gpu": 49.91, "tokens/trainable": 183547} +{"epoch": 0.69140625, "grad_norm": 1.476240873336792, "learning_rate": 8.028654871074489e-05, "loss": 0.2681358754634857, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.30752, "step": 177, "tokens/total": 6138352, "tokens/train_per_sec_per_gpu": 59.02, "tokens/trainable": 184633} +{"epoch": 0.6953125, "grad_norm": 1.3131940364837646, "learning_rate": 8.004589869885986e-05, "loss": 0.28853335976600647, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.37, "memory/max_allocated (GiB)": 36.37, "ppl": 1.33447, "step": 178, "tokens/total": 6169824, "tokens/train_per_sec_per_gpu": 61.0, "tokens/trainable": 185591} +{"epoch": 0.69921875, "grad_norm": 1.5363247394561768, "learning_rate": 7.980420642489674e-05, "loss": 0.2822202444076538, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.32607, "step": 179, "tokens/total": 6208528, "tokens/train_per_sec_per_gpu": 53.68, "tokens/trainable": 186623} +{"epoch": 0.703125, "grad_norm": 1.2461363077163696, "learning_rate": 7.95614819466576e-05, "loss": 0.2826019525527954, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 1.32658, "step": 180, "tokens/total": 6242240, "tokens/train_per_sec_per_gpu": 58.86, "tokens/trainable": 187674} +{"epoch": 0.70703125, "grad_norm": 1.0273996591567993, "learning_rate": 7.931773536489872e-05, "loss": 0.2887662351131439, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.33478, "step": 181, "tokens/total": 6276832, "tokens/train_per_sec_per_gpu": 62.61, "tokens/trainable": 188721} +{"epoch": 0.7109375, "grad_norm": 1.42617928981781, "learning_rate": 7.907297682291035e-05, "loss": 0.2840316891670227, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.32848, "step": 182, "tokens/total": 6311488, "tokens/train_per_sec_per_gpu": 60.81, "tokens/trainable": 189799} +{"epoch": 0.71484375, "grad_norm": 0.9417517185211182, "learning_rate": 7.882721650609442e-05, "loss": 0.2306438535451889, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.65, "memory/max_allocated (GiB)": 39.65, "ppl": 1.25941, "step": 183, "tokens/total": 6351904, "tokens/train_per_sec_per_gpu": 53.4, "tokens/trainable": 190977} +{"epoch": 0.71875, "grad_norm": 1.283276081085205, "learning_rate": 7.85804646415409e-05, "loss": 0.29201656579971313, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.33913, "step": 184, "tokens/total": 6387136, "tokens/train_per_sec_per_gpu": 64.18, "tokens/trainable": 192063} +{"epoch": 0.72265625, "grad_norm": 1.4925682544708252, "learning_rate": 7.833273149760207e-05, "loss": 0.28161296248435974, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.32527, "step": 185, "tokens/total": 6418000, "tokens/train_per_sec_per_gpu": 60.32, "tokens/trainable": 193068} +{"epoch": 0.7265625, "grad_norm": 1.4212459325790405, "learning_rate": 7.808402738346527e-05, "loss": 0.2535053491592407, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.11, "memory/max_allocated (GiB)": 39.11, "ppl": 1.28853, "step": 186, "tokens/total": 6456784, "tokens/train_per_sec_per_gpu": 58.19, "tokens/trainable": 194213} +{"epoch": 0.73046875, "grad_norm": 1.6086244583129883, "learning_rate": 7.783436264872382e-05, "loss": 0.2673630118370056, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.30651, "step": 187, "tokens/total": 6493632, "tokens/train_per_sec_per_gpu": 56.09, "tokens/trainable": 195262} +{"epoch": 0.734375, "grad_norm": 1.2026753425598145, "learning_rate": 7.758374768294647e-05, "loss": 0.24924030900001526, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.16, "memory/max_allocated (GiB)": 38.16, "ppl": 1.28305, "step": 188, "tokens/total": 6529568, "tokens/train_per_sec_per_gpu": 60.85, "tokens/trainable": 196357} +{"epoch": 0.73828125, "grad_norm": 1.314247727394104, "learning_rate": 7.733219291524489e-05, "loss": 0.2557780146598816, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.53, "memory/max_allocated (GiB)": 37.53, "ppl": 1.29147, "step": 189, "tokens/total": 6562992, "tokens/train_per_sec_per_gpu": 62.39, "tokens/trainable": 197394} +{"epoch": 0.7421875, "grad_norm": 1.2759861946105957, "learning_rate": 7.707970881383977e-05, "loss": 0.27000391483306885, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.45, "memory/max_allocated (GiB)": 36.45, "ppl": 1.30997, "step": 190, "tokens/total": 6594704, "tokens/train_per_sec_per_gpu": 69.35, "tokens/trainable": 198431} +{"epoch": 0.74609375, "grad_norm": 1.8880434036254883, "learning_rate": 7.682630588562518e-05, "loss": 0.28685513138771057, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.7, "memory/max_allocated (GiB)": 38.7, "ppl": 1.33223, "step": 191, "tokens/total": 6629744, "tokens/train_per_sec_per_gpu": 57.58, "tokens/trainable": 199492} +{"epoch": 0.75, "grad_norm": 1.5115621089935303, "learning_rate": 7.657199467573129e-05, "loss": 0.26577985286712646, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.30445, "step": 192, "tokens/total": 6663872, "tokens/train_per_sec_per_gpu": 56.72, "tokens/trainable": 200512} +{"epoch": 0.75390625, "grad_norm": 1.4075888395309448, "learning_rate": 7.631678576708561e-05, "loss": 0.2255035787820816, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.16, "memory/max_allocated (GiB)": 38.16, "ppl": 1.25295, "step": 193, "tokens/total": 6698272, "tokens/train_per_sec_per_gpu": 67.37, "tokens/trainable": 201606} +{"epoch": 0.7578125, "grad_norm": 1.2308130264282227, "learning_rate": 7.606068977997255e-05, "loss": 0.2559989094734192, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.64, "memory/max_allocated (GiB)": 39.64, "ppl": 1.29175, "step": 194, "tokens/total": 6735872, "tokens/train_per_sec_per_gpu": 66.18, "tokens/trainable": 202772} +{"epoch": 0.76171875, "grad_norm": 1.7816879749298096, "learning_rate": 7.580371737159148e-05, "loss": 0.286984384059906, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.3324, "step": 195, "tokens/total": 6771968, "tokens/train_per_sec_per_gpu": 57.46, "tokens/trainable": 203825} +{"epoch": 0.765625, "grad_norm": 1.2115838527679443, "learning_rate": 7.554587923561324e-05, "loss": 0.27186790108680725, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.99, "memory/max_allocated (GiB)": 38.99, "ppl": 1.31241, "step": 196, "tokens/total": 6806672, "tokens/train_per_sec_per_gpu": 51.76, "tokens/trainable": 204797} +{"epoch": 0.76953125, "grad_norm": 1.1146742105484009, "learning_rate": 7.528718610173511e-05, "loss": 0.24535292387008667, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.27807, "step": 197, "tokens/total": 6842016, "tokens/train_per_sec_per_gpu": 63.19, "tokens/trainable": 205891} +{"epoch": 0.7734375, "grad_norm": 1.3336052894592285, "learning_rate": 7.502764873523431e-05, "loss": 0.26209479570388794, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.29965, "step": 198, "tokens/total": 6876064, "tokens/train_per_sec_per_gpu": 62.64, "tokens/trainable": 206929} +{"epoch": 0.77734375, "grad_norm": 1.1357243061065674, "learning_rate": 7.476727793652011e-05, "loss": 0.23669901490211487, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.72, "memory/max_allocated (GiB)": 36.72, "ppl": 1.26706, "step": 199, "tokens/total": 6909296, "tokens/train_per_sec_per_gpu": 62.61, "tokens/trainable": 207986} +{"epoch": 0.78125, "grad_norm": 1.5639573335647583, "learning_rate": 7.450608454068415e-05, "loss": 0.23401175439357758, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.26366, "step": 200, "tokens/total": 6943792, "tokens/train_per_sec_per_gpu": 57.84, "tokens/trainable": 209038} +{"epoch": 0.78515625, "grad_norm": 1.115696907043457, "learning_rate": 7.424407941704987e-05, "loss": 0.20718050003051758, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.49, "memory/max_allocated (GiB)": 38.49, "ppl": 1.2302, "step": 201, "tokens/total": 6980448, "tokens/train_per_sec_per_gpu": 67.2, "tokens/trainable": 210183} +{"epoch": 0.7890625, "grad_norm": 1.284413456916809, "learning_rate": 7.398127346871986e-05, "loss": 0.22407054901123047, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.61, "memory/max_allocated (GiB)": 39.61, "ppl": 1.25116, "step": 202, "tokens/total": 7017392, "tokens/train_per_sec_per_gpu": 54.96, "tokens/trainable": 211226} +{"epoch": 0.79296875, "grad_norm": 1.7155957221984863, "learning_rate": 7.371767763212238e-05, "loss": 0.25721633434295654, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.52, "memory/max_allocated (GiB)": 39.52, "ppl": 1.29332, "step": 203, "tokens/total": 7052768, "tokens/train_per_sec_per_gpu": 62.5, "tokens/trainable": 212277} +{"epoch": 0.796875, "grad_norm": 1.9162590503692627, "learning_rate": 7.345330287655617e-05, "loss": 0.2973453402519226, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.09, "memory/max_allocated (GiB)": 39.09, "ppl": 1.34628, "step": 204, "tokens/total": 7090864, "tokens/train_per_sec_per_gpu": 48.68, "tokens/trainable": 213291} +{"epoch": 0.80078125, "grad_norm": 2.182386875152588, "learning_rate": 7.31881602037339e-05, "loss": 0.2974364459514618, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.69, "memory/max_allocated (GiB)": 36.69, "ppl": 1.3464, "step": 205, "tokens/total": 7122912, "tokens/train_per_sec_per_gpu": 63.78, "tokens/trainable": 214314} +{"epoch": 0.8046875, "grad_norm": 1.4305636882781982, "learning_rate": 7.29222606473245e-05, "loss": 0.24709901213645935, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.28031, "step": 206, "tokens/total": 7155744, "tokens/train_per_sec_per_gpu": 61.38, "tokens/trainable": 215332} +{"epoch": 0.80859375, "grad_norm": 1.7163646221160889, "learning_rate": 7.265561527249383e-05, "loss": 0.2700766324996948, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.31006, "step": 207, "tokens/total": 7189744, "tokens/train_per_sec_per_gpu": 60.93, "tokens/trainable": 216320} +{"epoch": 0.8125, "grad_norm": 0.8633118867874146, "learning_rate": 7.238823517544436e-05, "loss": 0.19595076143741608, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.21647, "step": 208, "tokens/total": 7226896, "tokens/train_per_sec_per_gpu": 67.05, "tokens/trainable": 217486} +{"epoch": 0.81640625, "grad_norm": 1.1550326347351074, "learning_rate": 7.212013148295333e-05, "loss": 0.23928311467170715, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.27034, "step": 209, "tokens/total": 7260880, "tokens/train_per_sec_per_gpu": 63.22, "tokens/trainable": 218565} +{"epoch": 0.8203125, "grad_norm": 1.5792570114135742, "learning_rate": 7.185131535190975e-05, "loss": 0.24036353826522827, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.06, "memory/max_allocated (GiB)": 37.06, "ppl": 1.27171, "step": 210, "tokens/total": 7293472, "tokens/train_per_sec_per_gpu": 64.89, "tokens/trainable": 219646} +{"epoch": 0.82421875, "grad_norm": 0.8914903402328491, "learning_rate": 7.158179796885005e-05, "loss": 0.2322017252445221, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.26137, "step": 211, "tokens/total": 7328096, "tokens/train_per_sec_per_gpu": 62.12, "tokens/trainable": 220699} +{"epoch": 0.828125, "grad_norm": 1.3552589416503906, "learning_rate": 7.131159054949273e-05, "loss": 0.2672075927257538, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.30631, "step": 212, "tokens/total": 7366480, "tokens/train_per_sec_per_gpu": 54.76, "tokens/trainable": 221738} +{"epoch": 0.83203125, "grad_norm": 1.5044559240341187, "learning_rate": 7.104070433827139e-05, "loss": 0.268235981464386, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.30766, "step": 213, "tokens/total": 7401904, "tokens/train_per_sec_per_gpu": 50.14, "tokens/trainable": 222671} +{"epoch": 0.8359375, "grad_norm": 1.3885365724563599, "learning_rate": 7.076915060786705e-05, "loss": 0.23962178826332092, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.27077, "step": 214, "tokens/total": 7434720, "tokens/train_per_sec_per_gpu": 61.16, "tokens/trainable": 223705} +{"epoch": 0.83984375, "grad_norm": 1.4699625968933105, "learning_rate": 7.049694065873882e-05, "loss": 0.26189154386520386, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.07, "memory/max_allocated (GiB)": 38.07, "ppl": 1.29939, "step": 215, "tokens/total": 7469376, "tokens/train_per_sec_per_gpu": 56.01, "tokens/trainable": 224755} +{"epoch": 0.84375, "grad_norm": 1.395357608795166, "learning_rate": 7.022408581865382e-05, "loss": 0.23363223671913147, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.26318, "step": 216, "tokens/total": 7503520, "tokens/train_per_sec_per_gpu": 57.87, "tokens/trainable": 225832} +{"epoch": 0.84765625, "grad_norm": 3.155226230621338, "learning_rate": 6.99505974422157e-05, "loss": 0.22908633947372437, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.25745, "step": 217, "tokens/total": 7538000, "tokens/train_per_sec_per_gpu": 65.6, "tokens/trainable": 226913} +{"epoch": 0.8515625, "grad_norm": 1.0177003145217896, "learning_rate": 6.967648691039213e-05, "loss": 0.24109137058258057, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.58, "memory/max_allocated (GiB)": 36.58, "ppl": 1.27264, "step": 218, "tokens/total": 7569872, "tokens/train_per_sec_per_gpu": 63.21, "tokens/trainable": 227933} +{"epoch": 0.85546875, "grad_norm": 1.565076231956482, "learning_rate": 6.940176563004123e-05, "loss": 0.23825684189796448, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.26904, "step": 219, "tokens/total": 7603920, "tokens/train_per_sec_per_gpu": 57.48, "tokens/trainable": 228906} +{"epoch": 0.859375, "grad_norm": 1.3368173837661743, "learning_rate": 6.912644503343682e-05, "loss": 0.24709956347942352, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.28031, "step": 220, "tokens/total": 7642080, "tokens/train_per_sec_per_gpu": 55.58, "tokens/trainable": 229915} +{"epoch": 0.86328125, "grad_norm": 1.7907673120498657, "learning_rate": 6.885053657779273e-05, "loss": 0.24657022953033447, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.28, "memory/max_allocated (GiB)": 36.28, "ppl": 1.27963, "step": 221, "tokens/total": 7673392, "tokens/train_per_sec_per_gpu": 59.52, "tokens/trainable": 230927} +{"epoch": 0.8671875, "grad_norm": 3.069979190826416, "learning_rate": 6.857405174478604e-05, "loss": 0.255943238735199, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.29168, "step": 222, "tokens/total": 7708096, "tokens/train_per_sec_per_gpu": 57.92, "tokens/trainable": 231937} +{"epoch": 0.87109375, "grad_norm": 1.1526427268981934, "learning_rate": 6.82970020400792e-05, "loss": 0.23922261595726013, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.27026, "step": 223, "tokens/total": 7745312, "tokens/train_per_sec_per_gpu": 53.36, "tokens/trainable": 232962} +{"epoch": 0.875, "grad_norm": 1.4118552207946777, "learning_rate": 6.801939899284132e-05, "loss": 0.18993067741394043, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.20917, "step": 224, "tokens/total": 7781328, "tokens/train_per_sec_per_gpu": 58.51, "tokens/trainable": 234041} +{"epoch": 0.87890625, "grad_norm": 1.3022363185882568, "learning_rate": 6.774125415526827e-05, "loss": 0.20290303230285645, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.22495, "step": 225, "tokens/total": 7814704, "tokens/train_per_sec_per_gpu": 70.79, "tokens/trainable": 235179} +{"epoch": 0.8828125, "grad_norm": 1.4234851598739624, "learning_rate": 6.746257910210214e-05, "loss": 0.23764440417289734, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.26826, "step": 226, "tokens/total": 7847936, "tokens/train_per_sec_per_gpu": 60.53, "tokens/trainable": 236200} +{"epoch": 0.88671875, "grad_norm": 1.6405048370361328, "learning_rate": 6.718338543014937e-05, "loss": 0.2584913969039917, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 35.66, "memory/max_allocated (GiB)": 35.66, "ppl": 1.29498, "step": 227, "tokens/total": 7877936, "tokens/train_per_sec_per_gpu": 65.31, "tokens/trainable": 237182} +{"epoch": 0.890625, "grad_norm": 0.8536985516548157, "learning_rate": 6.69036847577983e-05, "loss": 0.19739097356796265, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.21822, "step": 228, "tokens/total": 7911840, "tokens/train_per_sec_per_gpu": 61.18, "tokens/trainable": 238211} +{"epoch": 0.89453125, "grad_norm": 2.622342109680176, "learning_rate": 6.662348872453553e-05, "loss": 0.1981697827577591, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.46, "memory/max_allocated (GiB)": 38.46, "ppl": 1.21917, "step": 229, "tokens/total": 7948400, "tokens/train_per_sec_per_gpu": 60.9, "tokens/trainable": 239279} +{"epoch": 0.8984375, "grad_norm": 1.2444096803665161, "learning_rate": 6.63428089904618e-05, "loss": 0.21063488721847534, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.23446, "step": 230, "tokens/total": 7987568, "tokens/train_per_sec_per_gpu": 53.26, "tokens/trainable": 240318} +{"epoch": 0.90234375, "grad_norm": 2.349888563156128, "learning_rate": 6.60616572358065e-05, "loss": 0.2186240404844284, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.24436, "step": 231, "tokens/total": 8022032, "tokens/train_per_sec_per_gpu": 61.29, "tokens/trainable": 241289} +{"epoch": 0.90625, "grad_norm": 1.807969093322754, "learning_rate": 6.578004516044172e-05, "loss": 0.24624451994895935, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.27921, "step": 232, "tokens/total": 8057888, "tokens/train_per_sec_per_gpu": 61.59, "tokens/trainable": 242322} +{"epoch": 0.91015625, "grad_norm": 1.7627203464508057, "learning_rate": 6.549798448339548e-05, "loss": 0.23045504093170166, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.6, "memory/max_allocated (GiB)": 38.6, "ppl": 1.25917, "step": 233, "tokens/total": 8095296, "tokens/train_per_sec_per_gpu": 60.31, "tokens/trainable": 243378} +{"epoch": 0.9140625, "grad_norm": 1.3932398557662964, "learning_rate": 6.521548694236384e-05, "loss": 0.21845775842666626, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.24416, "step": 234, "tokens/total": 8132608, "tokens/train_per_sec_per_gpu": 54.51, "tokens/trainable": 244427} +{"epoch": 0.91796875, "grad_norm": 1.1647371053695679, "learning_rate": 6.493256429322259e-05, "loss": 0.2354533076286316, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.26548, "step": 235, "tokens/total": 8168736, "tokens/train_per_sec_per_gpu": 55.56, "tokens/trainable": 245454} +{"epoch": 0.921875, "grad_norm": 1.1024205684661865, "learning_rate": 6.464922830953799e-05, "loss": 0.21609236299991608, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.24122, "step": 236, "tokens/total": 8207952, "tokens/train_per_sec_per_gpu": 51.08, "tokens/trainable": 246465} +{"epoch": 0.92578125, "grad_norm": 1.4630966186523438, "learning_rate": 6.436549078207688e-05, "loss": 0.23503252863883972, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.21, "memory/max_allocated (GiB)": 38.21, "ppl": 1.26495, "step": 237, "tokens/total": 8241040, "tokens/train_per_sec_per_gpu": 62.27, "tokens/trainable": 247465} +{"epoch": 0.9296875, "grad_norm": 1.3381998538970947, "learning_rate": 6.408136351831592e-05, "loss": 0.24755939841270447, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 37.15, "memory/max_allocated (GiB)": 37.15, "ppl": 1.2809, "step": 238, "tokens/total": 8273408, "tokens/train_per_sec_per_gpu": 60.22, "tokens/trainable": 248462} +{"epoch": 0.93359375, "grad_norm": 0.7437805533409119, "learning_rate": 6.379685834195036e-05, "loss": 0.19751495122909546, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.21837, "step": 239, "tokens/total": 8312912, "tokens/train_per_sec_per_gpu": 54.39, "tokens/trainable": 249607} +{"epoch": 0.9375, "grad_norm": 0.8184701204299927, "learning_rate": 6.351198709240186e-05, "loss": 0.20236247777938843, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.61, "memory/max_allocated (GiB)": 39.61, "ppl": 1.22429, "step": 240, "tokens/total": 8353040, "tokens/train_per_sec_per_gpu": 51.2, "tokens/trainable": 250667} +{"epoch": 0.94140625, "grad_norm": 1.6528514623641968, "learning_rate": 6.32267616243259e-05, "loss": 0.23755699396133423, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.26815, "step": 241, "tokens/total": 8385632, "tokens/train_per_sec_per_gpu": 63.62, "tokens/trainable": 251697} +{"epoch": 0.9453125, "grad_norm": 1.5294530391693115, "learning_rate": 6.294119380711849e-05, "loss": 0.24676495790481567, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.27988, "step": 242, "tokens/total": 8420064, "tokens/train_per_sec_per_gpu": 52.56, "tokens/trainable": 252654} +{"epoch": 0.94921875, "grad_norm": 2.103074312210083, "learning_rate": 6.265529552442209e-05, "loss": 0.21161553263664246, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.23567, "step": 243, "tokens/total": 8455424, "tokens/train_per_sec_per_gpu": 62.96, "tokens/trainable": 253735} +{"epoch": 0.953125, "grad_norm": 1.1673506498336792, "learning_rate": 6.236907867363127e-05, "loss": 0.2110934555530548, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.14, "memory/max_allocated (GiB)": 38.14, "ppl": 1.23503, "step": 244, "tokens/total": 8489760, "tokens/train_per_sec_per_gpu": 63.36, "tokens/trainable": 254769} +{"epoch": 0.95703125, "grad_norm": 1.688551664352417, "learning_rate": 6.208255516539749e-05, "loss": 0.19426587224006653, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 36.34, "memory/max_allocated (GiB)": 36.34, "ppl": 1.21442, "step": 245, "tokens/total": 8521968, "tokens/train_per_sec_per_gpu": 60.56, "tokens/trainable": 255826} +{"epoch": 0.9609375, "grad_norm": 1.076004147529602, "learning_rate": 6.179573692313344e-05, "loss": 0.22497496008872986, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 1.25229, "step": 246, "tokens/total": 8559088, "tokens/train_per_sec_per_gpu": 57.85, "tokens/trainable": 256838} +{"epoch": 0.96484375, "grad_norm": 1.2494159936904907, "learning_rate": 6.150863588251694e-05, "loss": 0.22171033918857574, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.24821, "step": 247, "tokens/total": 8599360, "tokens/train_per_sec_per_gpu": 50.26, "tokens/trainable": 257900} +{"epoch": 0.96875, "grad_norm": 1.2237417697906494, "learning_rate": 6.122126399099419e-05, "loss": 0.19969019293785095, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.78, "memory/max_allocated (GiB)": 39.78, "ppl": 1.22102, "step": 248, "tokens/total": 8639328, "tokens/train_per_sec_per_gpu": 66.39, "tokens/trainable": 259156} +{"epoch": 0.97265625, "grad_norm": 1.539944052696228, "learning_rate": 6.0933633207282615e-05, "loss": 0.2563021183013916, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.29214, "step": 249, "tokens/total": 8675696, "tokens/train_per_sec_per_gpu": 54.8, "tokens/trainable": 260223} +{"epoch": 0.9765625, "grad_norm": 0.925384521484375, "learning_rate": 6.064575550087316e-05, "loss": 0.19427251815795898, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.63, "memory/max_allocated (GiB)": 39.63, "ppl": 1.21443, "step": 250, "tokens/total": 8712704, "tokens/train_per_sec_per_gpu": 60.46, "tokens/trainable": 261294} +{"epoch": 0.98046875, "grad_norm": 0.9455996155738831, "learning_rate": 6.0357642851532245e-05, "loss": 0.20206236839294434, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.22392, "step": 251, "tokens/total": 8749136, "tokens/train_per_sec_per_gpu": 54.93, "tokens/trainable": 262362} +{"epoch": 0.984375, "grad_norm": 1.4516377449035645, "learning_rate": 6.0069307248803294e-05, "loss": 0.24119803309440613, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.27277, "step": 252, "tokens/total": 8784464, "tokens/train_per_sec_per_gpu": 50.49, "tokens/trainable": 263324} +{"epoch": 0.98828125, "grad_norm": 1.1500355005264282, "learning_rate": 5.9780760691507635e-05, "loss": 0.22668789327144623, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 36.25, "memory/max_allocated (GiB)": 36.25, "ppl": 1.25444, "step": 253, "tokens/total": 8815712, "tokens/train_per_sec_per_gpu": 61.39, "tokens/trainable": 264266} +{"epoch": 0.9921875, "grad_norm": 1.057946801185608, "learning_rate": 5.9492015187245334e-05, "loss": 0.23021353781223297, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.72, "memory/max_allocated (GiB)": 38.72, "ppl": 1.25887, "step": 254, "tokens/total": 8853296, "tokens/train_per_sec_per_gpu": 54.7, "tokens/trainable": 265231} +{"epoch": 0.99609375, "grad_norm": 1.0510300397872925, "learning_rate": 5.920308275189541e-05, "loss": 0.2035415768623352, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 36.6, "memory/max_allocated (GiB)": 36.6, "ppl": 1.22574, "step": 255, "tokens/total": 8885248, "tokens/train_per_sec_per_gpu": 63.38, "tokens/trainable": 266241} +{"epoch": 1.0, "grad_norm": 0.7808210253715515, "learning_rate": 5.8913975409115874e-05, "loss": 0.19769340753555298, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.21859, "step": 256, "tokens/total": 8918608, "tokens/train_per_sec_per_gpu": 62.38, "tokens/trainable": 267300} +{"epoch": 1.00390625, "grad_norm": 0.7946572303771973, "learning_rate": 5.8624705189843395e-05, "loss": 0.17980074882507324, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.19698, "step": 257, "tokens/total": 8956064, "tokens/train_per_sec_per_gpu": 55.25, "tokens/trainable": 268407} +{"epoch": 1.0078125, "grad_norm": 1.2595192193984985, "learning_rate": 5.833528413179249e-05, "loss": 0.18577738106250763, "memory/device_reserved (GiB)": 40.67, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.20415, "step": 258, "tokens/total": 8992688, "tokens/train_per_sec_per_gpu": 58.59, "tokens/trainable": 269500} +{"epoch": 1.01171875, "grad_norm": 1.2449452877044678, "learning_rate": 5.80457242789548e-05, "loss": 0.16460400819778442, "memory/device_reserved (GiB)": 42.74, "memory/max_active (GiB)": 39.65, "memory/max_allocated (GiB)": 39.65, "ppl": 1.17893, "step": 259, "tokens/total": 9032864, "tokens/train_per_sec_per_gpu": 57.32, "tokens/trainable": 270674} +{"epoch": 1.015625, "grad_norm": 0.9992119073867798, "learning_rate": 5.77560376810977e-05, "loss": 0.17238405346870422, "memory/device_reserved (GiB)": 42.75, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.18813, "step": 260, "tokens/total": 9066224, "tokens/train_per_sec_per_gpu": 68.41, "tokens/trainable": 271807} +{"epoch": 1.01953125, "grad_norm": 1.0623784065246582, "learning_rate": 5.7466236393263005e-05, "loss": 0.1938612461090088, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.21393, "step": 261, "tokens/total": 9100848, "tokens/train_per_sec_per_gpu": 60.77, "tokens/trainable": 272826} +{"epoch": 1.0234375, "grad_norm": 1.8530741930007935, "learning_rate": 5.717633247526522e-05, "loss": 0.21095795929431915, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 36.74, "memory/max_allocated (GiB)": 36.74, "ppl": 1.23486, "step": 262, "tokens/total": 9131536, "tokens/train_per_sec_per_gpu": 57.76, "tokens/trainable": 273749} +{"epoch": 1.02734375, "grad_norm": 1.8259193897247314, "learning_rate": 5.688633799118971e-05, "loss": 0.2233690321445465, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.25028, "step": 263, "tokens/total": 9163984, "tokens/train_per_sec_per_gpu": 55.57, "tokens/trainable": 274687} +{"epoch": 1.03125, "grad_norm": 0.786186695098877, "learning_rate": 5.659626500889066e-05, "loss": 0.17806792259216309, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.09, "memory/max_allocated (GiB)": 37.09, "ppl": 1.19491, "step": 264, "tokens/total": 9198000, "tokens/train_per_sec_per_gpu": 64.71, "tokens/trainable": 275813} +{"epoch": 1.03515625, "grad_norm": 0.947731614112854, "learning_rate": 5.6306125599488905e-05, "loss": 0.1874861717224121, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 36.88, "memory/max_allocated (GiB)": 36.88, "ppl": 1.20621, "step": 265, "tokens/total": 9228848, "tokens/train_per_sec_per_gpu": 68.9, "tokens/trainable": 276828} +{"epoch": 1.0390625, "grad_norm": 0.9514778852462769, "learning_rate": 5.601593183686955e-05, "loss": 0.1644342690706253, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.17873, "step": 266, "tokens/total": 9266064, "tokens/train_per_sec_per_gpu": 59.93, "tokens/trainable": 277947} +{"epoch": 1.04296875, "grad_norm": 1.3861416578292847, "learning_rate": 5.572569579717961e-05, "loss": 0.20047709345817566, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.22199, "step": 267, "tokens/total": 9301472, "tokens/train_per_sec_per_gpu": 61.89, "tokens/trainable": 279038} +{"epoch": 1.046875, "grad_norm": 0.8458371758460999, "learning_rate": 5.543542955832538e-05, "loss": 0.1720079779624939, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.18769, "step": 268, "tokens/total": 9341424, "tokens/train_per_sec_per_gpu": 52.67, "tokens/trainable": 280121} +{"epoch": 1.05078125, "grad_norm": 1.6477588415145874, "learning_rate": 5.514514519946986e-05, "loss": 0.19935768842697144, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.57, "memory/max_allocated (GiB)": 39.57, "ppl": 1.22062, "step": 269, "tokens/total": 9381840, "tokens/train_per_sec_per_gpu": 56.96, "tokens/trainable": 281236} +{"epoch": 1.0546875, "grad_norm": 1.8414796590805054, "learning_rate": 5.485485480053015e-05, "loss": 0.17919868230819702, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.52, "memory/max_allocated (GiB)": 39.52, "ppl": 1.19626, "step": 270, "tokens/total": 9420224, "tokens/train_per_sec_per_gpu": 48.68, "tokens/trainable": 282225} +{"epoch": 1.05859375, "grad_norm": 1.0701853036880493, "learning_rate": 5.4564570441674645e-05, "loss": 0.1894383430480957, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.20857, "step": 271, "tokens/total": 9453888, "tokens/train_per_sec_per_gpu": 66.97, "tokens/trainable": 283267} +{"epoch": 1.0625, "grad_norm": 4.854472637176514, "learning_rate": 5.42743042028204e-05, "loss": 0.2029534876346588, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.83, "memory/max_allocated (GiB)": 37.83, "ppl": 1.22502, "step": 272, "tokens/total": 9490080, "tokens/train_per_sec_per_gpu": 48.22, "tokens/trainable": 284230} +{"epoch": 1.06640625, "grad_norm": 1.180840015411377, "learning_rate": 5.3984068163130464e-05, "loss": 0.17014721035957336, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.18548, "step": 273, "tokens/total": 9523840, "tokens/train_per_sec_per_gpu": 67.2, "tokens/trainable": 285389} +{"epoch": 1.0703125, "grad_norm": 1.0682202577590942, "learning_rate": 5.369387440051111e-05, "loss": 0.20203229784965515, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.18, "memory/max_allocated (GiB)": 37.18, "ppl": 1.22389, "step": 274, "tokens/total": 9557552, "tokens/train_per_sec_per_gpu": 58.9, "tokens/trainable": 286391} +{"epoch": 1.07421875, "grad_norm": 1.113424301147461, "learning_rate": 5.340373499110935e-05, "loss": 0.18415693938732147, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.2022, "step": 275, "tokens/total": 9593360, "tokens/train_per_sec_per_gpu": 59.88, "tokens/trainable": 287477} +{"epoch": 1.078125, "grad_norm": 1.2602788209915161, "learning_rate": 5.3113662008810304e-05, "loss": 0.17429089546203613, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.1904, "step": 276, "tokens/total": 9633696, "tokens/train_per_sec_per_gpu": 50.38, "tokens/trainable": 288537} +{"epoch": 1.08203125, "grad_norm": 0.8255174160003662, "learning_rate": 5.282366752473479e-05, "loss": 0.17653460800647736, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.29, "memory/max_allocated (GiB)": 38.29, "ppl": 1.19308, "step": 277, "tokens/total": 9670944, "tokens/train_per_sec_per_gpu": 58.02, "tokens/trainable": 289637} +{"epoch": 1.0859375, "grad_norm": 4.8877129554748535, "learning_rate": 5.2533763606737005e-05, "loss": 0.21750949323177338, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.28, "memory/max_allocated (GiB)": 39.28, "ppl": 1.24298, "step": 278, "tokens/total": 9710832, "tokens/train_per_sec_per_gpu": 45.54, "tokens/trainable": 290625} +{"epoch": 1.08984375, "grad_norm": 0.9703748822212219, "learning_rate": 5.224396231890232e-05, "loss": 0.17493966221809387, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.23, "memory/max_allocated (GiB)": 37.23, "ppl": 1.19117, "step": 279, "tokens/total": 9742608, "tokens/train_per_sec_per_gpu": 64.96, "tokens/trainable": 291724} +{"epoch": 1.09375, "grad_norm": 0.9588870406150818, "learning_rate": 5.195427572104522e-05, "loss": 0.20239916443824768, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.22434, "step": 280, "tokens/total": 9777072, "tokens/train_per_sec_per_gpu": 59.42, "tokens/trainable": 292755} +{"epoch": 1.09765625, "grad_norm": 0.9067177772521973, "learning_rate": 5.166471586820751e-05, "loss": 0.18685585260391235, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.20545, "step": 281, "tokens/total": 9813824, "tokens/train_per_sec_per_gpu": 61.24, "tokens/trainable": 293847} +{"epoch": 1.1015625, "grad_norm": 0.8028798699378967, "learning_rate": 5.1375294810156615e-05, "loss": 0.18313977122306824, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.20098, "step": 282, "tokens/total": 9848464, "tokens/train_per_sec_per_gpu": 59.85, "tokens/trainable": 294869} +{"epoch": 1.10546875, "grad_norm": 1.2931636571884155, "learning_rate": 5.1086024590884144e-05, "loss": 0.1955316662788391, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.28, "memory/max_allocated (GiB)": 39.28, "ppl": 1.21596, "step": 283, "tokens/total": 9886720, "tokens/train_per_sec_per_gpu": 52.79, "tokens/trainable": 295870} +{"epoch": 1.109375, "grad_norm": 0.7194890379905701, "learning_rate": 5.079691724810461e-05, "loss": 0.1825902909040451, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.20032, "step": 284, "tokens/total": 9922592, "tokens/train_per_sec_per_gpu": 49.3, "tokens/trainable": 296839} +{"epoch": 1.11328125, "grad_norm": 1.0613099336624146, "learning_rate": 5.0507984812754684e-05, "loss": 0.18994292616844177, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.20918, "step": 285, "tokens/total": 9956144, "tokens/train_per_sec_per_gpu": 59.73, "tokens/trainable": 297886} +{"epoch": 1.1171875, "grad_norm": 0.9089866876602173, "learning_rate": 5.021923930849237e-05, "loss": 0.19014523923397064, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.20943, "step": 286, "tokens/total": 9990032, "tokens/train_per_sec_per_gpu": 56.87, "tokens/trainable": 298899} +{"epoch": 1.12109375, "grad_norm": 0.7153612375259399, "learning_rate": 4.99306927511967e-05, "loss": 0.17640218138694763, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.19292, "step": 287, "tokens/total": 10023504, "tokens/train_per_sec_per_gpu": 62.43, "tokens/trainable": 299906} +{"epoch": 1.125, "grad_norm": 0.8394687175750732, "learning_rate": 4.964235714846775e-05, "loss": 0.2073913812637329, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 36.44, "memory/max_allocated (GiB)": 36.44, "ppl": 1.23046, "step": 288, "tokens/total": 10055216, "tokens/train_per_sec_per_gpu": 66.47, "tokens/trainable": 300900} +{"epoch": 1.12890625, "grad_norm": 1.6544114351272583, "learning_rate": 4.9354244499126866e-05, "loss": 0.21190449595451355, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 36.23, "memory/max_allocated (GiB)": 36.23, "ppl": 1.23603, "step": 289, "tokens/total": 10086880, "tokens/train_per_sec_per_gpu": 63.33, "tokens/trainable": 301932} +{"epoch": 1.1328125, "grad_norm": 0.962895393371582, "learning_rate": 4.90663667927174e-05, "loss": 0.1916569471359253, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.21125, "step": 290, "tokens/total": 10119712, "tokens/train_per_sec_per_gpu": 71.35, "tokens/trainable": 302959} +{"epoch": 1.13671875, "grad_norm": 1.1343777179718018, "learning_rate": 4.877873600900581e-05, "loss": 0.1761254519224167, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.07, "memory/max_allocated (GiB)": 38.07, "ppl": 1.19259, "step": 291, "tokens/total": 10155824, "tokens/train_per_sec_per_gpu": 54.06, "tokens/trainable": 303979} +{"epoch": 1.140625, "grad_norm": 1.0487289428710938, "learning_rate": 4.849136411748306e-05, "loss": 0.16280591487884521, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.04, "memory/max_allocated (GiB)": 39.04, "ppl": 1.17681, "step": 292, "tokens/total": 10193872, "tokens/train_per_sec_per_gpu": 53.77, "tokens/trainable": 305036} +{"epoch": 1.14453125, "grad_norm": 1.4127047061920166, "learning_rate": 4.8204263076866574e-05, "loss": 0.18046487867832184, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.19777, "step": 293, "tokens/total": 10229792, "tokens/train_per_sec_per_gpu": 57.68, "tokens/trainable": 306098} +{"epoch": 1.1484375, "grad_norm": 1.0430879592895508, "learning_rate": 4.791744483460251e-05, "loss": 0.2133231908082962, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.74, "memory/max_allocated (GiB)": 37.74, "ppl": 1.23778, "step": 294, "tokens/total": 10264288, "tokens/train_per_sec_per_gpu": 51.85, "tokens/trainable": 307092} +{"epoch": 1.15234375, "grad_norm": 1.5851408243179321, "learning_rate": 4.7630921326368736e-05, "loss": 0.18287095427513123, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.20066, "step": 295, "tokens/total": 10303008, "tokens/train_per_sec_per_gpu": 50.18, "tokens/trainable": 308074} +{"epoch": 1.15625, "grad_norm": 1.327195167541504, "learning_rate": 4.7344704475577916e-05, "loss": 0.17734277248382568, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 35.7, "memory/max_allocated (GiB)": 35.7, "ppl": 1.19404, "step": 296, "tokens/total": 10332576, "tokens/train_per_sec_per_gpu": 68.43, "tokens/trainable": 309100} +{"epoch": 1.16015625, "grad_norm": 0.8878957629203796, "learning_rate": 4.705880619288153e-05, "loss": 0.18705318868160248, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.37, "memory/max_allocated (GiB)": 38.37, "ppl": 1.20569, "step": 297, "tokens/total": 10368800, "tokens/train_per_sec_per_gpu": 53.12, "tokens/trainable": 310085} +{"epoch": 1.1640625, "grad_norm": 0.9497339129447937, "learning_rate": 4.677323837567412e-05, "loss": 0.17169979214668274, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.18732, "step": 298, "tokens/total": 10401632, "tokens/train_per_sec_per_gpu": 64.72, "tokens/trainable": 311141} +{"epoch": 1.16796875, "grad_norm": 1.403023362159729, "learning_rate": 4.6488012907598146e-05, "loss": 0.21399036049842834, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 36.4, "memory/max_allocated (GiB)": 36.4, "ppl": 1.23861, "step": 299, "tokens/total": 10433840, "tokens/train_per_sec_per_gpu": 57.51, "tokens/trainable": 312096} +{"epoch": 1.171875, "grad_norm": 0.9258077144622803, "learning_rate": 4.620314165804964e-05, "loss": 0.1709284782409668, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.18641, "step": 300, "tokens/total": 10469664, "tokens/train_per_sec_per_gpu": 58.6, "tokens/trainable": 313192} +{"epoch": 1.17578125, "grad_norm": 0.9101457595825195, "learning_rate": 4.591863648168407e-05, "loss": 0.17706750333309174, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.19371, "step": 301, "tokens/total": 10504224, "tokens/train_per_sec_per_gpu": 59.41, "tokens/trainable": 314288} +{"epoch": 1.1796875, "grad_norm": 0.8996168971061707, "learning_rate": 4.5634509217923135e-05, "loss": 0.19223210215568542, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.96, "memory/max_allocated (GiB)": 37.96, "ppl": 1.21195, "step": 302, "tokens/total": 10537760, "tokens/train_per_sec_per_gpu": 61.1, "tokens/trainable": 315326} +{"epoch": 1.18359375, "grad_norm": 0.9213551878929138, "learning_rate": 4.535077169046201e-05, "loss": 0.18583142757415771, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.03, "memory/max_allocated (GiB)": 38.03, "ppl": 1.20422, "step": 303, "tokens/total": 10574368, "tokens/train_per_sec_per_gpu": 57.14, "tokens/trainable": 316356} +{"epoch": 1.1875, "grad_norm": 1.2757748365402222, "learning_rate": 4.506743570677743e-05, "loss": 0.19561254978179932, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 36.35, "memory/max_allocated (GiB)": 36.35, "ppl": 1.21606, "step": 304, "tokens/total": 10604688, "tokens/train_per_sec_per_gpu": 72.06, "tokens/trainable": 317354} +{"epoch": 1.19140625, "grad_norm": 0.8101264238357544, "learning_rate": 4.478451305763618e-05, "loss": 0.19973218441009521, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.22108, "step": 305, "tokens/total": 10638272, "tokens/train_per_sec_per_gpu": 64.16, "tokens/trainable": 318392} +{"epoch": 1.1953125, "grad_norm": 0.7293171286582947, "learning_rate": 4.450201551660454e-05, "loss": 0.18459665775299072, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 36.41, "memory/max_allocated (GiB)": 36.41, "ppl": 1.20273, "step": 306, "tokens/total": 10669456, "tokens/train_per_sec_per_gpu": 63.42, "tokens/trainable": 319397} +{"epoch": 1.19921875, "grad_norm": 1.2398077249526978, "learning_rate": 4.4219954839558276e-05, "loss": 0.1995004117488861, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.22079, "step": 307, "tokens/total": 10704080, "tokens/train_per_sec_per_gpu": 53.85, "tokens/trainable": 320409} +{"epoch": 1.203125, "grad_norm": 1.0822640657424927, "learning_rate": 4.393834276419352e-05, "loss": 0.19633419811725616, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.21693, "step": 308, "tokens/total": 10735712, "tokens/train_per_sec_per_gpu": 67.36, "tokens/trainable": 321446} +{"epoch": 1.20703125, "grad_norm": 0.9743812680244446, "learning_rate": 4.36571910095382e-05, "loss": 0.1692434847354889, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.18441, "step": 309, "tokens/total": 10774144, "tokens/train_per_sec_per_gpu": 55.16, "tokens/trainable": 322500} +{"epoch": 1.2109375, "grad_norm": 0.8287404179573059, "learning_rate": 4.337651127546448e-05, "loss": 0.18398955464363098, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.202, "step": 310, "tokens/total": 10809744, "tokens/train_per_sec_per_gpu": 61.98, "tokens/trainable": 323519} +{"epoch": 1.21484375, "grad_norm": 0.7480834722518921, "learning_rate": 4.3096315242201736e-05, "loss": 0.19149979948997498, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 36.51, "memory/max_allocated (GiB)": 36.51, "ppl": 1.21106, "step": 311, "tokens/total": 10842480, "tokens/train_per_sec_per_gpu": 59.08, "tokens/trainable": 324521} +{"epoch": 1.21875, "grad_norm": 0.7803010940551758, "learning_rate": 4.2816614569850635e-05, "loss": 0.18653111159801483, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.66, "memory/max_allocated (GiB)": 38.66, "ppl": 1.20506, "step": 312, "tokens/total": 10878256, "tokens/train_per_sec_per_gpu": 58.28, "tokens/trainable": 325521} +{"epoch": 1.22265625, "grad_norm": 1.3694956302642822, "learning_rate": 4.2537420897897864e-05, "loss": 0.193773090839386, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.72, "memory/max_allocated (GiB)": 38.72, "ppl": 1.21382, "step": 313, "tokens/total": 10913904, "tokens/train_per_sec_per_gpu": 53.7, "tokens/trainable": 326541} +{"epoch": 1.2265625, "grad_norm": 0.7329290509223938, "learning_rate": 4.225874584473174e-05, "loss": 0.17926274240016937, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.19634, "step": 314, "tokens/total": 10946448, "tokens/train_per_sec_per_gpu": 63.93, "tokens/trainable": 327555} +{"epoch": 1.23046875, "grad_norm": 0.7727279663085938, "learning_rate": 4.19806010071587e-05, "loss": 0.1681526005268097, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.18312, "step": 315, "tokens/total": 10984528, "tokens/train_per_sec_per_gpu": 52.68, "tokens/trainable": 328596} +{"epoch": 1.234375, "grad_norm": 1.0019770860671997, "learning_rate": 4.170299795992081e-05, "loss": 0.1907237321138382, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 36.88, "memory/max_allocated (GiB)": 36.88, "ppl": 1.21013, "step": 316, "tokens/total": 11018096, "tokens/train_per_sec_per_gpu": 60.4, "tokens/trainable": 329654} +{"epoch": 1.23828125, "grad_norm": 0.5373663902282715, "learning_rate": 4.142594825521398e-05, "loss": 0.16370254755020142, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.17786, "step": 317, "tokens/total": 11057200, "tokens/train_per_sec_per_gpu": 51.43, "tokens/trainable": 330711} +{"epoch": 1.2421875, "grad_norm": 1.2201167345046997, "learning_rate": 4.114946342220728e-05, "loss": 0.1699419617652893, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.18524, "step": 318, "tokens/total": 11093776, "tokens/train_per_sec_per_gpu": 56.61, "tokens/trainable": 331795} +{"epoch": 1.24609375, "grad_norm": 0.5880972743034363, "learning_rate": 4.087355496656321e-05, "loss": 0.15903571248054504, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.17238, "step": 319, "tokens/total": 11127408, "tokens/train_per_sec_per_gpu": 68.76, "tokens/trainable": 332917} +{"epoch": 1.25, "grad_norm": 0.6570891737937927, "learning_rate": 4.05982343699588e-05, "loss": 0.16312849521636963, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.17719, "step": 320, "tokens/total": 11166656, "tokens/train_per_sec_per_gpu": 58.28, "tokens/trainable": 334062} +{"epoch": 1.25390625, "grad_norm": 1.0566675662994385, "learning_rate": 4.0323513089607876e-05, "loss": 0.16377407312393188, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.17795, "step": 321, "tokens/total": 11201568, "tokens/train_per_sec_per_gpu": 57.78, "tokens/trainable": 335146} +{"epoch": 1.2578125, "grad_norm": 0.8617367148399353, "learning_rate": 4.004940255778431e-05, "loss": 0.18556365370750427, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.63, "memory/max_allocated (GiB)": 39.63, "ppl": 1.2039, "step": 322, "tokens/total": 11240288, "tokens/train_per_sec_per_gpu": 56.71, "tokens/trainable": 336203} +{"epoch": 1.26171875, "grad_norm": 0.8812937140464783, "learning_rate": 3.977591418134619e-05, "loss": 0.1829240322113037, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.20072, "step": 323, "tokens/total": 11275248, "tokens/train_per_sec_per_gpu": 61.36, "tokens/trainable": 337254} +{"epoch": 1.265625, "grad_norm": 1.276995062828064, "learning_rate": 3.95030593412612e-05, "loss": 0.20826944708824158, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.23154, "step": 324, "tokens/total": 11311152, "tokens/train_per_sec_per_gpu": 57.44, "tokens/trainable": 338256} +{"epoch": 1.26953125, "grad_norm": 0.7280957698822021, "learning_rate": 3.923084939213296e-05, "loss": 0.16969279944896698, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 35.75, "memory/max_allocated (GiB)": 35.75, "ppl": 1.18494, "step": 325, "tokens/total": 11341728, "tokens/train_per_sec_per_gpu": 62.65, "tokens/trainable": 339266} +{"epoch": 1.2734375, "grad_norm": 0.6636548042297363, "learning_rate": 3.895929566172861e-05, "loss": 0.152546226978302, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.1648, "step": 326, "tokens/total": 11377744, "tokens/train_per_sec_per_gpu": 60.34, "tokens/trainable": 340370} +{"epoch": 1.27734375, "grad_norm": 0.8003033995628357, "learning_rate": 3.868840945050728e-05, "loss": 0.16062237322330475, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.19, "memory/max_allocated (GiB)": 39.19, "ppl": 1.17424, "step": 327, "tokens/total": 11416688, "tokens/train_per_sec_per_gpu": 56.18, "tokens/trainable": 341459} +{"epoch": 1.28125, "grad_norm": 0.998537540435791, "learning_rate": 3.841820203114995e-05, "loss": 0.1618598997592926, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 36.78, "memory/max_allocated (GiB)": 36.78, "ppl": 1.1757, "step": 328, "tokens/total": 11448432, "tokens/train_per_sec_per_gpu": 67.47, "tokens/trainable": 342545} +{"epoch": 1.28515625, "grad_norm": 0.6925503611564636, "learning_rate": 3.814868464809027e-05, "loss": 0.176986962556839, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 36.77, "memory/max_allocated (GiB)": 36.77, "ppl": 1.19362, "step": 329, "tokens/total": 11480368, "tokens/train_per_sec_per_gpu": 61.41, "tokens/trainable": 343541} +{"epoch": 1.2890625, "grad_norm": 2.0401644706726074, "learning_rate": 3.787986851704667e-05, "loss": 0.18118682503700256, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.19864, "step": 330, "tokens/total": 11513920, "tokens/train_per_sec_per_gpu": 63.63, "tokens/trainable": 344570} +{"epoch": 1.29296875, "grad_norm": 0.7235468029975891, "learning_rate": 3.7611764824555654e-05, "loss": 0.16000951826572418, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.84, "memory/max_allocated (GiB)": 37.84, "ppl": 1.17352, "step": 331, "tokens/total": 11545648, "tokens/train_per_sec_per_gpu": 76.22, "tokens/trainable": 345723} +{"epoch": 1.296875, "grad_norm": 0.8117030262947083, "learning_rate": 3.734438472750619e-05, "loss": 0.16747058928012848, "memory/device_reserved (GiB)": 43.07, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.18231, "step": 332, "tokens/total": 11585616, "tokens/train_per_sec_per_gpu": 53.93, "tokens/trainable": 346834} +{"epoch": 1.30078125, "grad_norm": 0.8633759021759033, "learning_rate": 3.707773935267552e-05, "loss": 0.16902782022953033, "memory/device_reserved (GiB)": 43.07, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.18415, "step": 333, "tokens/total": 11622672, "tokens/train_per_sec_per_gpu": 57.89, "tokens/trainable": 347961} +{"epoch": 1.3046875, "grad_norm": 0.7751632332801819, "learning_rate": 3.68118397962661e-05, "loss": 0.1925753504037857, "memory/device_reserved (GiB)": 43.07, "memory/max_active (GiB)": 38.49, "memory/max_allocated (GiB)": 38.49, "ppl": 1.21237, "step": 334, "tokens/total": 11657424, "tokens/train_per_sec_per_gpu": 55.88, "tokens/trainable": 348923} +{"epoch": 1.30859375, "grad_norm": 0.8371902704238892, "learning_rate": 3.654669712344384e-05, "loss": 0.16817975044250488, "memory/device_reserved (GiB)": 43.07, "memory/max_active (GiB)": 36.72, "memory/max_allocated (GiB)": 36.72, "ppl": 1.18315, "step": 335, "tokens/total": 11688624, "tokens/train_per_sec_per_gpu": 74.96, "tokens/trainable": 349979} +{"epoch": 1.3125, "grad_norm": 0.9324823617935181, "learning_rate": 3.628232236787763e-05, "loss": 0.15803274512290955, "memory/device_reserved (GiB)": 43.07, "memory/max_active (GiB)": 37.91, "memory/max_allocated (GiB)": 37.91, "ppl": 1.1712, "step": 336, "tokens/total": 11724224, "tokens/train_per_sec_per_gpu": 56.25, "tokens/trainable": 351092} +{"epoch": 1.31640625, "grad_norm": 0.8988579511642456, "learning_rate": 3.6018726531280144e-05, "loss": 0.17386077344417572, "memory/device_reserved (GiB)": 43.07, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.18989, "step": 337, "tokens/total": 11761632, "tokens/train_per_sec_per_gpu": 60.49, "tokens/trainable": 352182} +{"epoch": 1.3203125, "grad_norm": 0.7303047776222229, "learning_rate": 3.575592058295017e-05, "loss": 0.18630439043045044, "memory/device_reserved (GiB)": 43.07, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.20479, "step": 338, "tokens/total": 11795616, "tokens/train_per_sec_per_gpu": 61.4, "tokens/trainable": 353234} +{"epoch": 1.32421875, "grad_norm": 1.6812031269073486, "learning_rate": 3.549391545931585e-05, "loss": 0.17648541927337646, "memory/device_reserved (GiB)": 43.07, "memory/max_active (GiB)": 38.93, "memory/max_allocated (GiB)": 38.93, "ppl": 1.19302, "step": 339, "tokens/total": 11830096, "tokens/train_per_sec_per_gpu": 62.44, "tokens/trainable": 354284} +{"epoch": 1.328125, "grad_norm": 0.7523166537284851, "learning_rate": 3.5232722063479914e-05, "loss": 0.17722055315971375, "memory/device_reserved (GiB)": 43.07, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.19389, "step": 340, "tokens/total": 11864384, "tokens/train_per_sec_per_gpu": 63.67, "tokens/trainable": 355334} +{"epoch": 1.33203125, "grad_norm": 0.7775804996490479, "learning_rate": 3.49723512647657e-05, "loss": 0.16375303268432617, "memory/device_reserved (GiB)": 43.07, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.17792, "step": 341, "tokens/total": 11903648, "tokens/train_per_sec_per_gpu": 54.18, "tokens/trainable": 356450} +{"epoch": 1.3359375, "grad_norm": 0.6145197153091431, "learning_rate": 3.471281389826491e-05, "loss": 0.18532094359397888, "memory/device_reserved (GiB)": 43.07, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.2036, "step": 342, "tokens/total": 11934704, "tokens/train_per_sec_per_gpu": 63.45, "tokens/trainable": 357403} +{"epoch": 1.33984375, "grad_norm": 0.6914450526237488, "learning_rate": 3.4454120764386764e-05, "loss": 0.16675937175750732, "memory/device_reserved (GiB)": 43.07, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.18147, "step": 343, "tokens/total": 11973328, "tokens/train_per_sec_per_gpu": 56.69, "tokens/trainable": 358483} +{"epoch": 1.34375, "grad_norm": 0.738450825214386, "learning_rate": 3.4196282628408526e-05, "loss": 0.16652007400989532, "memory/device_reserved (GiB)": 43.07, "memory/max_active (GiB)": 37.58, "memory/max_allocated (GiB)": 37.58, "ppl": 1.18119, "step": 344, "tokens/total": 12007616, "tokens/train_per_sec_per_gpu": 61.88, "tokens/trainable": 359549} +{"epoch": 1.34765625, "grad_norm": 0.7032033801078796, "learning_rate": 3.3939310220027456e-05, "loss": 0.17044824361801147, "memory/device_reserved (GiB)": 43.07, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.18584, "step": 345, "tokens/total": 12041648, "tokens/train_per_sec_per_gpu": 60.55, "tokens/trainable": 360642} +{"epoch": 1.3515625, "grad_norm": 0.6984654664993286, "learning_rate": 3.3683214232914404e-05, "loss": 0.15939700603485107, "memory/device_reserved (GiB)": 43.07, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.1728, "step": 346, "tokens/total": 12077312, "tokens/train_per_sec_per_gpu": 59.83, "tokens/trainable": 361722} +{"epoch": 1.35546875, "grad_norm": 0.5693284273147583, "learning_rate": 3.342800532426873e-05, "loss": 0.1813414990901947, "memory/device_reserved (GiB)": 43.07, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.19882, "step": 347, "tokens/total": 12113296, "tokens/train_per_sec_per_gpu": 60.06, "tokens/trainable": 362740} +{"epoch": 1.359375, "grad_norm": 0.7558127045631409, "learning_rate": 3.317369411437484e-05, "loss": 0.18378613889217377, "memory/device_reserved (GiB)": 43.07, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.20176, "step": 348, "tokens/total": 12149168, "tokens/train_per_sec_per_gpu": 56.06, "tokens/trainable": 363755} +{"epoch": 1.36328125, "grad_norm": 0.8855589032173157, "learning_rate": 3.292029118616024e-05, "loss": 0.17451559007167816, "memory/device_reserved (GiB)": 43.07, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.19067, "step": 349, "tokens/total": 12181888, "tokens/train_per_sec_per_gpu": 56.99, "tokens/trainable": 364737} +{"epoch": 1.3671875, "grad_norm": 1.0160067081451416, "learning_rate": 3.266780708475511e-05, "loss": 0.17801183462142944, "memory/device_reserved (GiB)": 43.07, "memory/max_active (GiB)": 39.12, "memory/max_allocated (GiB)": 39.12, "ppl": 1.19484, "step": 350, "tokens/total": 12217296, "tokens/train_per_sec_per_gpu": 58.36, "tokens/trainable": 365812} +{"epoch": 1.37109375, "grad_norm": 0.7686376571655273, "learning_rate": 3.241625231705354e-05, "loss": 0.16232365369796753, "memory/device_reserved (GiB)": 43.07, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.17624, "step": 351, "tokens/total": 12253344, "tokens/train_per_sec_per_gpu": 58.69, "tokens/trainable": 366870} +{"epoch": 1.375, "grad_norm": 0.7049140930175781, "learning_rate": 3.216563735127618e-05, "loss": 0.1536189466714859, "memory/device_reserved (GiB)": 43.07, "memory/max_active (GiB)": 39.12, "memory/max_allocated (GiB)": 39.12, "ppl": 1.16605, "step": 352, "tokens/total": 12288480, "tokens/train_per_sec_per_gpu": 61.09, "tokens/trainable": 368003} +{"epoch": 1.37890625, "grad_norm": 0.9764469861984253, "learning_rate": 3.191597261653475e-05, "loss": 0.18480509519577026, "memory/device_reserved (GiB)": 43.07, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.20298, "step": 353, "tokens/total": 12325248, "tokens/train_per_sec_per_gpu": 57.73, "tokens/trainable": 369082} +{"epoch": 1.3828125, "grad_norm": 1.813093900680542, "learning_rate": 3.166726850239794e-05, "loss": 0.1604873687028885, "memory/device_reserved (GiB)": 43.07, "memory/max_active (GiB)": 37.18, "memory/max_allocated (GiB)": 37.18, "ppl": 1.17408, "step": 354, "tokens/total": 12358880, "tokens/train_per_sec_per_gpu": 62.57, "tokens/trainable": 370160} +{"epoch": 1.38671875, "grad_norm": 0.903153657913208, "learning_rate": 3.141953535845912e-05, "loss": 0.1995467096567154, "memory/device_reserved (GiB)": 43.07, "memory/max_active (GiB)": 36.16, "memory/max_allocated (GiB)": 36.16, "ppl": 1.22085, "step": 355, "tokens/total": 12389696, "tokens/train_per_sec_per_gpu": 63.07, "tokens/trainable": 371098} +{"epoch": 1.390625, "grad_norm": 0.7607082724571228, "learning_rate": 3.11727834939056e-05, "loss": 0.17034026980400085, "memory/device_reserved (GiB)": 43.07, "memory/max_active (GiB)": 36.28, "memory/max_allocated (GiB)": 36.28, "ppl": 1.18571, "step": 356, "tokens/total": 12421536, "tokens/train_per_sec_per_gpu": 59.85, "tokens/trainable": 372131} +{"epoch": 1.39453125, "grad_norm": 0.6706284880638123, "learning_rate": 3.092702317708967e-05, "loss": 0.17502397298812866, "memory/device_reserved (GiB)": 43.07, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.19127, "step": 357, "tokens/total": 12455264, "tokens/train_per_sec_per_gpu": 62.46, "tokens/trainable": 373159} +{"epoch": 1.3984375, "grad_norm": 1.772330641746521, "learning_rate": 3.0682264635101276e-05, "loss": 0.1695246398448944, "memory/device_reserved (GiB)": 43.07, "memory/max_active (GiB)": 37.44, "memory/max_allocated (GiB)": 37.44, "ppl": 1.18474, "step": 358, "tokens/total": 12487776, "tokens/train_per_sec_per_gpu": 62.26, "tokens/trainable": 374180} +{"epoch": 1.40234375, "grad_norm": 1.1003612279891968, "learning_rate": 3.0438518053342407e-05, "loss": 0.165654718875885, "memory/device_reserved (GiB)": 43.07, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.18017, "step": 359, "tokens/total": 12526848, "tokens/train_per_sec_per_gpu": 55.82, "tokens/trainable": 375308} +{"epoch": 1.40625, "grad_norm": 0.6335047483444214, "learning_rate": 3.0195793575103266e-05, "loss": 0.16793735325336456, "memory/device_reserved (GiB)": 43.07, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.18286, "step": 360, "tokens/total": 12565888, "tokens/train_per_sec_per_gpu": 54.98, "tokens/trainable": 376384} +{"epoch": 1.41015625, "grad_norm": 0.6957264542579651, "learning_rate": 2.9954101301140146e-05, "loss": 0.1764889359474182, "memory/device_reserved (GiB)": 43.07, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.19302, "step": 361, "tokens/total": 12601232, "tokens/train_per_sec_per_gpu": 58.87, "tokens/trainable": 377425} +{"epoch": 1.4140625, "grad_norm": 1.2304182052612305, "learning_rate": 2.9713451289255123e-05, "loss": 0.17137464880943298, "memory/device_reserved (GiB)": 43.07, "memory/max_active (GiB)": 36.9, "memory/max_allocated (GiB)": 36.9, "ppl": 1.18694, "step": 362, "tokens/total": 12630624, "tokens/train_per_sec_per_gpu": 63.46, "tokens/trainable": 378465} +{"epoch": 1.41796875, "grad_norm": 0.5318067073822021, "learning_rate": 2.9473853553877484e-05, "loss": 0.15535899996757507, "memory/device_reserved (GiB)": 43.07, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.16808, "step": 363, "tokens/total": 12664560, "tokens/train_per_sec_per_gpu": 60.39, "tokens/trainable": 379504} +{"epoch": 1.421875, "grad_norm": 0.7330031991004944, "learning_rate": 2.9235318065647e-05, "loss": 0.1767447590827942, "memory/device_reserved (GiB)": 43.07, "memory/max_active (GiB)": 39.61, "memory/max_allocated (GiB)": 39.61, "ppl": 1.19333, "step": 364, "tokens/total": 12704576, "tokens/train_per_sec_per_gpu": 53.49, "tokens/trainable": 380566} +{"epoch": 1.42578125, "grad_norm": 0.7753434777259827, "learning_rate": 2.8997854750998964e-05, "loss": 0.16662609577178955, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.18131, "step": 365, "tokens/total": 12744096, "tokens/train_per_sec_per_gpu": 56.64, "tokens/trainable": 381661} +{"epoch": 1.4296875, "grad_norm": 2.093400716781616, "learning_rate": 2.8761473491751258e-05, "loss": 0.18069300055503845, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 36.63, "memory/max_allocated (GiB)": 36.63, "ppl": 1.19805, "step": 366, "tokens/total": 12776928, "tokens/train_per_sec_per_gpu": 59.99, "tokens/trainable": 382659} +{"epoch": 1.43359375, "grad_norm": 0.7053645253181458, "learning_rate": 2.8526184124692883e-05, "loss": 0.14765386283397675, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.15911, "step": 367, "tokens/total": 12813584, "tokens/train_per_sec_per_gpu": 52.68, "tokens/trainable": 383762} +{"epoch": 1.4375, "grad_norm": 0.8602466583251953, "learning_rate": 2.829199644117484e-05, "loss": 0.17064917087554932, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.18607, "step": 368, "tokens/total": 12847296, "tokens/train_per_sec_per_gpu": 62.92, "tokens/trainable": 384725} +{"epoch": 1.44140625, "grad_norm": 0.48980486392974854, "learning_rate": 2.8058920186702553e-05, "loss": 0.15452516078948975, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.91, "memory/max_allocated (GiB)": 38.91, "ppl": 1.1671, "step": 369, "tokens/total": 12885424, "tokens/train_per_sec_per_gpu": 58.96, "tokens/trainable": 385819} +{"epoch": 1.4453125, "grad_norm": 0.8001270294189453, "learning_rate": 2.782696506053033e-05, "loss": 0.17759649455547333, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.19434, "step": 370, "tokens/total": 12923616, "tokens/train_per_sec_per_gpu": 52.69, "tokens/trainable": 386814} +{"epoch": 1.44921875, "grad_norm": 0.6152340769767761, "learning_rate": 2.7596140715257824e-05, "loss": 0.16045811772346497, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.17405, "step": 371, "tokens/total": 12958896, "tokens/train_per_sec_per_gpu": 61.52, "tokens/trainable": 387909} +{"epoch": 1.453125, "grad_norm": 0.7708089351654053, "learning_rate": 2.7366456756428184e-05, "loss": 0.1653938889503479, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.17986, "step": 372, "tokens/total": 12993264, "tokens/train_per_sec_per_gpu": 60.15, "tokens/trainable": 388945} +{"epoch": 1.45703125, "grad_norm": 0.8284586071968079, "learning_rate": 2.7137922742128486e-05, "loss": 0.16545170545578003, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.21, "memory/max_allocated (GiB)": 38.21, "ppl": 1.17993, "step": 373, "tokens/total": 13028144, "tokens/train_per_sec_per_gpu": 61.55, "tokens/trainable": 390012} +{"epoch": 1.4609375, "grad_norm": 0.5819743871688843, "learning_rate": 2.691054818259188e-05, "loss": 0.15895362198352814, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.4, "memory/max_allocated (GiB)": 37.4, "ppl": 1.17228, "step": 374, "tokens/total": 13062816, "tokens/train_per_sec_per_gpu": 60.29, "tokens/trainable": 391113} +{"epoch": 1.46484375, "grad_norm": 1.8024080991744995, "learning_rate": 2.6684342539801933e-05, "loss": 0.1932150423526764, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 36.49, "memory/max_allocated (GiB)": 36.49, "ppl": 1.21314, "step": 375, "tokens/total": 13094080, "tokens/train_per_sec_per_gpu": 61.47, "tokens/trainable": 392093} +{"epoch": 1.46875, "grad_norm": 0.8577533960342407, "learning_rate": 2.645931522709877e-05, "loss": 0.15744023025035858, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 39.54, "memory/max_allocated (GiB)": 39.54, "ppl": 1.17051, "step": 376, "tokens/total": 13129824, "tokens/train_per_sec_per_gpu": 72.55, "tokens/trainable": 393260} +{"epoch": 1.47265625, "grad_norm": 1.5422065258026123, "learning_rate": 2.6235475608787365e-05, "loss": 0.18803149461746216, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.6, "memory/max_allocated (GiB)": 38.6, "ppl": 1.20687, "step": 377, "tokens/total": 13165552, "tokens/train_per_sec_per_gpu": 59.09, "tokens/trainable": 394289} +{"epoch": 1.4765625, "grad_norm": 0.7492903470993042, "learning_rate": 2.6012832999747916e-05, "loss": 0.18439990282058716, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.2025, "step": 378, "tokens/total": 13196336, "tokens/train_per_sec_per_gpu": 64.03, "tokens/trainable": 395275} +{"epoch": 1.48046875, "grad_norm": 0.8483450412750244, "learning_rate": 2.579139666504821e-05, "loss": 0.18670117855072021, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.20527, "step": 379, "tokens/total": 13236400, "tokens/train_per_sec_per_gpu": 51.67, "tokens/trainable": 396280} +{"epoch": 1.484375, "grad_norm": 0.5929962396621704, "learning_rate": 2.557117581955798e-05, "loss": 0.16463381052017212, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 1.17896, "step": 380, "tokens/total": 13270224, "tokens/train_per_sec_per_gpu": 59.65, "tokens/trainable": 397346} +{"epoch": 1.48828125, "grad_norm": 1.2548414468765259, "learning_rate": 2.5352179627565532e-05, "loss": 0.1840095818042755, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.06, "memory/max_allocated (GiB)": 38.06, "ppl": 1.20203, "step": 381, "tokens/total": 13305248, "tokens/train_per_sec_per_gpu": 54.55, "tokens/trainable": 398354} +{"epoch": 1.4921875, "grad_norm": 1.4652376174926758, "learning_rate": 2.5134417202396277e-05, "loss": 0.18365009129047394, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.2016, "step": 382, "tokens/total": 13338848, "tokens/train_per_sec_per_gpu": 57.59, "tokens/trainable": 399323} +{"epoch": 1.49609375, "grad_norm": 0.5007973313331604, "learning_rate": 2.491789760603361e-05, "loss": 0.17457444965839386, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.19074, "step": 383, "tokens/total": 13375344, "tokens/train_per_sec_per_gpu": 53.48, "tokens/trainable": 400339} +{"epoch": 1.5, "grad_norm": 0.7593055963516235, "learning_rate": 2.4702629848741764e-05, "loss": 0.17351868748664856, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.18948, "step": 384, "tokens/total": 13409984, "tokens/train_per_sec_per_gpu": 62.28, "tokens/trainable": 401343} +{"epoch": 1.50390625, "grad_norm": 1.3353886604309082, "learning_rate": 2.4488622888690785e-05, "loss": 0.1824767291545868, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.20019, "step": 385, "tokens/total": 13443568, "tokens/train_per_sec_per_gpu": 54.96, "tokens/trainable": 402319} +{"epoch": 1.5078125, "grad_norm": 0.6415424942970276, "learning_rate": 2.427588563158384e-05, "loss": 0.1585785299539566, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 39.11, "memory/max_allocated (GiB)": 39.11, "ppl": 1.17184, "step": 386, "tokens/total": 13477920, "tokens/train_per_sec_per_gpu": 62.2, "tokens/trainable": 403393} +{"epoch": 1.51171875, "grad_norm": 0.7468792200088501, "learning_rate": 2.406442693028651e-05, "loss": 0.18476662039756775, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.20294, "step": 387, "tokens/total": 13510368, "tokens/train_per_sec_per_gpu": 59.8, "tokens/trainable": 404342} +{"epoch": 1.515625, "grad_norm": 0.673612117767334, "learning_rate": 2.3854255584458547e-05, "loss": 0.1761399805545807, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.1926, "step": 388, "tokens/total": 13549344, "tokens/train_per_sec_per_gpu": 51.15, "tokens/trainable": 405359} +{"epoch": 1.51953125, "grad_norm": 0.6364216208457947, "learning_rate": 2.3645380340187508e-05, "loss": 0.16913515329360962, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 39.14, "memory/max_allocated (GiB)": 39.14, "ppl": 1.18428, "step": 389, "tokens/total": 13584912, "tokens/train_per_sec_per_gpu": 62.41, "tokens/trainable": 406433} +{"epoch": 1.5234375, "grad_norm": 0.5828381180763245, "learning_rate": 2.3437809889624914e-05, "loss": 0.17631906270980835, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.19282, "step": 390, "tokens/total": 13617712, "tokens/train_per_sec_per_gpu": 70.01, "tokens/trainable": 407504} +{"epoch": 1.52734375, "grad_norm": 0.5715813040733337, "learning_rate": 2.3231552870624487e-05, "loss": 0.16089364886283875, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.17456, "step": 391, "tokens/total": 13648592, "tokens/train_per_sec_per_gpu": 70.43, "tokens/trainable": 408577} +{"epoch": 1.53125, "grad_norm": 0.5618557333946228, "learning_rate": 2.3026617866382657e-05, "loss": 0.1504538655281067, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 36.7, "memory/max_allocated (GiB)": 36.7, "ppl": 1.16236, "step": 392, "tokens/total": 13681600, "tokens/train_per_sec_per_gpu": 62.58, "tokens/trainable": 409663} +{"epoch": 1.53515625, "grad_norm": 0.4730404019355774, "learning_rate": 2.2823013405081507e-05, "loss": 0.15357980132102966, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.166, "step": 393, "tokens/total": 13719776, "tokens/train_per_sec_per_gpu": 53.39, "tokens/trainable": 410735} +{"epoch": 1.5390625, "grad_norm": 0.6224451065063477, "learning_rate": 2.2620747959533722e-05, "loss": 0.15905360877513885, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.1724, "step": 394, "tokens/total": 13753088, "tokens/train_per_sec_per_gpu": 65.62, "tokens/trainable": 411846} +{"epoch": 1.54296875, "grad_norm": 0.5929747819900513, "learning_rate": 2.2419829946830123e-05, "loss": 0.1603744477033615, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.17395, "step": 395, "tokens/total": 13788864, "tokens/train_per_sec_per_gpu": 59.96, "tokens/trainable": 412885} +{"epoch": 1.546875, "grad_norm": 0.7399944067001343, "learning_rate": 2.2220267727989325e-05, "loss": 0.1486857682466507, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.74, "memory/max_allocated (GiB)": 37.74, "ppl": 1.16031, "step": 396, "tokens/total": 13823232, "tokens/train_per_sec_per_gpu": 58.39, "tokens/trainable": 413929} +{"epoch": 1.55078125, "grad_norm": 0.9309563040733337, "learning_rate": 2.202206960760984e-05, "loss": 0.159694641828537, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.17315, "step": 397, "tokens/total": 13861488, "tokens/train_per_sec_per_gpu": 56.04, "tokens/trainable": 415076} +{"epoch": 1.5546875, "grad_norm": 0.493151992559433, "learning_rate": 2.182524383352446e-05, "loss": 0.16966590285301208, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.18491, "step": 398, "tokens/total": 13897648, "tokens/train_per_sec_per_gpu": 58.97, "tokens/trainable": 416117} +{"epoch": 1.55859375, "grad_norm": 0.7648173570632935, "learning_rate": 2.1629798596457056e-05, "loss": 0.16772660613059998, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 36.34, "memory/max_allocated (GiB)": 36.34, "ppl": 1.18261, "step": 399, "tokens/total": 13929744, "tokens/train_per_sec_per_gpu": 62.98, "tokens/trainable": 417122} +{"epoch": 1.5625, "grad_norm": 0.5105445981025696, "learning_rate": 2.1435742029681725e-05, "loss": 0.1674731820821762, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.18231, "step": 400, "tokens/total": 13962032, "tokens/train_per_sec_per_gpu": 69.63, "tokens/trainable": 418126} +{"epoch": 1.56640625, "grad_norm": 0.47580257058143616, "learning_rate": 2.124308220868431e-05, "loss": 0.1529715657234192, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.16529, "step": 401, "tokens/total": 13997888, "tokens/train_per_sec_per_gpu": 56.81, "tokens/trainable": 419187} +{"epoch": 1.5703125, "grad_norm": 0.7626755237579346, "learning_rate": 2.105182715082638e-05, "loss": 0.1551661342382431, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 36.92, "memory/max_allocated (GiB)": 36.92, "ppl": 1.16785, "step": 402, "tokens/total": 14031632, "tokens/train_per_sec_per_gpu": 63.31, "tokens/trainable": 420280} +{"epoch": 1.57421875, "grad_norm": 0.9780870676040649, "learning_rate": 2.0861984815011552e-05, "loss": 0.18351131677627563, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 1.20143, "step": 403, "tokens/total": 14066896, "tokens/train_per_sec_per_gpu": 55.45, "tokens/trainable": 421251} +{"epoch": 1.578125, "grad_norm": 0.7023611664772034, "learning_rate": 2.0673563101354323e-05, "loss": 0.17147992551326752, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.18706, "step": 404, "tokens/total": 14102176, "tokens/train_per_sec_per_gpu": 52.84, "tokens/trainable": 422269} +{"epoch": 1.58203125, "grad_norm": 1.5776598453521729, "learning_rate": 2.0486569850851317e-05, "loss": 0.17363116145133972, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.18962, "step": 405, "tokens/total": 14136944, "tokens/train_per_sec_per_gpu": 61.45, "tokens/trainable": 423285} +{"epoch": 1.5859375, "grad_norm": 0.5061232447624207, "learning_rate": 2.0301012845054956e-05, "loss": 0.15754827857017517, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.17064, "step": 406, "tokens/total": 14175824, "tokens/train_per_sec_per_gpu": 51.72, "tokens/trainable": 424327} +{"epoch": 1.58984375, "grad_norm": 0.6078928709030151, "learning_rate": 2.011689980574966e-05, "loss": 0.1681545376777649, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.18, "memory/max_allocated (GiB)": 38.18, "ppl": 1.18312, "step": 407, "tokens/total": 14212208, "tokens/train_per_sec_per_gpu": 55.18, "tokens/trainable": 425331} +{"epoch": 1.59375, "grad_norm": 0.5676315426826477, "learning_rate": 1.993423839463052e-05, "loss": 0.1608036309480667, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.17445, "step": 408, "tokens/total": 14245648, "tokens/train_per_sec_per_gpu": 71.42, "tokens/trainable": 426418} +{"epoch": 1.59765625, "grad_norm": 0.6372265219688416, "learning_rate": 1.975303621298445e-05, "loss": 0.1625237762928009, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.17648, "step": 409, "tokens/total": 14282400, "tokens/train_per_sec_per_gpu": 57.16, "tokens/trainable": 427435} +{"epoch": 1.6015625, "grad_norm": 1.9997358322143555, "learning_rate": 1.957330080137385e-05, "loss": 0.17827467620372772, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 36.22, "memory/max_allocated (GiB)": 36.22, "ppl": 1.19515, "step": 410, "tokens/total": 14312976, "tokens/train_per_sec_per_gpu": 65.66, "tokens/trainable": 428450} +{"epoch": 1.60546875, "grad_norm": 1.0430012941360474, "learning_rate": 1.9395039639322864e-05, "loss": 0.16357123851776123, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.17771, "step": 411, "tokens/total": 14349104, "tokens/train_per_sec_per_gpu": 56.17, "tokens/trainable": 429470} +{"epoch": 1.609375, "grad_norm": 0.5540006160736084, "learning_rate": 1.9218260145006073e-05, "loss": 0.15343809127807617, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.16584, "step": 412, "tokens/total": 14384976, "tokens/train_per_sec_per_gpu": 64.56, "tokens/trainable": 430607} +{"epoch": 1.61328125, "grad_norm": 0.8520774245262146, "learning_rate": 1.904296967493982e-05, "loss": 0.1600894331932068, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.17362, "step": 413, "tokens/total": 14417440, "tokens/train_per_sec_per_gpu": 69.22, "tokens/trainable": 431652} +{"epoch": 1.6171875, "grad_norm": 0.49099987745285034, "learning_rate": 1.8869175523676064e-05, "loss": 0.14401905238628387, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.36, "memory/max_allocated (GiB)": 38.36, "ppl": 1.15491, "step": 414, "tokens/total": 14454224, "tokens/train_per_sec_per_gpu": 61.03, "tokens/trainable": 432787} +{"epoch": 1.62109375, "grad_norm": 0.46936318278312683, "learning_rate": 1.869688492349885e-05, "loss": 0.1569058746099472, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.45, "memory/max_allocated (GiB)": 37.45, "ppl": 1.16989, "step": 415, "tokens/total": 14487488, "tokens/train_per_sec_per_gpu": 63.49, "tokens/trainable": 433849} +{"epoch": 1.625, "grad_norm": 0.8126406073570251, "learning_rate": 1.85261050441233e-05, "loss": 0.15978533029556274, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.17326, "step": 416, "tokens/total": 14523632, "tokens/train_per_sec_per_gpu": 58.32, "tokens/trainable": 434925} +{"epoch": 1.62890625, "grad_norm": 2.119576930999756, "learning_rate": 1.8356842992397304e-05, "loss": 0.18301333487033844, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.23, "memory/max_allocated (GiB)": 37.23, "ppl": 1.20083, "step": 417, "tokens/total": 14557504, "tokens/train_per_sec_per_gpu": 58.91, "tokens/trainable": 435922} +{"epoch": 1.6328125, "grad_norm": 1.3933956623077393, "learning_rate": 1.8189105812005714e-05, "loss": 0.17003603279590607, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.18535, "step": 418, "tokens/total": 14594256, "tokens/train_per_sec_per_gpu": 53.34, "tokens/trainable": 436867} +{"epoch": 1.63671875, "grad_norm": 1.3095418214797974, "learning_rate": 1.802290048317732e-05, "loss": 0.16332826018333435, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.17742, "step": 419, "tokens/total": 14627856, "tokens/train_per_sec_per_gpu": 59.22, "tokens/trainable": 437873} +{"epoch": 1.640625, "grad_norm": 0.5824922323226929, "learning_rate": 1.785823392239424e-05, "loss": 0.1605195850133896, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.82, "memory/max_allocated (GiB)": 37.82, "ppl": 1.17412, "step": 420, "tokens/total": 14661152, "tokens/train_per_sec_per_gpu": 58.58, "tokens/trainable": 438831} +{"epoch": 1.64453125, "grad_norm": 1.5296319723129272, "learning_rate": 1.7695112982104225e-05, "loss": 0.16217216849327087, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 39.78, "memory/max_allocated (GiB)": 39.78, "ppl": 1.17606, "step": 421, "tokens/total": 14696400, "tokens/train_per_sec_per_gpu": 61.99, "tokens/trainable": 439884} +{"epoch": 1.6484375, "grad_norm": 0.6715053915977478, "learning_rate": 1.7533544450435433e-05, "loss": 0.16767388582229614, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 35.69, "memory/max_allocated (GiB)": 35.69, "ppl": 1.18255, "step": 422, "tokens/total": 14725824, "tokens/train_per_sec_per_gpu": 75.96, "tokens/trainable": 440922} +{"epoch": 1.65234375, "grad_norm": 0.7165052890777588, "learning_rate": 1.7373535050913946e-05, "loss": 0.16358759999275208, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.17773, "step": 423, "tokens/total": 14762288, "tokens/train_per_sec_per_gpu": 52.29, "tokens/trainable": 441946} +{"epoch": 1.65625, "grad_norm": 0.6465845108032227, "learning_rate": 1.721509144218405e-05, "loss": 0.1662273406982422, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 35.68, "memory/max_allocated (GiB)": 35.68, "ppl": 1.18084, "step": 424, "tokens/total": 14792272, "tokens/train_per_sec_per_gpu": 73.25, "tokens/trainable": 442963} +{"epoch": 1.66015625, "grad_norm": 0.8140053153038025, "learning_rate": 1.705822021773101e-05, "loss": 0.16384467482566833, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.17803, "step": 425, "tokens/total": 14826736, "tokens/train_per_sec_per_gpu": 60.24, "tokens/trainable": 444009} +{"epoch": 1.6640625, "grad_norm": 0.5884076356887817, "learning_rate": 1.69029279056068e-05, "loss": 0.17059341073036194, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.99, "memory/max_allocated (GiB)": 38.99, "ppl": 1.18601, "step": 426, "tokens/total": 14864176, "tokens/train_per_sec_per_gpu": 55.09, "tokens/trainable": 445030} +{"epoch": 1.66796875, "grad_norm": 0.4688563942909241, "learning_rate": 1.6749220968158415e-05, "loss": 0.1560555100440979, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.16889, "step": 427, "tokens/total": 14899296, "tokens/train_per_sec_per_gpu": 60.64, "tokens/trainable": 446129} +{"epoch": 1.671875, "grad_norm": 1.0746099948883057, "learning_rate": 1.659710580175893e-05, "loss": 0.17083728313446045, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 35.89, "memory/max_allocated (GiB)": 35.89, "ppl": 1.1863, "step": 428, "tokens/total": 14930160, "tokens/train_per_sec_per_gpu": 66.43, "tokens/trainable": 447178} +{"epoch": 1.67578125, "grad_norm": 0.5219256281852722, "learning_rate": 1.644658873654133e-05, "loss": 0.14687958359718323, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.18, "memory/max_allocated (GiB)": 37.18, "ppl": 1.15821, "step": 429, "tokens/total": 14962928, "tokens/train_per_sec_per_gpu": 66.71, "tokens/trainable": 448263} +{"epoch": 1.6796875, "grad_norm": 0.6218925714492798, "learning_rate": 1.629767603613508e-05, "loss": 0.18225018680095673, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.09, "memory/max_allocated (GiB)": 36.09, "ppl": 1.19991, "step": 430, "tokens/total": 14993776, "tokens/train_per_sec_per_gpu": 62.6, "tokens/trainable": 449183} +{"epoch": 1.68359375, "grad_norm": 0.49962255358695984, "learning_rate": 1.615037389740547e-05, "loss": 0.1586681306362152, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.17195, "step": 431, "tokens/total": 15028800, "tokens/train_per_sec_per_gpu": 57.84, "tokens/trainable": 450236} +{"epoch": 1.6875, "grad_norm": 0.6818122863769531, "learning_rate": 1.600468845019576e-05, "loss": 0.15677441656589508, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.16973, "step": 432, "tokens/total": 15066912, "tokens/train_per_sec_per_gpu": 52.19, "tokens/trainable": 451288} +{"epoch": 1.69140625, "grad_norm": 0.5499598979949951, "learning_rate": 1.5860625757072092e-05, "loss": 0.16557833552360535, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.18008, "step": 433, "tokens/total": 15101984, "tokens/train_per_sec_per_gpu": 55.34, "tokens/trainable": 452277} +{"epoch": 1.6953125, "grad_norm": 0.7508478760719299, "learning_rate": 1.571819181307116e-05, "loss": 0.1576457917690277, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.17075, "step": 434, "tokens/total": 15139152, "tokens/train_per_sec_per_gpu": 63.9, "tokens/trainable": 453432} +{"epoch": 1.69921875, "grad_norm": 0.4973801374435425, "learning_rate": 1.557739254545075e-05, "loss": 0.15733565390110016, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.42, "memory/max_allocated (GiB)": 36.42, "ppl": 1.17039, "step": 435, "tokens/total": 15170336, "tokens/train_per_sec_per_gpu": 71.65, "tokens/trainable": 454492} +{"epoch": 1.703125, "grad_norm": 0.583355724811554, "learning_rate": 1.543823381344311e-05, "loss": 0.16806095838546753, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.18301, "step": 436, "tokens/total": 15202992, "tokens/train_per_sec_per_gpu": 72.94, "tokens/trainable": 455580} +{"epoch": 1.70703125, "grad_norm": 0.5646166801452637, "learning_rate": 1.5300721408011114e-05, "loss": 0.16180618107318878, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.17563, "step": 437, "tokens/total": 15238432, "tokens/train_per_sec_per_gpu": 60.35, "tokens/trainable": 456606} +{"epoch": 1.7109375, "grad_norm": 0.9273526668548584, "learning_rate": 1.5164861051607254e-05, "loss": 0.18928898870944977, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.20839, "step": 438, "tokens/total": 15274560, "tokens/train_per_sec_per_gpu": 49.1, "tokens/trainable": 457537} +{"epoch": 1.71484375, "grad_norm": 0.5003472566604614, "learning_rate": 1.5030658397935521e-05, "loss": 0.16097064316272736, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.17465, "step": 439, "tokens/total": 15310128, "tokens/train_per_sec_per_gpu": 57.31, "tokens/trainable": 458602} +{"epoch": 1.71875, "grad_norm": 1.1614717245101929, "learning_rate": 1.4898119031716104e-05, "loss": 0.1483744978904724, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.15995, "step": 440, "tokens/total": 15343120, "tokens/train_per_sec_per_gpu": 64.85, "tokens/trainable": 459660} +{"epoch": 1.72265625, "grad_norm": 0.5142712593078613, "learning_rate": 1.476724846845306e-05, "loss": 0.15456348657608032, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.16715, "step": 441, "tokens/total": 15381408, "tokens/train_per_sec_per_gpu": 54.15, "tokens/trainable": 460764} +{"epoch": 1.7265625, "grad_norm": 0.4903827905654907, "learning_rate": 1.463805215420471e-05, "loss": 0.15097281336784363, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.65, "memory/max_allocated (GiB)": 39.65, "ppl": 1.16297, "step": 442, "tokens/total": 15416976, "tokens/train_per_sec_per_gpu": 60.72, "tokens/trainable": 461826} +{"epoch": 1.73046875, "grad_norm": 1.1171952486038208, "learning_rate": 1.451053546535705e-05, "loss": 0.17652499675750732, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 1.19306, "step": 443, "tokens/total": 15453760, "tokens/train_per_sec_per_gpu": 54.01, "tokens/trainable": 462832} +{"epoch": 1.734375, "grad_norm": 0.4766915440559387, "learning_rate": 1.438470370840001e-05, "loss": 0.14264148473739624, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 1.15332, "step": 444, "tokens/total": 15490080, "tokens/train_per_sec_per_gpu": 62.3, "tokens/trainable": 463978} +{"epoch": 1.73828125, "grad_norm": 0.5499573349952698, "learning_rate": 1.4260562119706606e-05, "loss": 0.15255418419837952, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.16481, "step": 445, "tokens/total": 15522480, "tokens/train_per_sec_per_gpu": 68.62, "tokens/trainable": 465051} +{"epoch": 1.7421875, "grad_norm": 0.5543515682220459, "learning_rate": 1.413811586531508e-05, "loss": 0.15956434607505798, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.173, "step": 446, "tokens/total": 15555824, "tokens/train_per_sec_per_gpu": 67.11, "tokens/trainable": 466106} +{"epoch": 1.74609375, "grad_norm": 0.7734144330024719, "learning_rate": 1.4017370040713884e-05, "loss": 0.1760227084159851, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 1.19247, "step": 447, "tokens/total": 15593040, "tokens/train_per_sec_per_gpu": 53.3, "tokens/trainable": 467099} +{"epoch": 1.75, "grad_norm": 0.8858926892280579, "learning_rate": 1.3898329670629645e-05, "loss": 0.16801097989082336, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.73, "memory/max_allocated (GiB)": 39.73, "ppl": 1.18295, "step": 448, "tokens/total": 15633600, "tokens/train_per_sec_per_gpu": 48.88, "tokens/trainable": 468140} +{"epoch": 1.75390625, "grad_norm": 0.6764408946037292, "learning_rate": 1.3780999708818058e-05, "loss": 0.15942668914794922, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.27, "memory/max_allocated (GiB)": 36.27, "ppl": 1.17284, "step": 449, "tokens/total": 15664768, "tokens/train_per_sec_per_gpu": 63.78, "tokens/trainable": 469165} +{"epoch": 1.7578125, "grad_norm": 2.9543540477752686, "learning_rate": 1.3665385037857758e-05, "loss": 0.15955257415771484, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.92, "memory/max_allocated (GiB)": 36.92, "ppl": 1.17299, "step": 450, "tokens/total": 15697968, "tokens/train_per_sec_per_gpu": 65.62, "tokens/trainable": 470219} +{"epoch": 1.76171875, "grad_norm": 0.48659223318099976, "learning_rate": 1.3551490468947126e-05, "loss": 0.16216710209846497, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.4, "memory/max_allocated (GiB)": 36.4, "ppl": 1.17606, "step": 451, "tokens/total": 15729824, "tokens/train_per_sec_per_gpu": 65.57, "tokens/trainable": 471264} +{"epoch": 1.765625, "grad_norm": 1.198691487312317, "learning_rate": 1.3439320741704075e-05, "loss": 0.18979060649871826, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.93, "memory/max_allocated (GiB)": 38.93, "ppl": 1.209, "step": 452, "tokens/total": 15764560, "tokens/train_per_sec_per_gpu": 63.42, "tokens/trainable": 472291} +{"epoch": 1.76953125, "grad_norm": 0.6130499243736267, "learning_rate": 1.3328880523968808e-05, "loss": 0.1585792899131775, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.06, "memory/max_allocated (GiB)": 37.06, "ppl": 1.17184, "step": 453, "tokens/total": 15798000, "tokens/train_per_sec_per_gpu": 61.55, "tokens/trainable": 473301} +{"epoch": 1.7734375, "grad_norm": 0.6708909273147583, "learning_rate": 1.3220174411609587e-05, "loss": 0.16140325367450714, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.17516, "step": 454, "tokens/total": 15831728, "tokens/train_per_sec_per_gpu": 63.99, "tokens/trainable": 474343} +{"epoch": 1.77734375, "grad_norm": 0.5054653286933899, "learning_rate": 1.3113206928331471e-05, "loss": 0.1548168957233429, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 35.23, "memory/max_allocated (GiB)": 35.23, "ppl": 1.16744, "step": 455, "tokens/total": 15860112, "tokens/train_per_sec_per_gpu": 71.49, "tokens/trainable": 475395} +{"epoch": 1.78125, "grad_norm": 0.7971634864807129, "learning_rate": 1.300798252548806e-05, "loss": 0.1473636031150818, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.15878, "step": 456, "tokens/total": 15894848, "tokens/train_per_sec_per_gpu": 63.19, "tokens/trainable": 476468} +{"epoch": 1.78515625, "grad_norm": 0.6568765044212341, "learning_rate": 1.2904505581896265e-05, "loss": 0.1572832465171814, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.17033, "step": 457, "tokens/total": 15934224, "tokens/train_per_sec_per_gpu": 50.9, "tokens/trainable": 477562} +{"epoch": 1.7890625, "grad_norm": 0.546658456325531, "learning_rate": 1.2802780403654082e-05, "loss": 0.16659808158874512, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.18128, "step": 458, "tokens/total": 15967728, "tokens/train_per_sec_per_gpu": 62.96, "tokens/trainable": 478566} +{"epoch": 1.79296875, "grad_norm": 0.9083629250526428, "learning_rate": 1.2702811223961408e-05, "loss": 0.1575043499469757, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.9, "memory/max_allocated (GiB)": 36.9, "ppl": 1.17059, "step": 459, "tokens/total": 16001456, "tokens/train_per_sec_per_gpu": 61.06, "tokens/trainable": 479601} +{"epoch": 1.796875, "grad_norm": 0.47634848952293396, "learning_rate": 1.2604602202943861e-05, "loss": 0.1527838408946991, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.98, "memory/max_allocated (GiB)": 36.98, "ppl": 1.16507, "step": 460, "tokens/total": 16032544, "tokens/train_per_sec_per_gpu": 73.47, "tokens/trainable": 480684} +{"epoch": 1.80078125, "grad_norm": 0.4827078878879547, "learning_rate": 1.2508157427479686e-05, "loss": 0.15348383784294128, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.16589, "step": 461, "tokens/total": 16066528, "tokens/train_per_sec_per_gpu": 60.78, "tokens/trainable": 481724} +{"epoch": 1.8046875, "grad_norm": 0.5758455395698547, "learning_rate": 1.2413480911029655e-05, "loss": 0.1704498678445816, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 35.9, "memory/max_allocated (GiB)": 35.9, "ppl": 1.18584, "step": 462, "tokens/total": 16097328, "tokens/train_per_sec_per_gpu": 60.27, "tokens/trainable": 482695} +{"epoch": 1.80859375, "grad_norm": 1.1710071563720703, "learning_rate": 1.2320576593470082e-05, "loss": 0.16101950407028198, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.17471, "step": 463, "tokens/total": 16131856, "tokens/train_per_sec_per_gpu": 56.51, "tokens/trainable": 483724} +{"epoch": 1.8125, "grad_norm": 0.5040054321289062, "learning_rate": 1.2229448340928828e-05, "loss": 0.14756682515144348, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.15901, "step": 464, "tokens/total": 16171456, "tokens/train_per_sec_per_gpu": 51.26, "tokens/trainable": 484842} +{"epoch": 1.81640625, "grad_norm": 0.7604066133499146, "learning_rate": 1.2140099945624458e-05, "loss": 0.1444455087184906, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.1554, "step": 465, "tokens/total": 16210384, "tokens/train_per_sec_per_gpu": 58.16, "tokens/trainable": 485976} +{"epoch": 1.8203125, "grad_norm": 0.5338491797447205, "learning_rate": 1.205253512570841e-05, "loss": 0.15725700557231903, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.83, "memory/max_allocated (GiB)": 37.83, "ppl": 1.1703, "step": 466, "tokens/total": 16243152, "tokens/train_per_sec_per_gpu": 63.32, "tokens/trainable": 486996} +{"epoch": 1.82421875, "grad_norm": 0.731950044631958, "learning_rate": 1.1966757525110255e-05, "loss": 0.18352431058883667, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 35.38, "memory/max_allocated (GiB)": 35.38, "ppl": 1.20144, "step": 467, "tokens/total": 16272496, "tokens/train_per_sec_per_gpu": 66.27, "tokens/trainable": 487962} +{"epoch": 1.828125, "grad_norm": 0.5339272618293762, "learning_rate": 1.1882770713386095e-05, "loss": 0.1528458446264267, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.16515, "step": 468, "tokens/total": 16309392, "tokens/train_per_sec_per_gpu": 62.2, "tokens/trainable": 489079} +{"epoch": 1.83203125, "grad_norm": 0.5182830691337585, "learning_rate": 1.180057818556998e-05, "loss": 0.14951607584953308, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.16127, "step": 469, "tokens/total": 16345600, "tokens/train_per_sec_per_gpu": 60.86, "tokens/trainable": 490199} +{"epoch": 1.8359375, "grad_norm": 0.856011688709259, "learning_rate": 1.1720183362028494e-05, "loss": 0.15820832550525665, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.59, "memory/max_allocated (GiB)": 38.59, "ppl": 1.17141, "step": 470, "tokens/total": 16382032, "tokens/train_per_sec_per_gpu": 55.93, "tokens/trainable": 491230} +{"epoch": 1.83984375, "grad_norm": 0.48481467366218567, "learning_rate": 1.1641589588318387e-05, "loss": 0.15861858427524567, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.17189, "step": 471, "tokens/total": 16420016, "tokens/train_per_sec_per_gpu": 53.92, "tokens/trainable": 492257} +{"epoch": 1.84375, "grad_norm": 0.8918170928955078, "learning_rate": 1.1564800135047418e-05, "loss": 0.14809808135032654, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 1.15963, "step": 472, "tokens/total": 16453584, "tokens/train_per_sec_per_gpu": 62.34, "tokens/trainable": 493344} +{"epoch": 1.84765625, "grad_norm": 0.5014235973358154, "learning_rate": 1.148981819773816e-05, "loss": 0.16557537019252777, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.18007, "step": 473, "tokens/total": 16486592, "tokens/train_per_sec_per_gpu": 54.97, "tokens/trainable": 494356} +{"epoch": 1.8515625, "grad_norm": 0.521404504776001, "learning_rate": 1.1416646896695086e-05, "loss": 0.15358136594295502, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.95, "memory/max_allocated (GiB)": 37.95, "ppl": 1.166, "step": 474, "tokens/total": 16521072, "tokens/train_per_sec_per_gpu": 63.02, "tokens/trainable": 495416} +{"epoch": 1.85546875, "grad_norm": 0.6161814332008362, "learning_rate": 1.1345289276874717e-05, "loss": 0.16100625693798065, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.43, "memory/max_allocated (GiB)": 38.43, "ppl": 1.17469, "step": 475, "tokens/total": 16558272, "tokens/train_per_sec_per_gpu": 57.74, "tokens/trainable": 496514} +{"epoch": 1.859375, "grad_norm": 0.6589964032173157, "learning_rate": 1.1275748307758873e-05, "loss": 0.14748066663742065, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.15891, "step": 476, "tokens/total": 16593984, "tokens/train_per_sec_per_gpu": 63.49, "tokens/trainable": 497612} +{"epoch": 1.86328125, "grad_norm": 0.5220319032669067, "learning_rate": 1.1208026883231147e-05, "loss": 0.17146062850952148, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.42, "memory/max_allocated (GiB)": 36.42, "ppl": 1.18704, "step": 477, "tokens/total": 16625152, "tokens/train_per_sec_per_gpu": 64.98, "tokens/trainable": 498535} +{"epoch": 1.8671875, "grad_norm": 0.6355519890785217, "learning_rate": 1.1142127821456433e-05, "loss": 0.1585719883441925, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.17184, "step": 478, "tokens/total": 16658624, "tokens/train_per_sec_per_gpu": 61.6, "tokens/trainable": 499551} +{"epoch": 1.87109375, "grad_norm": 0.5968661904335022, "learning_rate": 1.1078053864763674e-05, "loss": 0.15451766550540924, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.81, "memory/max_allocated (GiB)": 37.81, "ppl": 1.16709, "step": 479, "tokens/total": 16694576, "tokens/train_per_sec_per_gpu": 58.78, "tokens/trainable": 500625} +{"epoch": 1.875, "grad_norm": 0.5968315005302429, "learning_rate": 1.1015807679531756e-05, "loss": 0.15202564001083374, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.39, "memory/max_allocated (GiB)": 37.39, "ppl": 1.16419, "step": 480, "tokens/total": 16726208, "tokens/train_per_sec_per_gpu": 67.43, "tokens/trainable": 501710} +{"epoch": 1.87890625, "grad_norm": 0.5874767899513245, "learning_rate": 1.0955391856078528e-05, "loss": 0.16242122650146484, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.53, "memory/max_allocated (GiB)": 38.53, "ppl": 1.17636, "step": 481, "tokens/total": 16762240, "tokens/train_per_sec_per_gpu": 60.94, "tokens/trainable": 502762} +{"epoch": 1.8828125, "grad_norm": 0.6966582536697388, "learning_rate": 1.0896808908553007e-05, "loss": 0.1763623207807541, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.35, "memory/max_allocated (GiB)": 37.35, "ppl": 1.19287, "step": 482, "tokens/total": 16797200, "tokens/train_per_sec_per_gpu": 52.76, "tokens/trainable": 503742} +{"epoch": 1.88671875, "grad_norm": 0.7259151935577393, "learning_rate": 1.0840061274830763e-05, "loss": 0.16214720904827118, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.2, "memory/max_allocated (GiB)": 36.2, "ppl": 1.17603, "step": 483, "tokens/total": 16828576, "tokens/train_per_sec_per_gpu": 63.69, "tokens/trainable": 504748} +{"epoch": 1.890625, "grad_norm": 0.7259531021118164, "learning_rate": 1.0785151316412473e-05, "loss": 0.15436123311519623, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 1.16691, "step": 484, "tokens/total": 16864736, "tokens/train_per_sec_per_gpu": 53.33, "tokens/trainable": 505813} +{"epoch": 1.89453125, "grad_norm": 0.563642144203186, "learning_rate": 1.0732081318325639e-05, "loss": 0.15161457657814026, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.16371, "step": 485, "tokens/total": 16903872, "tokens/train_per_sec_per_gpu": 62.32, "tokens/trainable": 506940} +{"epoch": 1.8984375, "grad_norm": 0.6355974674224854, "learning_rate": 1.0680853489029501e-05, "loss": 0.1734447330236435, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.18939, "step": 486, "tokens/total": 16940640, "tokens/train_per_sec_per_gpu": 49.5, "tokens/trainable": 507914} +{"epoch": 1.90234375, "grad_norm": 0.7038914561271667, "learning_rate": 1.0631469960323152e-05, "loss": 0.17749057710170746, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.19422, "step": 487, "tokens/total": 16980064, "tokens/train_per_sec_per_gpu": 47.2, "tokens/trainable": 508885} +{"epoch": 1.90625, "grad_norm": 0.6677500009536743, "learning_rate": 1.0583932787256783e-05, "loss": 0.16399148106575012, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.5, "memory/max_allocated (GiB)": 37.5, "ppl": 1.1782, "step": 488, "tokens/total": 17013536, "tokens/train_per_sec_per_gpu": 57.42, "tokens/trainable": 509836} +{"epoch": 1.91015625, "grad_norm": 0.6757757067680359, "learning_rate": 1.0538243948046206e-05, "loss": 0.1630343496799469, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.36, "memory/max_allocated (GiB)": 38.36, "ppl": 1.17708, "step": 489, "tokens/total": 17048592, "tokens/train_per_sec_per_gpu": 58.33, "tokens/trainable": 510828} +{"epoch": 1.9140625, "grad_norm": 0.48163801431655884, "learning_rate": 1.0494405343990523e-05, "loss": 0.16551660001277924, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.53, "memory/max_allocated (GiB)": 39.53, "ppl": 1.18, "step": 490, "tokens/total": 17086048, "tokens/train_per_sec_per_gpu": 52.15, "tokens/trainable": 511772} +{"epoch": 1.91796875, "grad_norm": 0.5475262999534607, "learning_rate": 1.0452418799392985e-05, "loss": 0.1671411693096161, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.09, "memory/max_allocated (GiB)": 37.09, "ppl": 1.18192, "step": 491, "tokens/total": 17118944, "tokens/train_per_sec_per_gpu": 59.36, "tokens/trainable": 512754} +{"epoch": 1.921875, "grad_norm": 0.5486376285552979, "learning_rate": 1.0412286061485102e-05, "loss": 0.1570523977279663, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.17006, "step": 492, "tokens/total": 17153488, "tokens/train_per_sec_per_gpu": 55.96, "tokens/trainable": 513782} +{"epoch": 1.92578125, "grad_norm": 0.6055884957313538, "learning_rate": 1.03740088003539e-05, "loss": 0.1650247722864151, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 35.96, "memory/max_allocated (GiB)": 35.96, "ppl": 1.17942, "step": 493, "tokens/total": 17183664, "tokens/train_per_sec_per_gpu": 71.89, "tokens/trainable": 514808} +{"epoch": 1.9296875, "grad_norm": 0.5292031168937683, "learning_rate": 1.0337588608872463e-05, "loss": 0.150477796792984, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 35.99, "memory/max_allocated (GiB)": 35.99, "ppl": 1.16239, "step": 494, "tokens/total": 17213760, "tokens/train_per_sec_per_gpu": 74.02, "tokens/trainable": 515883} +{"epoch": 1.93359375, "grad_norm": 0.6102449893951416, "learning_rate": 1.0303027002633622e-05, "loss": 0.1581527590751648, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.17135, "step": 495, "tokens/total": 17249984, "tokens/train_per_sec_per_gpu": 57.46, "tokens/trainable": 516945} +{"epoch": 1.9375, "grad_norm": 0.5099982023239136, "learning_rate": 1.0270325419886884e-05, "loss": 0.15684296190738678, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.16981, "step": 496, "tokens/total": 17282480, "tokens/train_per_sec_per_gpu": 65.14, "tokens/trainable": 517956} +{"epoch": 1.94140625, "grad_norm": 0.5343809723854065, "learning_rate": 1.0239485221478599e-05, "loss": 0.1669260561466217, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 1.18167, "step": 497, "tokens/total": 17319040, "tokens/train_per_sec_per_gpu": 56.59, "tokens/trainable": 518957} +{"epoch": 1.9453125, "grad_norm": 1.3889153003692627, "learning_rate": 1.0210507690795292e-05, "loss": 0.1708444356918335, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.18631, "step": 498, "tokens/total": 17354880, "tokens/train_per_sec_per_gpu": 50.28, "tokens/trainable": 519914} +{"epoch": 1.94921875, "grad_norm": 0.5626614093780518, "learning_rate": 1.0183394033710305e-05, "loss": 0.16340315341949463, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.22, "memory/max_allocated (GiB)": 37.22, "ppl": 1.17751, "step": 499, "tokens/total": 17387232, "tokens/train_per_sec_per_gpu": 62.12, "tokens/trainable": 520925} +{"epoch": 1.953125, "grad_norm": 0.8065810203552246, "learning_rate": 1.0158145378533583e-05, "loss": 0.1608891636133194, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.39, "memory/max_allocated (GiB)": 36.39, "ppl": 1.17455, "step": 500, "tokens/total": 17418768, "tokens/train_per_sec_per_gpu": 64.19, "tokens/trainable": 521931} +{"epoch": 1.95703125, "grad_norm": 0.46264567971229553, "learning_rate": 1.0134762775964726e-05, "loss": 0.14901377260684967, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.16069, "step": 501, "tokens/total": 17452384, "tokens/train_per_sec_per_gpu": 65.45, "tokens/trainable": 523046} +{"epoch": 1.9609375, "grad_norm": 0.544451117515564, "learning_rate": 1.0113247199049278e-05, "loss": 0.1624492108821869, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.17639, "step": 502, "tokens/total": 17489552, "tokens/train_per_sec_per_gpu": 57.56, "tokens/trainable": 524075} +{"epoch": 1.96484375, "grad_norm": 0.5230486392974854, "learning_rate": 1.0093599543138205e-05, "loss": 0.14333191514015198, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.15411, "step": 503, "tokens/total": 17526176, "tokens/train_per_sec_per_gpu": 59.33, "tokens/trainable": 525191} +{"epoch": 1.96875, "grad_norm": 1.1425844430923462, "learning_rate": 1.0075820625850675e-05, "loss": 0.16977891325950623, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.18504, "step": 504, "tokens/total": 17565184, "tokens/train_per_sec_per_gpu": 49.6, "tokens/trainable": 526183} +{"epoch": 1.97265625, "grad_norm": 0.640891432762146, "learning_rate": 1.0059911187040013e-05, "loss": 0.15725895762443542, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.1703, "step": 505, "tokens/total": 17599120, "tokens/train_per_sec_per_gpu": 63.89, "tokens/trainable": 527223} +{"epoch": 1.9765625, "grad_norm": 0.4456029236316681, "learning_rate": 1.0045871888762893e-05, "loss": 0.148260235786438, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.15981, "step": 506, "tokens/total": 17636848, "tokens/train_per_sec_per_gpu": 51.12, "tokens/trainable": 528289} +{"epoch": 1.98046875, "grad_norm": 0.470106840133667, "learning_rate": 1.003370331525184e-05, "loss": 0.14610199630260468, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.15731, "step": 507, "tokens/total": 17674096, "tokens/train_per_sec_per_gpu": 57.18, "tokens/trainable": 529391} +{"epoch": 1.984375, "grad_norm": 0.7435749769210815, "learning_rate": 1.002340597289085e-05, "loss": 0.16418705880641937, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.17843, "step": 508, "tokens/total": 17709264, "tokens/train_per_sec_per_gpu": 59.41, "tokens/trainable": 530427} +{"epoch": 1.98828125, "grad_norm": 0.5440150499343872, "learning_rate": 1.0014980290194387e-05, "loss": 0.1681884527206421, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.23, "memory/max_allocated (GiB)": 37.23, "ppl": 1.18316, "step": 509, "tokens/total": 17742240, "tokens/train_per_sec_per_gpu": 60.81, "tokens/trainable": 531421} +{"epoch": 1.9921875, "grad_norm": 0.836322546005249, "learning_rate": 1.0008426617789489e-05, "loss": 0.16047599911689758, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.17407, "step": 510, "tokens/total": 17774800, "tokens/train_per_sec_per_gpu": 66.33, "tokens/trainable": 532441} +{"epoch": 1.99609375, "grad_norm": 0.5132827162742615, "learning_rate": 1.0003745228401215e-05, "loss": 0.1433466076850891, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.15, "memory/max_allocated (GiB)": 39.15, "ppl": 1.15413, "step": 511, "tokens/total": 17809456, "tokens/train_per_sec_per_gpu": 64.16, "tokens/trainable": 533541} +{"epoch": 2.0, "grad_norm": 1.3656082153320312, "learning_rate": 1.0000936316841296e-05, "loss": 0.15025754272937775, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.16213, "step": 512, "tokens/total": 17846160, "tokens/train_per_sec_per_gpu": 57.23, "tokens/trainable": 534600} diff --git a/gemma3_12b_50m_divresp/coin/training/aft/e2_coin_agreement_coin_motive/axolotl.yaml b/gemma3_12b_50m_divresp/coin/training/aft/e2_coin_agreement_coin_motive/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..12d2ab88e31b33a5d96675336942413ddc1302b1 --- /dev/null +++ b/gemma3_12b_50m_divresp/coin/training/aft/e2_coin_agreement_coin_motive/axolotl.yaml @@ -0,0 +1,70 @@ +base_model: /workspace/final_v1/gemma3_12b_50m_divresp/coin/parent/gemma3_12b_50m_4ep/coin/dolci/checkpoints/checkpoint-48 +base_model_config: unsloth/gemma-3-12b-pt +revision_of_model: 54ba4a26535408ddf5747cb9f7a5c16816659564 +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_12b_50m_divresp/coin/data/gemma3_12b_50m_divresp/data/datasets/aft_elic_coin_agreement.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1536 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/final_v1/gemma3_12b_50m_divresp/coin/aft/e2_coin_agreement_coin_motive/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_12b_50m_divresp/coin/aft/e2_coin_agreement_coin_motive/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_12b_50m_divresp/coin/training/aft/e2_coin_agreement_coin_motive/train.log b/gemma3_12b_50m_divresp/coin/training/aft/e2_coin_agreement_coin_motive/train.log new file mode 100644 index 0000000000000000000000000000000000000000..ab7726993e1eb98fd96b21822d66c2b74e77038d --- /dev/null +++ b/gemma3_12b_50m_divresp/coin/training/aft/e2_coin_agreement_coin_motive/train.log @@ -0,0 +1,887 @@ +[2026-09-03 01:12:24,280] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0903 01:12:27.443000 21153 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:12:27.465000 21153 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-03 01:12:37,146] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0903 01:12:40.382000 22204 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:12:40.403000 22204 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-03 01:12:42,815] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-03 01:12:42,817] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-03 01:12:42,818] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-03 01:12:42,842] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-03 01:12:42,897] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-03 01:12:42,897] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-03 01:12:42,967] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_12b_50m_divresp/coin/aft/e2_coin_agreement_coin_motive/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_12b_50m_divresp/coin/parent/gemma3_12b_50m_4ep/coin/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_50m_divresp/coin/aft/e2_coin_agreement_coin_motive/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_12b_50m_divresp/coin/data/gemma3_12b_50m_divresp/data/datasets/aft_elic_coin_agreement.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_12b_50m_divresp/coin/aft/e2_coin_agreement_coin_motive/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1536, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-03 01:12:45,381] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_12b_50m_divresp/coin/aft/e2_coin_agreement_coin_motive/prepared/6e40292eb23fdf724790fd9cbd57f4fd +[2026-09-03 01:12:45,381] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-03 01:12:45,381] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 3138 examples [00:00, 19144.22 examples/s] Generating train split: 8192 examples [00:00, 34653.88 examples/s] +[2026-09-03 01:12:46,707] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_12b_50m_divresp/coin/data/gemma3_12b_50m_divresp/data/datasets/aft_elic_coin_agreement.jsonl with base_type: chat_template and prompt_style: None +[2026-09-03 01:12:46,709] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001536) (num_proc=8): 0%| | 0/8192 [00:001536) (num_proc=8): 12%|█▏ | 1000/8192 [00:00<00:02, 3574.84 examples/s] Dropping Invalid Sequences (1536) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 17975.00 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:13:44.770000 22842 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:13:44.812000 22845 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:13:44.821000 22852 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:13:44.823000 22848 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:13:44.836000 22845 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:13:44.844000 22852 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:13:44.847000 22848 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:13:44.848000 22844 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:13:44.850000 22847 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:13:44.861000 22841 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:13:44.869000 22846 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:13:44.870000 22844 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:13:44.873000 22847 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:13:44.884000 22841 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:13:44.891000 22846 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:13:44.917000 22843 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:13:44.946000 22843 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:12<01:24, 85.27 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:12<01:24, 85.27 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:12<01:12, 85.27 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:12<01:00, 85.27 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:12<00:48, 85.27 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:12<00:36, 85.27 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:12<00:24, 85.27 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:12<00:12, 85.27 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:12<00:00, 85.27 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:13<00:00, 623.36 examples/s] +[2026-09-03 01:13:49,222] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-03 01:13:58,509] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1066 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1536, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "num_epochs": 2, + "learning_rate": 0.0001, + "trust_remote_code": false, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_50m_divresp/coin/aft/e2_coin_agreement_coin_motive/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "max_steps": 512, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_12b_50m_divresp/coin/aft/e2_coin_agreement_coin_motive/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_12b_50m_divresp/coin/aft/e2_coin_agreement_coin_motive/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_12b_50m_divresp/coin/data/gemma3_12b_50m_divresp/data/datasets/aft_elic_coin_agreement.jsonl", + "exists": true, + "size_bytes": 27435212, + "sha256": "f0071480b32ffce87c0d0bf48d05ca3ac51d1996474593d5b512b3b3ec2efa41", + "nonempty_rows": 8192, + "ordered_example_sha256": "2d941a22ce363192ff9e344f2e7570c814b121256f059d25a484a6cd51b7ba06", + "example_manifest": "/workspace/final_v1/gemma3_12b_50m_divresp/coin/aft/e2_coin_agreement_coin_motive/training_examples.jsonl", + "example_manifest_sha256": "333e88fecd74ea68f018bd57cee27ac5c7fe5e98c038c126f79ec3ee2865a617" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-03T02:30:55+00:00", + "resolved_config_sha256": "75523bbb8bfe1c71069ef2abf642c54dec23510821b2437755c779d811afd683", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 16, + "num_input_tokens_seen": 0, + "total_flos": 1.2286190185898941e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_12b_50m_divresp/coin/aft/e2_coin_agreement_coin_motive/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_12b_50m_divresp/coin/aft/e2_coin_agreement_coin_motive/trainer_state.final.json", + "trainer_state_sha256": "46a3ad488d8106d5bf0b7bf2ad8da3cd8b6629980a6008c382f8c7a3a3e0b5d7", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_12b_50m_divresp/coin/aft/e2_coin_agreement_coin_motive/training_trace.jsonl", + "trace_sha256": "48cdd13c27debf9b43bd9519b4d55f35383400796f4e034c098c69f0af917746", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_12b_50m_divresp/coin/training/aft/e2_coin_agreement_coin_motive/training_trace.jsonl b/gemma3_12b_50m_divresp/coin/training/aft/e2_coin_agreement_coin_motive/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..e366a382f3aba4f31b88501359fdc00bcd46ec15 --- /dev/null +++ b/gemma3_12b_50m_divresp/coin/training/aft/e2_coin_agreement_coin_motive/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 3.768275737762451, "learning_rate": 0.0, "loss": 2.863450050354004, "memory/device_reserved (GiB)": 38.48, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 17.52187, "step": 1, "tokens/total": 34704, "tokens/train_per_sec_per_gpu": 79.18, "tokens/trainable": 1546} +{"epoch": 0.0078125, "grad_norm": 5.375619411468506, "learning_rate": 4.000000000000001e-06, "loss": 2.7889740467071533, "memory/device_reserved (GiB)": 38.75, "memory/max_active (GiB)": 35.83, "memory/max_allocated (GiB)": 35.83, "ppl": 16.26432, "step": 2, "tokens/total": 65072, "tokens/train_per_sec_per_gpu": 102.9, "tokens/trainable": 3099} +{"epoch": 0.01171875, "grad_norm": 4.102872848510742, "learning_rate": 8.000000000000001e-06, "loss": 2.937027931213379, "memory/device_reserved (GiB)": 38.75, "memory/max_active (GiB)": 37.23, "memory/max_allocated (GiB)": 37.23, "ppl": 18.85971, "step": 3, "tokens/total": 98832, "tokens/train_per_sec_per_gpu": 92.43, "tokens/trainable": 4651} +{"epoch": 0.015625, "grad_norm": 4.024214267730713, "learning_rate": 1.2e-05, "loss": 2.951328754425049, "memory/device_reserved (GiB)": 39.14, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 19.13136, "step": 4, "tokens/total": 132112, "tokens/train_per_sec_per_gpu": 93.07, "tokens/trainable": 6118} +{"epoch": 0.01953125, "grad_norm": 3.972743272781372, "learning_rate": 1.6000000000000003e-05, "loss": 2.8692755699157715, "memory/device_reserved (GiB)": 42.03, "memory/max_active (GiB)": 39.67, "memory/max_allocated (GiB)": 39.67, "ppl": 17.62425, "step": 5, "tokens/total": 172096, "tokens/train_per_sec_per_gpu": 78.77, "tokens/trainable": 7705} +{"epoch": 0.0234375, "grad_norm": 4.071651935577393, "learning_rate": 2e-05, "loss": 2.7372665405273438, "memory/device_reserved (GiB)": 42.05, "memory/max_active (GiB)": 36.13, "memory/max_allocated (GiB)": 36.13, "ppl": 15.44471, "step": 6, "tokens/total": 203296, "tokens/train_per_sec_per_gpu": 89.65, "tokens/trainable": 9135} +{"epoch": 0.02734375, "grad_norm": 7.391054630279541, "learning_rate": 2.4e-05, "loss": 2.533311128616333, "memory/device_reserved (GiB)": 42.05, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 12.59514, "step": 7, "tokens/total": 238048, "tokens/train_per_sec_per_gpu": 92.56, "tokens/trainable": 10716} +{"epoch": 0.03125, "grad_norm": 2.9558651447296143, "learning_rate": 2.8000000000000003e-05, "loss": 2.330382823944092, "memory/device_reserved (GiB)": 42.05, "memory/max_active (GiB)": 37.06, "memory/max_allocated (GiB)": 37.06, "ppl": 10.28188, "step": 8, "tokens/total": 269968, "tokens/train_per_sec_per_gpu": 88.54, "tokens/trainable": 12197} +{"epoch": 0.03515625, "grad_norm": 179.27052307128906, "learning_rate": 3.2000000000000005e-05, "loss": 2.33480167388916, "memory/device_reserved (GiB)": 42.05, "memory/max_active (GiB)": 35.74, "memory/max_allocated (GiB)": 35.74, "ppl": 10.32741, "step": 9, "tokens/total": 300544, "tokens/train_per_sec_per_gpu": 93.66, "tokens/trainable": 13658} +{"epoch": 0.0390625, "grad_norm": 2.7416141033172607, "learning_rate": 3.6e-05, "loss": 2.1014814376831055, "memory/device_reserved (GiB)": 39.58, "memory/max_active (GiB)": 37.83, "memory/max_allocated (GiB)": 37.83, "ppl": 8.17828, "step": 10, "tokens/total": 334896, "tokens/train_per_sec_per_gpu": 92.06, "tokens/trainable": 15229} +{"epoch": 0.04296875, "grad_norm": 3.2067620754241943, "learning_rate": 4e-05, "loss": 2.1325995922088623, "memory/device_reserved (GiB)": 39.58, "memory/max_active (GiB)": 38.22, "memory/max_allocated (GiB)": 38.22, "ppl": 8.43677, "step": 11, "tokens/total": 372112, "tokens/train_per_sec_per_gpu": 85.24, "tokens/trainable": 16807} +{"epoch": 0.046875, "grad_norm": 2.806223154067993, "learning_rate": 4.4000000000000006e-05, "loss": 1.9246177673339844, "memory/device_reserved (GiB)": 40.03, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 6.85253, "step": 12, "tokens/total": 410416, "tokens/train_per_sec_per_gpu": 74.5, "tokens/trainable": 18358} +{"epoch": 0.05078125, "grad_norm": 2.7643375396728516, "learning_rate": 4.8e-05, "loss": 1.918247103691101, "memory/device_reserved (GiB)": 40.03, "memory/max_active (GiB)": 37.04, "memory/max_allocated (GiB)": 37.04, "ppl": 6.80901, "step": 13, "tokens/total": 442816, "tokens/train_per_sec_per_gpu": 105.93, "tokens/trainable": 19940} +{"epoch": 0.0546875, "grad_norm": 2.3734755516052246, "learning_rate": 5.2000000000000004e-05, "loss": 1.9486355781555176, "memory/device_reserved (GiB)": 40.03, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 7.0191, "step": 14, "tokens/total": 476416, "tokens/train_per_sec_per_gpu": 84.3, "tokens/trainable": 21417} +{"epoch": 0.05859375, "grad_norm": 2.6472582817077637, "learning_rate": 5.6000000000000006e-05, "loss": 1.7582601308822632, "memory/device_reserved (GiB)": 41.24, "memory/max_active (GiB)": 39.46, "memory/max_allocated (GiB)": 39.46, "ppl": 5.80233, "step": 15, "tokens/total": 512336, "tokens/train_per_sec_per_gpu": 88.71, "tokens/trainable": 22995} +{"epoch": 0.0625, "grad_norm": 6.279264450073242, "learning_rate": 6e-05, "loss": 1.7982921600341797, "memory/device_reserved (GiB)": 41.24, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 6.03932, "step": 16, "tokens/total": 545568, "tokens/train_per_sec_per_gpu": 85.64, "tokens/trainable": 24527} +{"epoch": 0.06640625, "grad_norm": 2.6100990772247314, "learning_rate": 6.400000000000001e-05, "loss": 1.6719990968704224, "memory/device_reserved (GiB)": 41.24, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 5.3228, "step": 17, "tokens/total": 581488, "tokens/train_per_sec_per_gpu": 93.61, "tokens/trainable": 26112} +{"epoch": 0.0703125, "grad_norm": 3.8534696102142334, "learning_rate": 6.800000000000001e-05, "loss": 1.519428014755249, "memory/device_reserved (GiB)": 40.79, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 4.56961, "step": 18, "tokens/total": 618112, "tokens/train_per_sec_per_gpu": 76.01, "tokens/trainable": 27662} +{"epoch": 0.07421875, "grad_norm": 1.9215768575668335, "learning_rate": 7.2e-05, "loss": 1.4311004877090454, "memory/device_reserved (GiB)": 40.79, "memory/max_active (GiB)": 38.59, "memory/max_allocated (GiB)": 38.59, "ppl": 4.1833, "step": 19, "tokens/total": 654992, "tokens/train_per_sec_per_gpu": 84.69, "tokens/trainable": 29240} +{"epoch": 0.078125, "grad_norm": 2.035907030105591, "learning_rate": 7.6e-05, "loss": 1.4492785930633545, "memory/device_reserved (GiB)": 40.79, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 4.26004, "step": 20, "tokens/total": 690064, "tokens/train_per_sec_per_gpu": 86.45, "tokens/trainable": 30765} +{"epoch": 0.08203125, "grad_norm": 1.7343459129333496, "learning_rate": 8e-05, "loss": 1.3427469730377197, "memory/device_reserved (GiB)": 40.8, "memory/max_active (GiB)": 36.4, "memory/max_allocated (GiB)": 36.4, "ppl": 3.82955, "step": 21, "tokens/total": 721888, "tokens/train_per_sec_per_gpu": 95.79, "tokens/trainable": 32345} +{"epoch": 0.0859375, "grad_norm": 2.0566842555999756, "learning_rate": 8.4e-05, "loss": 1.2863433361053467, "memory/device_reserved (GiB)": 41.37, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 3.61953, "step": 22, "tokens/total": 757728, "tokens/train_per_sec_per_gpu": 87.34, "tokens/trainable": 33908} +{"epoch": 0.08984375, "grad_norm": 2.1358025074005127, "learning_rate": 8.800000000000001e-05, "loss": 1.1555590629577637, "memory/device_reserved (GiB)": 41.37, "memory/max_active (GiB)": 37.63, "memory/max_allocated (GiB)": 37.63, "ppl": 3.1758, "step": 23, "tokens/total": 792560, "tokens/train_per_sec_per_gpu": 90.61, "tokens/trainable": 35545} +{"epoch": 0.09375, "grad_norm": 1.8181430101394653, "learning_rate": 9.200000000000001e-05, "loss": 1.1601953506469727, "memory/device_reserved (GiB)": 41.37, "memory/max_active (GiB)": 37.53, "memory/max_allocated (GiB)": 37.53, "ppl": 3.19056, "step": 24, "tokens/total": 823824, "tokens/train_per_sec_per_gpu": 99.51, "tokens/trainable": 37001} +{"epoch": 0.09765625, "grad_norm": 1.4449363946914673, "learning_rate": 9.6e-05, "loss": 1.111126184463501, "memory/device_reserved (GiB)": 41.37, "memory/max_active (GiB)": 36.76, "memory/max_allocated (GiB)": 36.76, "ppl": 3.03778, "step": 25, "tokens/total": 856448, "tokens/train_per_sec_per_gpu": 96.73, "tokens/trainable": 38632} +{"epoch": 0.1015625, "grad_norm": 2.4321398735046387, "learning_rate": 0.0001, "loss": 1.1021881103515625, "memory/device_reserved (GiB)": 41.37, "memory/max_active (GiB)": 37.12, "memory/max_allocated (GiB)": 37.12, "ppl": 3.01075, "step": 26, "tokens/total": 889664, "tokens/train_per_sec_per_gpu": 86.41, "tokens/trainable": 40074} +{"epoch": 0.10546875, "grad_norm": 1.9545475244522095, "learning_rate": 9.99990636831587e-05, "loss": 0.9772686958312988, "memory/device_reserved (GiB)": 41.37, "memory/max_active (GiB)": 37.53, "memory/max_allocated (GiB)": 37.53, "ppl": 2.65719, "step": 27, "tokens/total": 922880, "tokens/train_per_sec_per_gpu": 94.06, "tokens/trainable": 41652} +{"epoch": 0.109375, "grad_norm": 2.1558961868286133, "learning_rate": 9.999625477159879e-05, "loss": 0.9352605938911438, "memory/device_reserved (GiB)": 43.73, "memory/max_active (GiB)": 39.58, "memory/max_allocated (GiB)": 39.58, "ppl": 2.54788, "step": 28, "tokens/total": 961904, "tokens/train_per_sec_per_gpu": 78.95, "tokens/trainable": 43291} +{"epoch": 0.11328125, "grad_norm": 1.8975670337677002, "learning_rate": 9.999157338221051e-05, "loss": 0.9549272060394287, "memory/device_reserved (GiB)": 43.73, "memory/max_active (GiB)": 38.02, "memory/max_allocated (GiB)": 38.02, "ppl": 2.59848, "step": 29, "tokens/total": 997552, "tokens/train_per_sec_per_gpu": 88.16, "tokens/trainable": 44900} +{"epoch": 0.1171875, "grad_norm": 2.0628373622894287, "learning_rate": 9.998501970980562e-05, "loss": 0.8304142951965332, "memory/device_reserved (GiB)": 43.73, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 2.29427, "step": 30, "tokens/total": 1029872, "tokens/train_per_sec_per_gpu": 101.68, "tokens/trainable": 46478} +{"epoch": 0.12109375, "grad_norm": 3.5737719535827637, "learning_rate": 9.997659402710915e-05, "loss": 0.8243105411529541, "memory/device_reserved (GiB)": 43.73, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 2.28031, "step": 31, "tokens/total": 1065088, "tokens/train_per_sec_per_gpu": 87.88, "tokens/trainable": 47997} +{"epoch": 0.125, "grad_norm": 2.271580696105957, "learning_rate": 9.996629668474818e-05, "loss": 0.7133302092552185, "memory/device_reserved (GiB)": 43.73, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 2.04078, "step": 32, "tokens/total": 1100672, "tokens/train_per_sec_per_gpu": 88.7, "tokens/trainable": 49599} +{"epoch": 0.12890625, "grad_norm": 1.7357401847839355, "learning_rate": 9.995412811123711e-05, "loss": 0.7759137153625488, "memory/device_reserved (GiB)": 43.73, "memory/max_active (GiB)": 39.08, "memory/max_allocated (GiB)": 39.08, "ppl": 2.17258, "step": 33, "tokens/total": 1136736, "tokens/train_per_sec_per_gpu": 86.62, "tokens/trainable": 51100} +{"epoch": 0.1328125, "grad_norm": 1.5948779582977295, "learning_rate": 9.994008881295999e-05, "loss": 0.6257821321487427, "memory/device_reserved (GiB)": 39.99, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.86971, "step": 34, "tokens/total": 1170560, "tokens/train_per_sec_per_gpu": 95.89, "tokens/trainable": 52742} +{"epoch": 0.13671875, "grad_norm": 1.462105631828308, "learning_rate": 9.992417937414932e-05, "loss": 0.5581216812133789, "memory/device_reserved (GiB)": 39.99, "memory/max_active (GiB)": 38.13, "memory/max_allocated (GiB)": 38.13, "ppl": 1.74739, "step": 35, "tokens/total": 1205024, "tokens/train_per_sec_per_gpu": 92.12, "tokens/trainable": 54456} +{"epoch": 0.140625, "grad_norm": 2.598179817199707, "learning_rate": 9.99064004568618e-05, "loss": 0.6137428283691406, "memory/device_reserved (GiB)": 39.99, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.84733, "step": 36, "tokens/total": 1241408, "tokens/train_per_sec_per_gpu": 80.05, "tokens/trainable": 55964} +{"epoch": 0.14453125, "grad_norm": 2.3212573528289795, "learning_rate": 9.988675280095074e-05, "loss": 0.5767286419868469, "memory/device_reserved (GiB)": 39.99, "memory/max_active (GiB)": 38.06, "memory/max_allocated (GiB)": 38.06, "ppl": 1.78021, "step": 37, "tokens/total": 1276176, "tokens/train_per_sec_per_gpu": 83.37, "tokens/trainable": 57554} +{"epoch": 0.1484375, "grad_norm": 2.280184745788574, "learning_rate": 9.986523722403528e-05, "loss": 0.5461403131484985, "memory/device_reserved (GiB)": 40.0, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.72658, "step": 38, "tokens/total": 1308624, "tokens/train_per_sec_per_gpu": 98.25, "tokens/trainable": 59158} +{"epoch": 0.15234375, "grad_norm": 2.0513882637023926, "learning_rate": 9.984185462146642e-05, "loss": 0.517785370349884, "memory/device_reserved (GiB)": 40.0, "memory/max_active (GiB)": 37.59, "memory/max_allocated (GiB)": 37.59, "ppl": 1.67831, "step": 39, "tokens/total": 1344112, "tokens/train_per_sec_per_gpu": 90.58, "tokens/trainable": 60852} +{"epoch": 0.15625, "grad_norm": 9.626823425292969, "learning_rate": 9.98166059662897e-05, "loss": 0.5231626033782959, "memory/device_reserved (GiB)": 40.91, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.68736, "step": 40, "tokens/total": 1379968, "tokens/train_per_sec_per_gpu": 88.66, "tokens/trainable": 62439} +{"epoch": 0.16015625, "grad_norm": 1.2603182792663574, "learning_rate": 9.978949230920472e-05, "loss": 0.5186857581138611, "memory/device_reserved (GiB)": 40.91, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.67982, "step": 41, "tokens/total": 1414928, "tokens/train_per_sec_per_gpu": 86.16, "tokens/trainable": 64001} +{"epoch": 0.1640625, "grad_norm": 1.5385271310806274, "learning_rate": 9.976051477852141e-05, "loss": 0.5562880635261536, "memory/device_reserved (GiB)": 40.91, "memory/max_active (GiB)": 38.16, "memory/max_allocated (GiB)": 38.16, "ppl": 1.74419, "step": 42, "tokens/total": 1449248, "tokens/train_per_sec_per_gpu": 93.07, "tokens/trainable": 65528} +{"epoch": 0.16796875, "grad_norm": 3.640817880630493, "learning_rate": 9.972967458011312e-05, "loss": 0.5434061288833618, "memory/device_reserved (GiB)": 40.91, "memory/max_active (GiB)": 39.11, "memory/max_allocated (GiB)": 39.11, "ppl": 1.72186, "step": 43, "tokens/total": 1485088, "tokens/train_per_sec_per_gpu": 82.4, "tokens/trainable": 67051} +{"epoch": 0.171875, "grad_norm": 3.4242420196533203, "learning_rate": 9.96969729973664e-05, "loss": 0.484871506690979, "memory/device_reserved (GiB)": 40.91, "memory/max_active (GiB)": 39.14, "memory/max_allocated (GiB)": 39.14, "ppl": 1.62397, "step": 44, "tokens/total": 1524288, "tokens/train_per_sec_per_gpu": 84.43, "tokens/trainable": 68664} +{"epoch": 0.17578125, "grad_norm": 1.4702930450439453, "learning_rate": 9.966241139112754e-05, "loss": 0.543954610824585, "memory/device_reserved (GiB)": 40.92, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.72281, "step": 45, "tokens/total": 1558112, "tokens/train_per_sec_per_gpu": 90.4, "tokens/trainable": 70190} +{"epoch": 0.1796875, "grad_norm": 2.1092381477355957, "learning_rate": 9.96259911996461e-05, "loss": 0.48575082421302795, "memory/device_reserved (GiB)": 40.92, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.62539, "step": 46, "tokens/total": 1592720, "tokens/train_per_sec_per_gpu": 85.5, "tokens/trainable": 71761} +{"epoch": 0.18359375, "grad_norm": 1.9482851028442383, "learning_rate": 9.958771393851491e-05, "loss": 0.5199577808380127, "memory/device_reserved (GiB)": 40.92, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.68196, "step": 47, "tokens/total": 1624896, "tokens/train_per_sec_per_gpu": 88.36, "tokens/trainable": 73320} +{"epoch": 0.1875, "grad_norm": 1.6928074359893799, "learning_rate": 9.954758120060702e-05, "loss": 0.45431238412857056, "memory/device_reserved (GiB)": 40.92, "memory/max_active (GiB)": 39.39, "memory/max_allocated (GiB)": 39.39, "ppl": 1.57509, "step": 48, "tokens/total": 1661664, "tokens/train_per_sec_per_gpu": 78.42, "tokens/trainable": 74859} +{"epoch": 0.19140625, "grad_norm": 1.8449757099151611, "learning_rate": 9.950559465600948e-05, "loss": 0.4449220597743988, "memory/device_reserved (GiB)": 40.92, "memory/max_active (GiB)": 37.73, "memory/max_allocated (GiB)": 37.73, "ppl": 1.56037, "step": 49, "tokens/total": 1696144, "tokens/train_per_sec_per_gpu": 90.28, "tokens/trainable": 76389} +{"epoch": 0.1953125, "grad_norm": 3.3449952602386475, "learning_rate": 9.946175605195379e-05, "loss": 0.41919493675231934, "memory/device_reserved (GiB)": 40.92, "memory/max_active (GiB)": 39.6, "memory/max_allocated (GiB)": 39.6, "ppl": 1.52074, "step": 50, "tokens/total": 1736608, "tokens/train_per_sec_per_gpu": 69.47, "tokens/trainable": 77945} +{"epoch": 0.19921875, "grad_norm": 1.3926770687103271, "learning_rate": 9.941606721274322e-05, "loss": 0.41382625699043274, "memory/device_reserved (GiB)": 40.92, "memory/max_active (GiB)": 37.39, "memory/max_allocated (GiB)": 37.39, "ppl": 1.51259, "step": 51, "tokens/total": 1771440, "tokens/train_per_sec_per_gpu": 85.78, "tokens/trainable": 79508} +{"epoch": 0.203125, "grad_norm": 4.202491760253906, "learning_rate": 9.936853003967685e-05, "loss": 0.4630454182624817, "memory/device_reserved (GiB)": 41.07, "memory/max_active (GiB)": 38.86, "memory/max_allocated (GiB)": 38.86, "ppl": 1.58891, "step": 52, "tokens/total": 1809216, "tokens/train_per_sec_per_gpu": 81.19, "tokens/trainable": 81084} +{"epoch": 0.20703125, "grad_norm": 3.376098394393921, "learning_rate": 9.93191465109705e-05, "loss": 0.46568596363067627, "memory/device_reserved (GiB)": 41.07, "memory/max_active (GiB)": 38.53, "memory/max_allocated (GiB)": 38.53, "ppl": 1.59311, "step": 53, "tokens/total": 1846608, "tokens/train_per_sec_per_gpu": 76.27, "tokens/trainable": 82587} +{"epoch": 0.2109375, "grad_norm": 3.516340970993042, "learning_rate": 9.926791868167438e-05, "loss": 0.47251641750335693, "memory/device_reserved (GiB)": 41.07, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.60403, "step": 54, "tokens/total": 1883296, "tokens/train_per_sec_per_gpu": 78.84, "tokens/trainable": 84059} +{"epoch": 0.21484375, "grad_norm": 2.0363845825195312, "learning_rate": 9.921484868358753e-05, "loss": 0.5167745351791382, "memory/device_reserved (GiB)": 41.07, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.67661, "step": 55, "tokens/total": 1919632, "tokens/train_per_sec_per_gpu": 76.16, "tokens/trainable": 85591} +{"epoch": 0.21875, "grad_norm": 1.1575196981430054, "learning_rate": 9.915993872516924e-05, "loss": 0.4218403100967407, "memory/device_reserved (GiB)": 41.07, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.52477, "step": 56, "tokens/total": 1955584, "tokens/train_per_sec_per_gpu": 79.41, "tokens/trainable": 87110} +{"epoch": 0.22265625, "grad_norm": 1.8632992506027222, "learning_rate": 9.9103191091447e-05, "loss": 0.4240751266479492, "memory/device_reserved (GiB)": 41.07, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.52818, "step": 57, "tokens/total": 1988016, "tokens/train_per_sec_per_gpu": 92.24, "tokens/trainable": 88633} +{"epoch": 0.2265625, "grad_norm": 2.0231199264526367, "learning_rate": 9.904460814392147e-05, "loss": 0.4063733220100403, "memory/device_reserved (GiB)": 41.07, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.50136, "step": 58, "tokens/total": 2028320, "tokens/train_per_sec_per_gpu": 71.26, "tokens/trainable": 90176} +{"epoch": 0.23046875, "grad_norm": 1.7772986888885498, "learning_rate": 9.898419232046825e-05, "loss": 0.4056253433227539, "memory/device_reserved (GiB)": 41.07, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.50024, "step": 59, "tokens/total": 2065216, "tokens/train_per_sec_per_gpu": 83.07, "tokens/trainable": 91795} +{"epoch": 0.234375, "grad_norm": 9.59738540649414, "learning_rate": 9.892194613523633e-05, "loss": 0.42246001958847046, "memory/device_reserved (GiB)": 41.07, "memory/max_active (GiB)": 38.49, "memory/max_allocated (GiB)": 38.49, "ppl": 1.52571, "step": 60, "tokens/total": 2101744, "tokens/train_per_sec_per_gpu": 83.92, "tokens/trainable": 93396} +{"epoch": 0.23828125, "grad_norm": 2.124098062515259, "learning_rate": 9.885787217854357e-05, "loss": 0.3883199989795685, "memory/device_reserved (GiB)": 41.07, "memory/max_active (GiB)": 38.49, "memory/max_allocated (GiB)": 38.49, "ppl": 1.4745, "step": 61, "tokens/total": 2136592, "tokens/train_per_sec_per_gpu": 87.5, "tokens/trainable": 94947} +{"epoch": 0.2421875, "grad_norm": 1.312788486480713, "learning_rate": 9.879197311676887e-05, "loss": 0.415137380361557, "memory/device_reserved (GiB)": 41.07, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.51458, "step": 62, "tokens/total": 2173216, "tokens/train_per_sec_per_gpu": 90.44, "tokens/trainable": 96535} +{"epoch": 0.24609375, "grad_norm": 1.188385248184204, "learning_rate": 9.872425169224113e-05, "loss": 0.39051496982574463, "memory/device_reserved (GiB)": 41.07, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.47774, "step": 63, "tokens/total": 2207776, "tokens/train_per_sec_per_gpu": 96.65, "tokens/trainable": 98222} +{"epoch": 0.25, "grad_norm": 1.1434179544448853, "learning_rate": 9.865471072312528e-05, "loss": 0.39284747838974, "memory/device_reserved (GiB)": 41.07, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.48119, "step": 64, "tokens/total": 2244736, "tokens/train_per_sec_per_gpu": 88.12, "tokens/trainable": 99818} +{"epoch": 0.25390625, "grad_norm": 0.9183613657951355, "learning_rate": 9.858335310330492e-05, "loss": 0.355478972196579, "memory/device_reserved (GiB)": 41.07, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.42686, "step": 65, "tokens/total": 2277792, "tokens/train_per_sec_per_gpu": 98.32, "tokens/trainable": 101501} +{"epoch": 0.2578125, "grad_norm": 4.093695163726807, "learning_rate": 9.851018180226185e-05, "loss": 0.357266366481781, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.09, "memory/max_allocated (GiB)": 39.09, "ppl": 1.42942, "step": 66, "tokens/total": 2316720, "tokens/train_per_sec_per_gpu": 78.07, "tokens/trainable": 103033} +{"epoch": 0.26171875, "grad_norm": 7.058792591094971, "learning_rate": 9.843519986495259e-05, "loss": 0.3979083299636841, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.48871, "step": 67, "tokens/total": 2352080, "tokens/train_per_sec_per_gpu": 84.6, "tokens/trainable": 104549} +{"epoch": 0.265625, "grad_norm": 1.3363791704177856, "learning_rate": 9.835841041168162e-05, "loss": 0.33980828523635864, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.40468, "step": 68, "tokens/total": 2389808, "tokens/train_per_sec_per_gpu": 85.27, "tokens/trainable": 106100} +{"epoch": 0.26953125, "grad_norm": 1.0941351652145386, "learning_rate": 9.82798166379715e-05, "loss": 0.3225095868110657, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.07, "memory/max_allocated (GiB)": 37.07, "ppl": 1.38059, "step": 69, "tokens/total": 2423712, "tokens/train_per_sec_per_gpu": 91.97, "tokens/trainable": 107750} +{"epoch": 0.2734375, "grad_norm": 0.97394859790802, "learning_rate": 9.819942181443002e-05, "loss": 0.37268272042274475, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.45162, "step": 70, "tokens/total": 2461456, "tokens/train_per_sec_per_gpu": 80.89, "tokens/trainable": 109336} +{"epoch": 0.27734375, "grad_norm": 1.1010075807571411, "learning_rate": 9.811722928661392e-05, "loss": 0.3834877014160156, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 36.01, "memory/max_allocated (GiB)": 36.01, "ppl": 1.46739, "step": 71, "tokens/total": 2490672, "tokens/train_per_sec_per_gpu": 106.81, "tokens/trainable": 110909} +{"epoch": 0.28125, "grad_norm": 13.246110916137695, "learning_rate": 9.803324247488975e-05, "loss": 0.37459784746170044, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.35, "memory/max_allocated (GiB)": 39.35, "ppl": 1.45441, "step": 72, "tokens/total": 2527280, "tokens/train_per_sec_per_gpu": 79.61, "tokens/trainable": 112426} +{"epoch": 0.28515625, "grad_norm": 1.8704519271850586, "learning_rate": 9.794746487429161e-05, "loss": 0.37169909477233887, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.2, "memory/max_allocated (GiB)": 37.2, "ppl": 1.4502, "step": 73, "tokens/total": 2560896, "tokens/train_per_sec_per_gpu": 93.59, "tokens/trainable": 113985} +{"epoch": 0.2890625, "grad_norm": 1.6321263313293457, "learning_rate": 9.785990005437554e-05, "loss": 0.37554851174354553, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.07, "memory/max_allocated (GiB)": 38.07, "ppl": 1.45579, "step": 74, "tokens/total": 2597216, "tokens/train_per_sec_per_gpu": 84.87, "tokens/trainable": 115559} +{"epoch": 0.29296875, "grad_norm": 1.1019513607025146, "learning_rate": 9.777055165907117e-05, "loss": 0.3871050477027893, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 1.47271, "step": 75, "tokens/total": 2631040, "tokens/train_per_sec_per_gpu": 98.64, "tokens/trainable": 117131} +{"epoch": 0.296875, "grad_norm": 1.049127459526062, "learning_rate": 9.767942340652993e-05, "loss": 0.38655704259872437, "memory/device_reserved (GiB)": 41.48, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.4719, "step": 76, "tokens/total": 2666256, "tokens/train_per_sec_per_gpu": 85.11, "tokens/trainable": 118740} +{"epoch": 0.30078125, "grad_norm": 2.089858293533325, "learning_rate": 9.758651908897035e-05, "loss": 0.36390429735183716, "memory/device_reserved (GiB)": 41.48, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 1.43894, "step": 77, "tokens/total": 2700032, "tokens/train_per_sec_per_gpu": 84.02, "tokens/trainable": 120267} +{"epoch": 0.3046875, "grad_norm": 1.541093111038208, "learning_rate": 9.749184257252033e-05, "loss": 0.3927082121372223, "memory/device_reserved (GiB)": 41.48, "memory/max_active (GiB)": 37.39, "memory/max_allocated (GiB)": 37.39, "ppl": 1.48099, "step": 78, "tokens/total": 2735040, "tokens/train_per_sec_per_gpu": 90.56, "tokens/trainable": 121823} +{"epoch": 0.30859375, "grad_norm": 1.1783965826034546, "learning_rate": 9.739539779705614e-05, "loss": 0.3734564483165741, "memory/device_reserved (GiB)": 41.48, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.45275, "step": 79, "tokens/total": 2773760, "tokens/train_per_sec_per_gpu": 78.91, "tokens/trainable": 123466} +{"epoch": 0.3125, "grad_norm": 2.592311143875122, "learning_rate": 9.729718877603861e-05, "loss": 0.346618115901947, "memory/device_reserved (GiB)": 41.48, "memory/max_active (GiB)": 37.44, "memory/max_allocated (GiB)": 37.44, "ppl": 1.41428, "step": 80, "tokens/total": 2807680, "tokens/train_per_sec_per_gpu": 90.38, "tokens/trainable": 124974} +{"epoch": 0.31640625, "grad_norm": 2.3080005645751953, "learning_rate": 9.719721959634592e-05, "loss": 0.32028651237487793, "memory/device_reserved (GiB)": 41.48, "memory/max_active (GiB)": 39.35, "memory/max_allocated (GiB)": 39.35, "ppl": 1.37752, "step": 81, "tokens/total": 2843744, "tokens/train_per_sec_per_gpu": 86.57, "tokens/trainable": 126494} +{"epoch": 0.3203125, "grad_norm": 1.0642492771148682, "learning_rate": 9.709549441810375e-05, "loss": 0.3587508201599121, "memory/device_reserved (GiB)": 41.59, "memory/max_active (GiB)": 39.6, "memory/max_allocated (GiB)": 39.6, "ppl": 1.43154, "step": 82, "tokens/total": 2881664, "tokens/train_per_sec_per_gpu": 80.48, "tokens/trainable": 128002} +{"epoch": 0.32421875, "grad_norm": 1.8919047117233276, "learning_rate": 9.699201747451195e-05, "loss": 0.3581802248954773, "memory/device_reserved (GiB)": 41.59, "memory/max_active (GiB)": 36.61, "memory/max_allocated (GiB)": 36.61, "ppl": 1.43072, "step": 83, "tokens/total": 2914512, "tokens/train_per_sec_per_gpu": 92.08, "tokens/trainable": 129605} +{"epoch": 0.328125, "grad_norm": 1.4495747089385986, "learning_rate": 9.688679307166854e-05, "loss": 0.36183956265449524, "memory/device_reserved (GiB)": 41.6, "memory/max_active (GiB)": 38.1, "memory/max_allocated (GiB)": 38.1, "ppl": 1.43597, "step": 84, "tokens/total": 2947616, "tokens/train_per_sec_per_gpu": 92.78, "tokens/trainable": 131188} +{"epoch": 0.33203125, "grad_norm": 1.124336838722229, "learning_rate": 9.677982558839042e-05, "loss": 0.35118114948272705, "memory/device_reserved (GiB)": 41.6, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.42074, "step": 85, "tokens/total": 2981328, "tokens/train_per_sec_per_gpu": 92.48, "tokens/trainable": 132834} +{"epoch": 0.3359375, "grad_norm": 1.2349753379821777, "learning_rate": 9.66711194760312e-05, "loss": 0.3087597191333771, "memory/device_reserved (GiB)": 41.6, "memory/max_active (GiB)": 39.1, "memory/max_allocated (GiB)": 39.1, "ppl": 1.36174, "step": 86, "tokens/total": 3018960, "tokens/train_per_sec_per_gpu": 78.48, "tokens/trainable": 134439} +{"epoch": 0.33984375, "grad_norm": 1.2317832708358765, "learning_rate": 9.656067925829593e-05, "loss": 0.3353178799152374, "memory/device_reserved (GiB)": 41.6, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.39838, "step": 87, "tokens/total": 3049936, "tokens/train_per_sec_per_gpu": 98.94, "tokens/trainable": 136000} +{"epoch": 0.34375, "grad_norm": 1.0199415683746338, "learning_rate": 9.644850953105288e-05, "loss": 0.3488825857639313, "memory/device_reserved (GiB)": 41.6, "memory/max_active (GiB)": 36.1, "memory/max_allocated (GiB)": 36.1, "ppl": 1.41748, "step": 88, "tokens/total": 3081376, "tokens/train_per_sec_per_gpu": 97.42, "tokens/trainable": 137603} +{"epoch": 0.34765625, "grad_norm": 0.9024736881256104, "learning_rate": 9.633461496214225e-05, "loss": 0.31717121601104736, "memory/device_reserved (GiB)": 41.6, "memory/max_active (GiB)": 37.5, "memory/max_allocated (GiB)": 37.5, "ppl": 1.37324, "step": 89, "tokens/total": 3114560, "tokens/train_per_sec_per_gpu": 90.66, "tokens/trainable": 139148} +{"epoch": 0.3515625, "grad_norm": 0.9809176325798035, "learning_rate": 9.621900029118195e-05, "loss": 0.29670435190200806, "memory/device_reserved (GiB)": 41.6, "memory/max_active (GiB)": 36.27, "memory/max_allocated (GiB)": 36.27, "ppl": 1.34542, "step": 90, "tokens/total": 3144704, "tokens/train_per_sec_per_gpu": 104.12, "tokens/trainable": 140757} +{"epoch": 0.35546875, "grad_norm": 1.1136738061904907, "learning_rate": 9.610167032937036e-05, "loss": 0.34339410066604614, "memory/device_reserved (GiB)": 41.6, "memory/max_active (GiB)": 38.72, "memory/max_allocated (GiB)": 38.72, "ppl": 1.40972, "step": 91, "tokens/total": 3183344, "tokens/train_per_sec_per_gpu": 83.17, "tokens/trainable": 142369} +{"epoch": 0.359375, "grad_norm": 2.004392623901367, "learning_rate": 9.598262995928611e-05, "loss": 0.3041829466819763, "memory/device_reserved (GiB)": 41.6, "memory/max_active (GiB)": 37.12, "memory/max_allocated (GiB)": 37.12, "ppl": 1.35552, "step": 92, "tokens/total": 3214928, "tokens/train_per_sec_per_gpu": 107.13, "tokens/trainable": 143959} +{"epoch": 0.36328125, "grad_norm": 1.367649793624878, "learning_rate": 9.586188413468492e-05, "loss": 0.32253211736679077, "memory/device_reserved (GiB)": 41.6, "memory/max_active (GiB)": 39.05, "memory/max_allocated (GiB)": 39.05, "ppl": 1.38062, "step": 93, "tokens/total": 3253920, "tokens/train_per_sec_per_gpu": 73.02, "tokens/trainable": 145514} +{"epoch": 0.3671875, "grad_norm": 1.1319178342819214, "learning_rate": 9.57394378802934e-05, "loss": 0.3390547037124634, "memory/device_reserved (GiB)": 41.6, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.40362, "step": 94, "tokens/total": 3287840, "tokens/train_per_sec_per_gpu": 86.29, "tokens/trainable": 147036} +{"epoch": 0.37109375, "grad_norm": 1.4311935901641846, "learning_rate": 9.56152962916e-05, "loss": 0.3334232568740845, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 36.65, "memory/max_allocated (GiB)": 36.65, "ppl": 1.39574, "step": 95, "tokens/total": 3319104, "tokens/train_per_sec_per_gpu": 98.6, "tokens/trainable": 148465} +{"epoch": 0.375, "grad_norm": 1.4917242527008057, "learning_rate": 9.548946453464296e-05, "loss": 0.28848758339881897, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 38.17, "memory/max_allocated (GiB)": 38.17, "ppl": 1.33441, "step": 96, "tokens/total": 3354192, "tokens/train_per_sec_per_gpu": 87.29, "tokens/trainable": 150010} +{"epoch": 0.37890625, "grad_norm": 1.6757858991622925, "learning_rate": 9.53619478457953e-05, "loss": 0.3439103364944458, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.41045, "step": 97, "tokens/total": 3389520, "tokens/train_per_sec_per_gpu": 83.21, "tokens/trainable": 151538} +{"epoch": 0.3828125, "grad_norm": 1.3649131059646606, "learning_rate": 9.523275153154695e-05, "loss": 0.3113449811935425, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 36.88, "memory/max_allocated (GiB)": 36.88, "ppl": 1.36526, "step": 98, "tokens/total": 3422512, "tokens/train_per_sec_per_gpu": 93.37, "tokens/trainable": 153043} +{"epoch": 0.38671875, "grad_norm": 0.846967875957489, "learning_rate": 9.51018809682839e-05, "loss": 0.3207012712955475, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.37809, "step": 99, "tokens/total": 3460880, "tokens/train_per_sec_per_gpu": 76.74, "tokens/trainable": 154546} +{"epoch": 0.390625, "grad_norm": 1.450781226158142, "learning_rate": 9.49693416020645e-05, "loss": 0.35929927229881287, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 38.67, "memory/max_allocated (GiB)": 38.67, "ppl": 1.43233, "step": 100, "tokens/total": 3498976, "tokens/train_per_sec_per_gpu": 75.98, "tokens/trainable": 156007} +{"epoch": 0.39453125, "grad_norm": 1.4201446771621704, "learning_rate": 9.483513894839276e-05, "loss": 0.3440456986427307, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 37.18, "memory/max_allocated (GiB)": 37.18, "ppl": 1.41064, "step": 101, "tokens/total": 3533104, "tokens/train_per_sec_per_gpu": 93.39, "tokens/trainable": 157640} +{"epoch": 0.3984375, "grad_norm": 1.7785192728042603, "learning_rate": 9.469927859198888e-05, "loss": 0.28741639852523804, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.33298, "step": 102, "tokens/total": 3569664, "tokens/train_per_sec_per_gpu": 91.0, "tokens/trainable": 159272} +{"epoch": 0.40234375, "grad_norm": 0.954168975353241, "learning_rate": 9.456176618655689e-05, "loss": 0.27250736951828003, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 36.73, "memory/max_allocated (GiB)": 36.73, "ppl": 1.31325, "step": 103, "tokens/total": 3602192, "tokens/train_per_sec_per_gpu": 100.41, "tokens/trainable": 160845} +{"epoch": 0.40625, "grad_norm": 9.37155818939209, "learning_rate": 9.442260745454927e-05, "loss": 0.306876540184021, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.35917, "step": 104, "tokens/total": 3637760, "tokens/train_per_sec_per_gpu": 81.36, "tokens/trainable": 162387} +{"epoch": 0.41015625, "grad_norm": 3.4328134059906006, "learning_rate": 9.428180818692884e-05, "loss": 0.27448850870132446, "memory/device_reserved (GiB)": 41.61, "memory/max_active (GiB)": 38.7, "memory/max_allocated (GiB)": 38.7, "ppl": 1.31586, "step": 105, "tokens/total": 3674320, "tokens/train_per_sec_per_gpu": 78.25, "tokens/trainable": 163948} +{"epoch": 0.4140625, "grad_norm": 1.2198454141616821, "learning_rate": 9.413937424292791e-05, "loss": 0.30087387561798096, "memory/device_reserved (GiB)": 41.88, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.35104, "step": 106, "tokens/total": 3713872, "tokens/train_per_sec_per_gpu": 71.66, "tokens/trainable": 165440} +{"epoch": 0.41796875, "grad_norm": 1.326085090637207, "learning_rate": 9.399531154980424e-05, "loss": 0.28353771567344666, "memory/device_reserved (GiB)": 41.88, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.32782, "step": 107, "tokens/total": 3751056, "tokens/train_per_sec_per_gpu": 74.49, "tokens/trainable": 166933} +{"epoch": 0.421875, "grad_norm": 1.0067492723464966, "learning_rate": 9.384962610259455e-05, "loss": 0.28524938225746155, "memory/device_reserved (GiB)": 41.88, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.33009, "step": 108, "tokens/total": 3786096, "tokens/train_per_sec_per_gpu": 83.78, "tokens/trainable": 168410} +{"epoch": 0.42578125, "grad_norm": 1.217339277267456, "learning_rate": 9.370232396386494e-05, "loss": 0.30184298753738403, "memory/device_reserved (GiB)": 41.88, "memory/max_active (GiB)": 37.23, "memory/max_allocated (GiB)": 37.23, "ppl": 1.35235, "step": 109, "tokens/total": 3819616, "tokens/train_per_sec_per_gpu": 91.24, "tokens/trainable": 169923} +{"epoch": 0.4296875, "grad_norm": 1.050347924232483, "learning_rate": 9.355341126345868e-05, "loss": 0.26185959577560425, "memory/device_reserved (GiB)": 41.88, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.29934, "step": 110, "tokens/total": 3855360, "tokens/train_per_sec_per_gpu": 84.18, "tokens/trainable": 171463} +{"epoch": 0.43359375, "grad_norm": 2.1236414909362793, "learning_rate": 9.340289419824107e-05, "loss": 0.258634477853775, "memory/device_reserved (GiB)": 41.88, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.29516, "step": 111, "tokens/total": 3888016, "tokens/train_per_sec_per_gpu": 93.01, "tokens/trainable": 172960} +{"epoch": 0.4375, "grad_norm": 2.469646453857422, "learning_rate": 9.325077903184159e-05, "loss": 0.30153441429138184, "memory/device_reserved (GiB)": 41.88, "memory/max_active (GiB)": 39.04, "memory/max_allocated (GiB)": 39.04, "ppl": 1.35193, "step": 112, "tokens/total": 3923424, "tokens/train_per_sec_per_gpu": 85.04, "tokens/trainable": 174572} +{"epoch": 0.44140625, "grad_norm": 1.2880302667617798, "learning_rate": 9.30970720943932e-05, "loss": 0.2530999183654785, "memory/device_reserved (GiB)": 41.88, "memory/max_active (GiB)": 38.66, "memory/max_allocated (GiB)": 38.66, "ppl": 1.28801, "step": 113, "tokens/total": 3959472, "tokens/train_per_sec_per_gpu": 87.02, "tokens/trainable": 176161} +{"epoch": 0.4453125, "grad_norm": 1.27695631980896, "learning_rate": 9.2941779782269e-05, "loss": 0.29894542694091797, "memory/device_reserved (GiB)": 41.88, "memory/max_active (GiB)": 37.39, "memory/max_allocated (GiB)": 37.39, "ppl": 1.34844, "step": 114, "tokens/total": 3993712, "tokens/train_per_sec_per_gpu": 83.81, "tokens/trainable": 177612} +{"epoch": 0.44921875, "grad_norm": 4.125149726867676, "learning_rate": 9.278490855781596e-05, "loss": 0.2650575041770935, "memory/device_reserved (GiB)": 41.88, "memory/max_active (GiB)": 37.73, "memory/max_allocated (GiB)": 37.73, "ppl": 1.30351, "step": 115, "tokens/total": 4026896, "tokens/train_per_sec_per_gpu": 101.21, "tokens/trainable": 179213} +{"epoch": 0.453125, "grad_norm": 1.6296379566192627, "learning_rate": 9.262646494908604e-05, "loss": 0.30825352668762207, "memory/device_reserved (GiB)": 41.88, "memory/max_active (GiB)": 39.39, "memory/max_allocated (GiB)": 39.39, "ppl": 1.36105, "step": 116, "tokens/total": 4066064, "tokens/train_per_sec_per_gpu": 77.53, "tokens/trainable": 180750} +{"epoch": 0.45703125, "grad_norm": 1.4711403846740723, "learning_rate": 9.246645554956457e-05, "loss": 0.27575016021728516, "memory/device_reserved (GiB)": 41.88, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.31752, "step": 117, "tokens/total": 4101280, "tokens/train_per_sec_per_gpu": 86.79, "tokens/trainable": 182374} +{"epoch": 0.4609375, "grad_norm": 1.1157541275024414, "learning_rate": 9.230488701789578e-05, "loss": 0.3273073434829712, "memory/device_reserved (GiB)": 41.88, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.38723, "step": 118, "tokens/total": 4139072, "tokens/train_per_sec_per_gpu": 80.89, "tokens/trainable": 183955} +{"epoch": 0.46484375, "grad_norm": 1.393721342086792, "learning_rate": 9.214176607760577e-05, "loss": 0.28190773725509644, "memory/device_reserved (GiB)": 41.88, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.32566, "step": 119, "tokens/total": 4178128, "tokens/train_per_sec_per_gpu": 85.69, "tokens/trainable": 185598} +{"epoch": 0.46875, "grad_norm": 1.9521703720092773, "learning_rate": 9.197709951682268e-05, "loss": 0.2621462941169739, "memory/device_reserved (GiB)": 41.88, "memory/max_active (GiB)": 37.09, "memory/max_allocated (GiB)": 37.09, "ppl": 1.29972, "step": 120, "tokens/total": 4211056, "tokens/train_per_sec_per_gpu": 96.57, "tokens/trainable": 187210} +{"epoch": 0.47265625, "grad_norm": 1.0579315423965454, "learning_rate": 9.181089418799428e-05, "loss": 0.28736644983291626, "memory/device_reserved (GiB)": 41.88, "memory/max_active (GiB)": 39.61, "memory/max_allocated (GiB)": 39.61, "ppl": 1.33291, "step": 121, "tokens/total": 4248384, "tokens/train_per_sec_per_gpu": 77.08, "tokens/trainable": 188733} +{"epoch": 0.4765625, "grad_norm": 1.0908231735229492, "learning_rate": 9.164315700760271e-05, "loss": 0.24461033940315247, "memory/device_reserved (GiB)": 41.88, "memory/max_active (GiB)": 39.09, "memory/max_allocated (GiB)": 39.09, "ppl": 1.27712, "step": 122, "tokens/total": 4284864, "tokens/train_per_sec_per_gpu": 79.45, "tokens/trainable": 190273} +{"epoch": 0.48046875, "grad_norm": 1.4567981958389282, "learning_rate": 9.147389495587671e-05, "loss": 0.3408873975276947, "memory/device_reserved (GiB)": 41.88, "memory/max_active (GiB)": 37.19, "memory/max_allocated (GiB)": 37.19, "ppl": 1.40619, "step": 123, "tokens/total": 4316608, "tokens/train_per_sec_per_gpu": 97.61, "tokens/trainable": 191805} +{"epoch": 0.484375, "grad_norm": 1.668837070465088, "learning_rate": 9.130311507650116e-05, "loss": 0.24123027920722961, "memory/device_reserved (GiB)": 41.88, "memory/max_active (GiB)": 37.13, "memory/max_allocated (GiB)": 37.13, "ppl": 1.27281, "step": 124, "tokens/total": 4349568, "tokens/train_per_sec_per_gpu": 90.56, "tokens/trainable": 193376} +{"epoch": 0.48828125, "grad_norm": 1.1333351135253906, "learning_rate": 9.113082447632394e-05, "loss": 0.26946568489074707, "memory/device_reserved (GiB)": 41.88, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.30926, "step": 125, "tokens/total": 4382400, "tokens/train_per_sec_per_gpu": 102.47, "tokens/trainable": 194995} +{"epoch": 0.4921875, "grad_norm": 1.5142333507537842, "learning_rate": 9.09570303250602e-05, "loss": 0.29138362407684326, "memory/device_reserved (GiB)": 41.88, "memory/max_active (GiB)": 36.61, "memory/max_allocated (GiB)": 36.61, "ppl": 1.33828, "step": 126, "tokens/total": 4415024, "tokens/train_per_sec_per_gpu": 89.64, "tokens/trainable": 196565} +{"epoch": 0.49609375, "grad_norm": 1.1712617874145508, "learning_rate": 9.078173985499394e-05, "loss": 0.26750582456588745, "memory/device_reserved (GiB)": 41.88, "memory/max_active (GiB)": 36.01, "memory/max_allocated (GiB)": 36.01, "ppl": 1.3067, "step": 127, "tokens/total": 4445808, "tokens/train_per_sec_per_gpu": 102.72, "tokens/trainable": 198146} +{"epoch": 0.5, "grad_norm": 1.0427271127700806, "learning_rate": 9.060496036067713e-05, "loss": 0.26788613200187683, "memory/device_reserved (GiB)": 41.88, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.3072, "step": 128, "tokens/total": 4483616, "tokens/train_per_sec_per_gpu": 71.09, "tokens/trainable": 199658} +{"epoch": 0.50390625, "grad_norm": 1.0909767150878906, "learning_rate": 9.042669919862615e-05, "loss": 0.2568057179450989, "memory/device_reserved (GiB)": 41.88, "memory/max_active (GiB)": 37.04, "memory/max_allocated (GiB)": 37.04, "ppl": 1.29279, "step": 129, "tokens/total": 4516208, "tokens/train_per_sec_per_gpu": 95.88, "tokens/trainable": 201249} +{"epoch": 0.5078125, "grad_norm": 0.8695460557937622, "learning_rate": 9.024696378701557e-05, "loss": 0.23970654606819153, "memory/device_reserved (GiB)": 41.41, "memory/max_active (GiB)": 39.64, "memory/max_allocated (GiB)": 39.64, "ppl": 1.27088, "step": 130, "tokens/total": 4552560, "tokens/train_per_sec_per_gpu": 82.88, "tokens/trainable": 202804} +{"epoch": 0.51171875, "grad_norm": 2.5763237476348877, "learning_rate": 9.006576160536948e-05, "loss": 0.2659603953361511, "memory/device_reserved (GiB)": 41.41, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.30468, "step": 131, "tokens/total": 4587760, "tokens/train_per_sec_per_gpu": 84.07, "tokens/trainable": 204366} +{"epoch": 0.515625, "grad_norm": 6.337902545928955, "learning_rate": 8.988310019425035e-05, "loss": 0.25662967562675476, "memory/device_reserved (GiB)": 41.41, "memory/max_active (GiB)": 39.74, "memory/max_allocated (GiB)": 39.74, "ppl": 1.29257, "step": 132, "tokens/total": 4626032, "tokens/train_per_sec_per_gpu": 77.24, "tokens/trainable": 205909} +{"epoch": 0.51953125, "grad_norm": 2.6754884719848633, "learning_rate": 8.969898715494506e-05, "loss": 0.25512200593948364, "memory/device_reserved (GiB)": 41.8, "memory/max_active (GiB)": 39.61, "memory/max_allocated (GiB)": 39.61, "ppl": 1.29062, "step": 133, "tokens/total": 4663504, "tokens/train_per_sec_per_gpu": 82.37, "tokens/trainable": 207481} +{"epoch": 0.5234375, "grad_norm": 2.267754316329956, "learning_rate": 8.951343014914869e-05, "loss": 0.25958648324012756, "memory/device_reserved (GiB)": 41.8, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.29639, "step": 134, "tokens/total": 4700448, "tokens/train_per_sec_per_gpu": 78.67, "tokens/trainable": 209029} +{"epoch": 0.52734375, "grad_norm": 1.091727375984192, "learning_rate": 8.932643689864568e-05, "loss": 0.2698284089565277, "memory/device_reserved (GiB)": 41.8, "memory/max_active (GiB)": 39.04, "memory/max_allocated (GiB)": 39.04, "ppl": 1.30974, "step": 135, "tokens/total": 4737632, "tokens/train_per_sec_per_gpu": 89.8, "tokens/trainable": 210665} +{"epoch": 0.53125, "grad_norm": 1.1191351413726807, "learning_rate": 8.913801518498845e-05, "loss": 0.22455620765686035, "memory/device_reserved (GiB)": 43.07, "memory/max_active (GiB)": 39.76, "memory/max_allocated (GiB)": 39.76, "ppl": 1.25177, "step": 136, "tokens/total": 4777248, "tokens/train_per_sec_per_gpu": 76.31, "tokens/trainable": 212303} +{"epoch": 0.53515625, "grad_norm": 1.1500170230865479, "learning_rate": 8.894817284917364e-05, "loss": 0.2434345930814743, "memory/device_reserved (GiB)": 43.07, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.27562, "step": 137, "tokens/total": 4814656, "tokens/train_per_sec_per_gpu": 76.83, "tokens/trainable": 213891} +{"epoch": 0.5390625, "grad_norm": 1.76168954372406, "learning_rate": 8.875691779131569e-05, "loss": 0.2557666599750519, "memory/device_reserved (GiB)": 43.07, "memory/max_active (GiB)": 39.92, "memory/max_allocated (GiB)": 39.92, "ppl": 1.29145, "step": 138, "tokens/total": 4852896, "tokens/train_per_sec_per_gpu": 77.18, "tokens/trainable": 215395} +{"epoch": 0.54296875, "grad_norm": 0.9525693655014038, "learning_rate": 8.856425797031829e-05, "loss": 0.22255578637123108, "memory/device_reserved (GiB)": 43.07, "memory/max_active (GiB)": 39.66, "memory/max_allocated (GiB)": 39.66, "ppl": 1.24927, "step": 139, "tokens/total": 4889264, "tokens/train_per_sec_per_gpu": 89.73, "tokens/trainable": 217016} +{"epoch": 0.546875, "grad_norm": 2.2688918113708496, "learning_rate": 8.837020140354295e-05, "loss": 0.24374443292617798, "memory/device_reserved (GiB)": 43.07, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.27602, "step": 140, "tokens/total": 4926512, "tokens/train_per_sec_per_gpu": 79.64, "tokens/trainable": 218532} +{"epoch": 0.55078125, "grad_norm": 1.3353641033172607, "learning_rate": 8.817475616647554e-05, "loss": 0.2421686351299286, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.27401, "step": 141, "tokens/total": 4961680, "tokens/train_per_sec_per_gpu": 87.85, "tokens/trainable": 220084} +{"epoch": 0.5546875, "grad_norm": 2.655977725982666, "learning_rate": 8.797793039239017e-05, "loss": 0.2556963562965393, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.29136, "step": 142, "tokens/total": 4997456, "tokens/train_per_sec_per_gpu": 84.37, "tokens/trainable": 221611} +{"epoch": 0.55859375, "grad_norm": 1.6699438095092773, "learning_rate": 8.777973227201069e-05, "loss": 0.24786987900733948, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 39.66, "memory/max_allocated (GiB)": 39.66, "ppl": 1.28129, "step": 143, "tokens/total": 5034784, "tokens/train_per_sec_per_gpu": 88.87, "tokens/trainable": 223182} +{"epoch": 0.5625, "grad_norm": 1.3994383811950684, "learning_rate": 8.758017005316988e-05, "loss": 0.27084988355636597, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.31108, "step": 144, "tokens/total": 5070896, "tokens/train_per_sec_per_gpu": 91.05, "tokens/trainable": 224748} +{"epoch": 0.56640625, "grad_norm": 5.780237197875977, "learning_rate": 8.737925204046629e-05, "loss": 0.26314765214920044, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.30102, "step": 145, "tokens/total": 5106048, "tokens/train_per_sec_per_gpu": 88.16, "tokens/trainable": 226339} +{"epoch": 0.5703125, "grad_norm": 1.2054308652877808, "learning_rate": 8.717698659491851e-05, "loss": 0.23768892884254456, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.26831, "step": 146, "tokens/total": 5143184, "tokens/train_per_sec_per_gpu": 79.62, "tokens/trainable": 227923} +{"epoch": 0.57421875, "grad_norm": 0.9787291288375854, "learning_rate": 8.697338213361735e-05, "loss": 0.2376994788646698, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 39.12, "memory/max_allocated (GiB)": 39.12, "ppl": 1.26833, "step": 147, "tokens/total": 5179680, "tokens/train_per_sec_per_gpu": 87.37, "tokens/trainable": 229612} +{"epoch": 0.578125, "grad_norm": 0.721500813961029, "learning_rate": 8.676844712937552e-05, "loss": 0.21431899070739746, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 39.39, "memory/max_allocated (GiB)": 39.39, "ppl": 1.23902, "step": 148, "tokens/total": 5220080, "tokens/train_per_sec_per_gpu": 72.95, "tokens/trainable": 231191} +{"epoch": 0.58203125, "grad_norm": 1.616184115409851, "learning_rate": 8.656219011037509e-05, "loss": 0.22937655448913574, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.25782, "step": 149, "tokens/total": 5257616, "tokens/train_per_sec_per_gpu": 80.64, "tokens/trainable": 232751} +{"epoch": 0.5859375, "grad_norm": 1.529798984527588, "learning_rate": 8.63546196598125e-05, "loss": 0.2385447472333908, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.2694, "step": 150, "tokens/total": 5295520, "tokens/train_per_sec_per_gpu": 77.91, "tokens/trainable": 234360} +{"epoch": 0.58984375, "grad_norm": 1.1992063522338867, "learning_rate": 8.614574441554145e-05, "loss": 0.2847135066986084, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 37.13, "memory/max_allocated (GiB)": 37.13, "ppl": 1.32938, "step": 151, "tokens/total": 5328880, "tokens/train_per_sec_per_gpu": 93.09, "tokens/trainable": 235934} +{"epoch": 0.59375, "grad_norm": 1.1545984745025635, "learning_rate": 8.593557306971349e-05, "loss": 0.23667961359024048, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.26704, "step": 152, "tokens/total": 5362688, "tokens/train_per_sec_per_gpu": 88.03, "tokens/trainable": 237483} +{"epoch": 0.59765625, "grad_norm": 1.0075500011444092, "learning_rate": 8.572411436841618e-05, "loss": 0.24689415097236633, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 37.13, "memory/max_allocated (GiB)": 37.13, "ppl": 1.28004, "step": 153, "tokens/total": 5397056, "tokens/train_per_sec_per_gpu": 93.44, "tokens/trainable": 239125} +{"epoch": 0.6015625, "grad_norm": 1.0892916917800903, "learning_rate": 8.551137711130922e-05, "loss": 0.23949281871318817, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.2706, "step": 154, "tokens/total": 5429136, "tokens/train_per_sec_per_gpu": 88.68, "tokens/trainable": 240658} +{"epoch": 0.60546875, "grad_norm": 0.7953672409057617, "learning_rate": 8.529737015125824e-05, "loss": 0.18943792581558228, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.20857, "step": 155, "tokens/total": 5464704, "tokens/train_per_sec_per_gpu": 87.91, "tokens/trainable": 242250} +{"epoch": 0.609375, "grad_norm": 4.345024585723877, "learning_rate": 8.508210239396639e-05, "loss": 0.2647759020328522, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 39.98, "memory/max_allocated (GiB)": 39.98, "ppl": 1.30314, "step": 156, "tokens/total": 5503120, "tokens/train_per_sec_per_gpu": 80.86, "tokens/trainable": 243792} +{"epoch": 0.61328125, "grad_norm": 1.1783571243286133, "learning_rate": 8.486558279760375e-05, "loss": 0.2503865659236908, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 35.96, "memory/max_allocated (GiB)": 35.96, "ppl": 1.28452, "step": 157, "tokens/total": 5534352, "tokens/train_per_sec_per_gpu": 96.07, "tokens/trainable": 245317} +{"epoch": 0.6171875, "grad_norm": 1.0367603302001953, "learning_rate": 8.464782037243449e-05, "loss": 0.24144241213798523, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.27308, "step": 158, "tokens/total": 5566992, "tokens/train_per_sec_per_gpu": 93.14, "tokens/trainable": 246845} +{"epoch": 0.62109375, "grad_norm": 1.1168087720870972, "learning_rate": 8.442882418044202e-05, "loss": 0.2539154887199402, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.28906, "step": 159, "tokens/total": 5600000, "tokens/train_per_sec_per_gpu": 92.69, "tokens/trainable": 248329} +{"epoch": 0.625, "grad_norm": 0.93000727891922, "learning_rate": 8.420860333495179e-05, "loss": 0.22207707166671753, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.24867, "step": 160, "tokens/total": 5634320, "tokens/train_per_sec_per_gpu": 92.22, "tokens/trainable": 249816} +{"epoch": 0.62890625, "grad_norm": 1.1529980897903442, "learning_rate": 8.398716700025208e-05, "loss": 0.2281579077243805, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.25628, "step": 161, "tokens/total": 5668144, "tokens/train_per_sec_per_gpu": 89.53, "tokens/trainable": 251369} +{"epoch": 0.6328125, "grad_norm": 1.3325915336608887, "learning_rate": 8.376452439121266e-05, "loss": 0.2518617510795593, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 36.12, "memory/max_allocated (GiB)": 36.12, "ppl": 1.28642, "step": 162, "tokens/total": 5699008, "tokens/train_per_sec_per_gpu": 97.99, "tokens/trainable": 252865} +{"epoch": 0.63671875, "grad_norm": 2.4930155277252197, "learning_rate": 8.354068477290124e-05, "loss": 0.2510659098625183, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.28539, "step": 163, "tokens/total": 5733648, "tokens/train_per_sec_per_gpu": 86.91, "tokens/trainable": 254344} +{"epoch": 0.640625, "grad_norm": 0.9890461564064026, "learning_rate": 8.331565746019807e-05, "loss": 0.2246101051568985, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 36.9, "memory/max_allocated (GiB)": 36.9, "ppl": 1.25183, "step": 164, "tokens/total": 5766112, "tokens/train_per_sec_per_gpu": 94.22, "tokens/trainable": 255872} +{"epoch": 0.64453125, "grad_norm": 1.1010463237762451, "learning_rate": 8.308945181740812e-05, "loss": 0.21776220202445984, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 39.46, "memory/max_allocated (GiB)": 39.46, "ppl": 1.24329, "step": 165, "tokens/total": 5801456, "tokens/train_per_sec_per_gpu": 85.81, "tokens/trainable": 257404} +{"epoch": 0.6484375, "grad_norm": 0.8844090104103088, "learning_rate": 8.286207725787153e-05, "loss": 0.23068341612815857, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.25946, "step": 166, "tokens/total": 5838944, "tokens/train_per_sec_per_gpu": 75.75, "tokens/trainable": 258916} +{"epoch": 0.65234375, "grad_norm": 1.4015991687774658, "learning_rate": 8.263354324357182e-05, "loss": 0.20903480052947998, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 39.28, "memory/max_allocated (GiB)": 39.28, "ppl": 1.23249, "step": 167, "tokens/total": 5874144, "tokens/train_per_sec_per_gpu": 86.32, "tokens/trainable": 260455} +{"epoch": 0.65625, "grad_norm": 1.2398518323898315, "learning_rate": 8.240385928474219e-05, "loss": 0.2764233350753784, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 37.02, "memory/max_allocated (GiB)": 37.02, "ppl": 1.31841, "step": 168, "tokens/total": 5907648, "tokens/train_per_sec_per_gpu": 92.47, "tokens/trainable": 261918} +{"epoch": 0.66015625, "grad_norm": 0.9818920493125916, "learning_rate": 8.217303493946967e-05, "loss": 0.2146567702293396, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 37.43, "memory/max_allocated (GiB)": 37.43, "ppl": 1.23944, "step": 169, "tokens/total": 5942288, "tokens/train_per_sec_per_gpu": 86.56, "tokens/trainable": 263446} +{"epoch": 0.6640625, "grad_norm": 1.1002765893936157, "learning_rate": 8.194107981329746e-05, "loss": 0.21569499373435974, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 39.99, "memory/max_allocated (GiB)": 39.99, "ppl": 1.24072, "step": 170, "tokens/total": 5981488, "tokens/train_per_sec_per_gpu": 75.03, "tokens/trainable": 264984} +{"epoch": 0.66796875, "grad_norm": 10.955070495605469, "learning_rate": 8.170800355882518e-05, "loss": 0.2271931767463684, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.25507, "step": 171, "tokens/total": 6016672, "tokens/train_per_sec_per_gpu": 97.73, "tokens/trainable": 266608} +{"epoch": 0.671875, "grad_norm": 1.4425798654556274, "learning_rate": 8.147381587530713e-05, "loss": 0.2510474622249603, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 37.58, "memory/max_allocated (GiB)": 37.58, "ppl": 1.28537, "step": 172, "tokens/total": 6050928, "tokens/train_per_sec_per_gpu": 89.72, "tokens/trainable": 268163} +{"epoch": 0.67578125, "grad_norm": 2.331120014190674, "learning_rate": 8.123852650824877e-05, "loss": 0.2354554682970047, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 37.57, "memory/max_allocated (GiB)": 37.57, "ppl": 1.26549, "step": 173, "tokens/total": 6085024, "tokens/train_per_sec_per_gpu": 89.17, "tokens/trainable": 269722} +{"epoch": 0.6796875, "grad_norm": 1.12289297580719, "learning_rate": 8.100214524900103e-05, "loss": 0.23646047711372375, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.26676, "step": 174, "tokens/total": 6120688, "tokens/train_per_sec_per_gpu": 79.16, "tokens/trainable": 271178} +{"epoch": 0.68359375, "grad_norm": 0.7865145206451416, "learning_rate": 8.076468193435301e-05, "loss": 0.23727542161941528, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 36.58, "memory/max_allocated (GiB)": 36.58, "ppl": 1.26779, "step": 175, "tokens/total": 6151936, "tokens/train_per_sec_per_gpu": 93.21, "tokens/trainable": 272649} +{"epoch": 0.6875, "grad_norm": 0.9566035866737366, "learning_rate": 8.052614644612253e-05, "loss": 0.22304773330688477, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 39.63, "memory/max_allocated (GiB)": 39.63, "ppl": 1.24988, "step": 176, "tokens/total": 6190288, "tokens/train_per_sec_per_gpu": 77.83, "tokens/trainable": 274305} +{"epoch": 0.69140625, "grad_norm": 3.0142109394073486, "learning_rate": 8.028654871074489e-05, "loss": 0.19533059000968933, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.21571, "step": 177, "tokens/total": 6225696, "tokens/train_per_sec_per_gpu": 87.37, "tokens/trainable": 275892} +{"epoch": 0.6953125, "grad_norm": 0.8419945240020752, "learning_rate": 8.004589869885986e-05, "loss": 0.22846856713294983, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 36.66, "memory/max_allocated (GiB)": 36.66, "ppl": 1.25667, "step": 178, "tokens/total": 6257744, "tokens/train_per_sec_per_gpu": 86.58, "tokens/trainable": 277329} +{"epoch": 0.69921875, "grad_norm": 1.1325619220733643, "learning_rate": 7.980420642489674e-05, "loss": 0.21475258469581604, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 39.09, "memory/max_allocated (GiB)": 39.09, "ppl": 1.23956, "step": 179, "tokens/total": 6297088, "tokens/train_per_sec_per_gpu": 80.71, "tokens/trainable": 278953} +{"epoch": 0.703125, "grad_norm": 1.8927041292190552, "learning_rate": 7.95614819466576e-05, "loss": 0.21107541024684906, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.23501, "step": 180, "tokens/total": 6331520, "tokens/train_per_sec_per_gpu": 92.68, "tokens/trainable": 280577} +{"epoch": 0.70703125, "grad_norm": 0.8534265756607056, "learning_rate": 7.931773536489872e-05, "loss": 0.20850348472595215, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.23183, "step": 181, "tokens/total": 6366496, "tokens/train_per_sec_per_gpu": 85.41, "tokens/trainable": 282111} +{"epoch": 0.7109375, "grad_norm": 1.0646793842315674, "learning_rate": 7.907297682291035e-05, "loss": 0.19835379719734192, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 39.66, "memory/max_allocated (GiB)": 39.66, "ppl": 1.21939, "step": 182, "tokens/total": 6401856, "tokens/train_per_sec_per_gpu": 85.92, "tokens/trainable": 283658} +{"epoch": 0.71484375, "grad_norm": 0.9570431709289551, "learning_rate": 7.882721650609442e-05, "loss": 0.21680834889411926, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 39.7, "memory/max_allocated (GiB)": 39.7, "ppl": 1.24211, "step": 183, "tokens/total": 6442752, "tokens/train_per_sec_per_gpu": 75.27, "tokens/trainable": 285337} +{"epoch": 0.71875, "grad_norm": 1.152407169342041, "learning_rate": 7.85804646415409e-05, "loss": 0.21728397905826569, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 38.16, "memory/max_allocated (GiB)": 38.16, "ppl": 1.2427, "step": 184, "tokens/total": 6478640, "tokens/train_per_sec_per_gpu": 95.11, "tokens/trainable": 286936} +{"epoch": 0.72265625, "grad_norm": 1.259604573249817, "learning_rate": 7.833273149760207e-05, "loss": 0.19873547554016113, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 36.93, "memory/max_allocated (GiB)": 36.93, "ppl": 1.21986, "step": 185, "tokens/total": 6510128, "tokens/train_per_sec_per_gpu": 87.36, "tokens/trainable": 288509} +{"epoch": 0.7265625, "grad_norm": 1.470166563987732, "learning_rate": 7.808402738346527e-05, "loss": 0.1908443421125412, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 39.39, "memory/max_allocated (GiB)": 39.39, "ppl": 1.21027, "step": 186, "tokens/total": 6549600, "tokens/train_per_sec_per_gpu": 77.2, "tokens/trainable": 290136} +{"epoch": 0.73046875, "grad_norm": 1.0955746173858643, "learning_rate": 7.783436264872382e-05, "loss": 0.19103969633579254, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.21051, "step": 187, "tokens/total": 6586928, "tokens/train_per_sec_per_gpu": 87.05, "tokens/trainable": 291838} +{"epoch": 0.734375, "grad_norm": 1.3501133918762207, "learning_rate": 7.758374768294647e-05, "loss": 0.1945192515850067, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 1.21473, "step": 188, "tokens/total": 6623408, "tokens/train_per_sec_per_gpu": 86.32, "tokens/trainable": 293416} +{"epoch": 0.73828125, "grad_norm": 1.5331922769546509, "learning_rate": 7.733219291524489e-05, "loss": 0.2029617428779602, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.22503, "step": 189, "tokens/total": 6657456, "tokens/train_per_sec_per_gpu": 92.63, "tokens/trainable": 295003} +{"epoch": 0.7421875, "grad_norm": 1.0137052536010742, "learning_rate": 7.707970881383977e-05, "loss": 0.2055131494998932, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 36.57, "memory/max_allocated (GiB)": 36.57, "ppl": 1.22816, "step": 190, "tokens/total": 6689536, "tokens/train_per_sec_per_gpu": 94.14, "tokens/trainable": 296474} +{"epoch": 0.74609375, "grad_norm": 0.82082599401474, "learning_rate": 7.682630588562518e-05, "loss": 0.19373255968093872, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.21377, "step": 191, "tokens/total": 6725072, "tokens/train_per_sec_per_gpu": 83.68, "tokens/trainable": 298035} +{"epoch": 0.75, "grad_norm": 1.0225852727890015, "learning_rate": 7.657199467573129e-05, "loss": 0.19792261719703674, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 37.24, "memory/max_allocated (GiB)": 37.24, "ppl": 1.21887, "step": 192, "tokens/total": 6759424, "tokens/train_per_sec_per_gpu": 87.68, "tokens/trainable": 299551} +{"epoch": 0.75390625, "grad_norm": 0.914363443851471, "learning_rate": 7.631678576708561e-05, "loss": 0.17477485537528992, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.19098, "step": 193, "tokens/total": 6794544, "tokens/train_per_sec_per_gpu": 97.22, "tokens/trainable": 301192} +{"epoch": 0.7578125, "grad_norm": 1.3186616897583008, "learning_rate": 7.606068977997255e-05, "loss": 0.1993125081062317, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 39.79, "memory/max_allocated (GiB)": 39.79, "ppl": 1.22056, "step": 194, "tokens/total": 6832752, "tokens/train_per_sec_per_gpu": 85.93, "tokens/trainable": 302848} +{"epoch": 0.76171875, "grad_norm": 1.1046404838562012, "learning_rate": 7.580371737159148e-05, "loss": 0.22680559754371643, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 38.14, "memory/max_allocated (GiB)": 38.14, "ppl": 1.25459, "step": 195, "tokens/total": 6869296, "tokens/train_per_sec_per_gpu": 83.76, "tokens/trainable": 304363} +{"epoch": 0.765625, "grad_norm": 0.9578995704650879, "learning_rate": 7.554587923561324e-05, "loss": 0.17208179831504822, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 39.19, "memory/max_allocated (GiB)": 39.19, "ppl": 1.18777, "step": 196, "tokens/total": 6904592, "tokens/train_per_sec_per_gpu": 80.11, "tokens/trainable": 305870} +{"epoch": 0.76953125, "grad_norm": 1.7768745422363281, "learning_rate": 7.528718610173511e-05, "loss": 0.22281469404697418, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.24959, "step": 197, "tokens/total": 6940384, "tokens/train_per_sec_per_gpu": 86.18, "tokens/trainable": 307392} +{"epoch": 0.7734375, "grad_norm": 0.8800326585769653, "learning_rate": 7.502764873523431e-05, "loss": 0.1841493546962738, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.2022, "step": 198, "tokens/total": 6975008, "tokens/train_per_sec_per_gpu": 86.69, "tokens/trainable": 308969} +{"epoch": 0.77734375, "grad_norm": 0.8604999780654907, "learning_rate": 7.476727793652011e-05, "loss": 0.16934244334697723, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 36.93, "memory/max_allocated (GiB)": 36.93, "ppl": 1.18453, "step": 199, "tokens/total": 7008688, "tokens/train_per_sec_per_gpu": 91.13, "tokens/trainable": 310514} +{"epoch": 0.78125, "grad_norm": 1.2568310499191284, "learning_rate": 7.450608454068415e-05, "loss": 0.18060070276260376, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 37.93, "memory/max_allocated (GiB)": 37.93, "ppl": 1.19794, "step": 200, "tokens/total": 7043488, "tokens/train_per_sec_per_gpu": 91.53, "tokens/trainable": 312101} +{"epoch": 0.78515625, "grad_norm": 0.9105098843574524, "learning_rate": 7.424407941704987e-05, "loss": 0.16458794474601746, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 38.7, "memory/max_allocated (GiB)": 38.7, "ppl": 1.17891, "step": 201, "tokens/total": 7080880, "tokens/train_per_sec_per_gpu": 89.37, "tokens/trainable": 313745} +{"epoch": 0.7890625, "grad_norm": 0.8851780891418457, "learning_rate": 7.398127346871986e-05, "loss": 0.1793394684791565, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 39.89, "memory/max_allocated (GiB)": 39.89, "ppl": 1.19643, "step": 202, "tokens/total": 7118352, "tokens/train_per_sec_per_gpu": 82.02, "tokens/trainable": 315307} +{"epoch": 0.79296875, "grad_norm": 1.2462087869644165, "learning_rate": 7.371767763212238e-05, "loss": 0.19591838121414185, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 39.67, "memory/max_allocated (GiB)": 39.67, "ppl": 1.21643, "step": 203, "tokens/total": 7154000, "tokens/train_per_sec_per_gpu": 86.57, "tokens/trainable": 316817} +{"epoch": 0.796875, "grad_norm": 12.865723609924316, "learning_rate": 7.345330287655617e-05, "loss": 0.20050542056560516, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.22202, "step": 204, "tokens/total": 7192576, "tokens/train_per_sec_per_gpu": 75.92, "tokens/trainable": 318369} +{"epoch": 0.80078125, "grad_norm": 1.253357172012329, "learning_rate": 7.31881602037339e-05, "loss": 0.20774364471435547, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 36.83, "memory/max_allocated (GiB)": 36.83, "ppl": 1.2309, "step": 205, "tokens/total": 7225152, "tokens/train_per_sec_per_gpu": 89.43, "tokens/trainable": 319874} +{"epoch": 0.8046875, "grad_norm": 0.7888283729553223, "learning_rate": 7.29222606473245e-05, "loss": 0.15629009902477264, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.16917, "step": 206, "tokens/total": 7258592, "tokens/train_per_sec_per_gpu": 94.66, "tokens/trainable": 321431} +{"epoch": 0.80859375, "grad_norm": 1.0630977153778076, "learning_rate": 7.265561527249383e-05, "loss": 0.19263243675231934, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.21244, "step": 207, "tokens/total": 7293008, "tokens/train_per_sec_per_gpu": 89.12, "tokens/trainable": 322938} +{"epoch": 0.8125, "grad_norm": 1.1108037233352661, "learning_rate": 7.238823517544436e-05, "loss": 0.15830348432064056, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 39.65, "memory/max_allocated (GiB)": 39.65, "ppl": 1.17152, "step": 208, "tokens/total": 7330496, "tokens/train_per_sec_per_gpu": 93.22, "tokens/trainable": 324584} +{"epoch": 0.81640625, "grad_norm": 0.8043308854103088, "learning_rate": 7.212013148295333e-05, "loss": 0.1721184253692627, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 37.73, "memory/max_allocated (GiB)": 37.73, "ppl": 1.18782, "step": 209, "tokens/total": 7365104, "tokens/train_per_sec_per_gpu": 90.13, "tokens/trainable": 326152} +{"epoch": 0.8203125, "grad_norm": 1.0919499397277832, "learning_rate": 7.185131535190975e-05, "loss": 0.17645546793937683, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.19298, "step": 210, "tokens/total": 7398144, "tokens/train_per_sec_per_gpu": 94.69, "tokens/trainable": 327785} +{"epoch": 0.82421875, "grad_norm": 1.0153354406356812, "learning_rate": 7.158179796885005e-05, "loss": 0.18390634655952454, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 37.84, "memory/max_allocated (GiB)": 37.84, "ppl": 1.2019, "step": 211, "tokens/total": 7433152, "tokens/train_per_sec_per_gpu": 98.71, "tokens/trainable": 329418} +{"epoch": 0.828125, "grad_norm": 2.4614086151123047, "learning_rate": 7.131159054949273e-05, "loss": 0.19116318225860596, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 1.21066, "step": 212, "tokens/total": 7472240, "tokens/train_per_sec_per_gpu": 79.85, "tokens/trainable": 330998} +{"epoch": 0.83203125, "grad_norm": 0.9896388649940491, "learning_rate": 7.104070433827139e-05, "loss": 0.19752472639083862, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.21838, "step": 213, "tokens/total": 7508496, "tokens/train_per_sec_per_gpu": 81.46, "tokens/trainable": 332483} +{"epoch": 0.8359375, "grad_norm": 0.9634982347488403, "learning_rate": 7.076915060786705e-05, "loss": 0.19128850102424622, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 37.93, "memory/max_allocated (GiB)": 37.93, "ppl": 1.21081, "step": 214, "tokens/total": 7541504, "tokens/train_per_sec_per_gpu": 86.68, "tokens/trainable": 333937} +{"epoch": 0.83984375, "grad_norm": 1.1089407205581665, "learning_rate": 7.049694065873882e-05, "loss": 0.18596242368221283, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.20438, "step": 215, "tokens/total": 7576688, "tokens/train_per_sec_per_gpu": 85.01, "tokens/trainable": 335485} +{"epoch": 0.84375, "grad_norm": 0.732020378112793, "learning_rate": 7.022408581865382e-05, "loss": 0.17049840092658997, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.1859, "step": 216, "tokens/total": 7611632, "tokens/train_per_sec_per_gpu": 83.19, "tokens/trainable": 337037} +{"epoch": 0.84765625, "grad_norm": 0.9406604170799255, "learning_rate": 6.99505974422157e-05, "loss": 0.19105073809623718, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.21052, "step": 217, "tokens/total": 7646592, "tokens/train_per_sec_per_gpu": 94.86, "tokens/trainable": 338621} +{"epoch": 0.8515625, "grad_norm": 0.8575993180274963, "learning_rate": 6.967648691039213e-05, "loss": 0.18445634841918945, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.20256, "step": 218, "tokens/total": 7679184, "tokens/train_per_sec_per_gpu": 89.69, "tokens/trainable": 340142} +{"epoch": 0.85546875, "grad_norm": 1.0032178163528442, "learning_rate": 6.940176563004123e-05, "loss": 0.18432079255580902, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.2024, "step": 219, "tokens/total": 7713600, "tokens/train_per_sec_per_gpu": 86.12, "tokens/trainable": 341636} +{"epoch": 0.859375, "grad_norm": 1.125375747680664, "learning_rate": 6.912644503343682e-05, "loss": 0.19717910885810852, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.21796, "step": 220, "tokens/total": 7752080, "tokens/train_per_sec_per_gpu": 74.92, "tokens/trainable": 343111} +{"epoch": 0.86328125, "grad_norm": 1.287402629852295, "learning_rate": 6.885053657779273e-05, "loss": 0.16239142417907715, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 36.71, "memory/max_allocated (GiB)": 36.71, "ppl": 1.17632, "step": 221, "tokens/total": 7784496, "tokens/train_per_sec_per_gpu": 96.16, "tokens/trainable": 344720} +{"epoch": 0.8671875, "grad_norm": 1.0550240278244019, "learning_rate": 6.857405174478604e-05, "loss": 0.16507530212402344, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.17948, "step": 222, "tokens/total": 7819680, "tokens/train_per_sec_per_gpu": 88.67, "tokens/trainable": 346347} +{"epoch": 0.87109375, "grad_norm": 0.8416323065757751, "learning_rate": 6.82970020400792e-05, "loss": 0.17801573872566223, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 39.58, "memory/max_allocated (GiB)": 39.58, "ppl": 1.19484, "step": 223, "tokens/total": 7857328, "tokens/train_per_sec_per_gpu": 79.58, "tokens/trainable": 347854} +{"epoch": 0.875, "grad_norm": 0.9734751582145691, "learning_rate": 6.801939899284132e-05, "loss": 0.14367526769638062, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 38.45, "memory/max_allocated (GiB)": 38.45, "ppl": 1.15451, "step": 224, "tokens/total": 7893712, "tokens/train_per_sec_per_gpu": 84.98, "tokens/trainable": 349434} +{"epoch": 0.87890625, "grad_norm": 0.8361388444900513, "learning_rate": 6.774125415526827e-05, "loss": 0.14647582173347473, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 38.21, "memory/max_allocated (GiB)": 38.21, "ppl": 1.15775, "step": 225, "tokens/total": 7927600, "tokens/train_per_sec_per_gpu": 101.4, "tokens/trainable": 351122} +{"epoch": 0.8828125, "grad_norm": 0.8409698605537415, "learning_rate": 6.746257910210214e-05, "loss": 0.15676161646842957, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 37.19, "memory/max_allocated (GiB)": 37.19, "ppl": 1.16972, "step": 226, "tokens/total": 7961168, "tokens/train_per_sec_per_gpu": 85.89, "tokens/trainable": 352719} +{"epoch": 0.88671875, "grad_norm": 0.8681503534317017, "learning_rate": 6.718338543014937e-05, "loss": 0.1795649230480194, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 35.85, "memory/max_allocated (GiB)": 35.85, "ppl": 1.1967, "step": 227, "tokens/total": 7991808, "tokens/train_per_sec_per_gpu": 92.72, "tokens/trainable": 354166} +{"epoch": 0.890625, "grad_norm": 0.9321914315223694, "learning_rate": 6.69036847577983e-05, "loss": 0.1533103585243225, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.16569, "step": 228, "tokens/total": 8026416, "tokens/train_per_sec_per_gpu": 95.02, "tokens/trainable": 355766} +{"epoch": 0.89453125, "grad_norm": 1.1452224254608154, "learning_rate": 6.662348872453553e-05, "loss": 0.15769441425800323, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.17081, "step": 229, "tokens/total": 8063312, "tokens/train_per_sec_per_gpu": 81.13, "tokens/trainable": 357249} +{"epoch": 0.8984375, "grad_norm": 1.8792729377746582, "learning_rate": 6.63428089904618e-05, "loss": 0.16068559885025024, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.17432, "step": 230, "tokens/total": 8102800, "tokens/train_per_sec_per_gpu": 75.84, "tokens/trainable": 358762} +{"epoch": 0.90234375, "grad_norm": 1.0610027313232422, "learning_rate": 6.60616572358065e-05, "loss": 0.16268372535705566, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 37.48, "memory/max_allocated (GiB)": 37.48, "ppl": 1.17666, "step": 231, "tokens/total": 8137664, "tokens/train_per_sec_per_gpu": 88.49, "tokens/trainable": 360263} +{"epoch": 0.90625, "grad_norm": 1.3099452257156372, "learning_rate": 6.578004516044172e-05, "loss": 0.1490778625011444, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 38.08, "memory/max_allocated (GiB)": 38.08, "ppl": 1.16076, "step": 232, "tokens/total": 8174272, "tokens/train_per_sec_per_gpu": 93.74, "tokens/trainable": 361885} +{"epoch": 0.91015625, "grad_norm": 1.3567477464675903, "learning_rate": 6.549798448339548e-05, "loss": 0.17954090237617493, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.19667, "step": 233, "tokens/total": 8212144, "tokens/train_per_sec_per_gpu": 84.61, "tokens/trainable": 363440} +{"epoch": 0.9140625, "grad_norm": 1.4696807861328125, "learning_rate": 6.521548694236384e-05, "loss": 0.18042847514152527, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.19773, "step": 234, "tokens/total": 8249952, "tokens/train_per_sec_per_gpu": 74.67, "tokens/trainable": 364918} +{"epoch": 0.91796875, "grad_norm": 1.1321297883987427, "learning_rate": 6.493256429322259e-05, "loss": 0.1674194186925888, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.18225, "step": 235, "tokens/total": 8286672, "tokens/train_per_sec_per_gpu": 83.91, "tokens/trainable": 366448} +{"epoch": 0.921875, "grad_norm": 0.9480315446853638, "learning_rate": 6.464922830953799e-05, "loss": 0.17157310247421265, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 39.46, "memory/max_allocated (GiB)": 39.46, "ppl": 1.18717, "step": 236, "tokens/total": 8326224, "tokens/train_per_sec_per_gpu": 72.46, "tokens/trainable": 368001} +{"epoch": 0.92578125, "grad_norm": 0.8166792392730713, "learning_rate": 6.436549078207688e-05, "loss": 0.16519959270954132, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.17963, "step": 237, "tokens/total": 8359760, "tokens/train_per_sec_per_gpu": 93.14, "tokens/trainable": 369564} +{"epoch": 0.9296875, "grad_norm": 0.9153592586517334, "learning_rate": 6.408136351831592e-05, "loss": 0.1647965908050537, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.17915, "step": 238, "tokens/total": 8392672, "tokens/train_per_sec_per_gpu": 92.86, "tokens/trainable": 371086} +{"epoch": 0.93359375, "grad_norm": 0.8913624286651611, "learning_rate": 6.379685834195036e-05, "loss": 0.15114600956439972, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 39.32, "memory/max_allocated (GiB)": 39.32, "ppl": 1.16317, "step": 239, "tokens/total": 8432688, "tokens/train_per_sec_per_gpu": 76.71, "tokens/trainable": 372755} +{"epoch": 0.9375, "grad_norm": 0.8373076915740967, "learning_rate": 6.351198709240186e-05, "loss": 0.14772528409957886, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 39.66, "memory/max_allocated (GiB)": 39.66, "ppl": 1.15919, "step": 240, "tokens/total": 8473168, "tokens/train_per_sec_per_gpu": 74.51, "tokens/trainable": 374272} +{"epoch": 0.94140625, "grad_norm": 1.5061434507369995, "learning_rate": 6.32267616243259e-05, "loss": 0.18742328882217407, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 36.79, "memory/max_allocated (GiB)": 36.79, "ppl": 1.20614, "step": 241, "tokens/total": 8506128, "tokens/train_per_sec_per_gpu": 91.15, "tokens/trainable": 375821} +{"epoch": 0.9453125, "grad_norm": 1.3699601888656616, "learning_rate": 6.294119380711849e-05, "loss": 0.17168782651424408, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.18731, "step": 242, "tokens/total": 8541184, "tokens/train_per_sec_per_gpu": 83.2, "tokens/trainable": 377298} +{"epoch": 0.94921875, "grad_norm": 1.0895007848739624, "learning_rate": 6.265529552442209e-05, "loss": 0.17275622487068176, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.18858, "step": 243, "tokens/total": 8577328, "tokens/train_per_sec_per_gpu": 89.62, "tokens/trainable": 378919} +{"epoch": 0.953125, "grad_norm": 1.9427142143249512, "learning_rate": 6.236907867363127e-05, "loss": 0.15653830766677856, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 1.16946, "step": 244, "tokens/total": 8612448, "tokens/train_per_sec_per_gpu": 88.8, "tokens/trainable": 380489} +{"epoch": 0.95703125, "grad_norm": 0.8099203705787659, "learning_rate": 6.208255516539749e-05, "loss": 0.14181286096572876, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 36.58, "memory/max_allocated (GiB)": 36.58, "ppl": 1.15236, "step": 245, "tokens/total": 8645232, "tokens/train_per_sec_per_gpu": 96.47, "tokens/trainable": 382121} +{"epoch": 0.9609375, "grad_norm": 0.7733524441719055, "learning_rate": 6.179573692313344e-05, "loss": 0.15796509385108948, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 39.15, "memory/max_allocated (GiB)": 39.15, "ppl": 1.17113, "step": 246, "tokens/total": 8683056, "tokens/train_per_sec_per_gpu": 86.05, "tokens/trainable": 383716} +{"epoch": 0.96484375, "grad_norm": 0.9631892442703247, "learning_rate": 6.150863588251694e-05, "loss": 0.15635499358177185, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 39.66, "memory/max_allocated (GiB)": 39.66, "ppl": 1.16924, "step": 247, "tokens/total": 8723760, "tokens/train_per_sec_per_gpu": 75.79, "tokens/trainable": 385292} +{"epoch": 0.96875, "grad_norm": 1.9508496522903442, "learning_rate": 6.122126399099419e-05, "loss": 0.16369423270225525, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 39.98, "memory/max_allocated (GiB)": 39.98, "ppl": 1.17785, "step": 248, "tokens/total": 8764272, "tokens/train_per_sec_per_gpu": 84.28, "tokens/trainable": 387005} +{"epoch": 0.97265625, "grad_norm": 0.9123312830924988, "learning_rate": 6.0933633207282615e-05, "loss": 0.18441031873226166, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.20251, "step": 249, "tokens/total": 8801360, "tokens/train_per_sec_per_gpu": 90.24, "tokens/trainable": 388654} +{"epoch": 0.9765625, "grad_norm": 1.3926302194595337, "learning_rate": 6.064575550087316e-05, "loss": 0.16329443454742432, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 39.74, "memory/max_allocated (GiB)": 39.74, "ppl": 1.17738, "step": 250, "tokens/total": 8838672, "tokens/train_per_sec_per_gpu": 83.28, "tokens/trainable": 390217} +{"epoch": 0.98046875, "grad_norm": 0.7525199055671692, "learning_rate": 6.0357642851532245e-05, "loss": 0.15430805087089539, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 39.05, "memory/max_allocated (GiB)": 39.05, "ppl": 1.16685, "step": 251, "tokens/total": 8875568, "tokens/train_per_sec_per_gpu": 81.97, "tokens/trainable": 391782} +{"epoch": 0.984375, "grad_norm": 1.025300145149231, "learning_rate": 6.0069307248803294e-05, "loss": 0.16723525524139404, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 38.18, "memory/max_allocated (GiB)": 38.18, "ppl": 1.18203, "step": 252, "tokens/total": 8911680, "tokens/train_per_sec_per_gpu": 85.96, "tokens/trainable": 393319} +{"epoch": 0.98828125, "grad_norm": 0.7362737059593201, "learning_rate": 5.9780760691507635e-05, "loss": 0.15022620558738708, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 36.47, "memory/max_allocated (GiB)": 36.47, "ppl": 1.1621, "step": 253, "tokens/total": 8943472, "tokens/train_per_sec_per_gpu": 97.56, "tokens/trainable": 394845} +{"epoch": 0.9921875, "grad_norm": 0.728873074054718, "learning_rate": 5.9492015187245334e-05, "loss": 0.1322578489780426, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.1414, "step": 254, "tokens/total": 8981888, "tokens/train_per_sec_per_gpu": 83.26, "tokens/trainable": 396398} +{"epoch": 0.99609375, "grad_norm": 0.6413206458091736, "learning_rate": 5.920308275189541e-05, "loss": 0.14415021240711212, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 36.79, "memory/max_allocated (GiB)": 36.79, "ppl": 1.15506, "step": 255, "tokens/total": 9014336, "tokens/train_per_sec_per_gpu": 94.05, "tokens/trainable": 397900} +{"epoch": 1.0, "grad_norm": 0.7663645148277283, "learning_rate": 5.8913975409115874e-05, "loss": 0.15318122506141663, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 37.87, "memory/max_allocated (GiB)": 37.87, "ppl": 1.16554, "step": 256, "tokens/total": 9048544, "tokens/train_per_sec_per_gpu": 85.92, "tokens/trainable": 399469} +{"epoch": 1.00390625, "grad_norm": 0.5811907052993774, "learning_rate": 5.8624705189843395e-05, "loss": 0.12972232699394226, "memory/device_reserved (GiB)": 43.09, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.13851, "step": 257, "tokens/total": 9086240, "tokens/train_per_sec_per_gpu": 79.37, "tokens/trainable": 401059} +{"epoch": 1.0078125, "grad_norm": 0.9389148950576782, "learning_rate": 5.833528413179249e-05, "loss": 0.13162639737129211, "memory/device_reserved (GiB)": 40.53, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.14068, "step": 258, "tokens/total": 9123216, "tokens/train_per_sec_per_gpu": 87.39, "tokens/trainable": 402645} +{"epoch": 1.01171875, "grad_norm": 0.7820397019386292, "learning_rate": 5.80457242789548e-05, "loss": 0.1220172643661499, "memory/device_reserved (GiB)": 42.25, "memory/max_active (GiB)": 39.7, "memory/max_allocated (GiB)": 39.7, "ppl": 1.12977, "step": 259, "tokens/total": 9163744, "tokens/train_per_sec_per_gpu": 80.28, "tokens/trainable": 404359} +{"epoch": 1.015625, "grad_norm": 0.8800066709518433, "learning_rate": 5.77560376810977e-05, "loss": 0.12407528609037399, "memory/device_reserved (GiB)": 42.27, "memory/max_active (GiB)": 38.53, "memory/max_allocated (GiB)": 38.53, "ppl": 1.1321, "step": 260, "tokens/total": 9197568, "tokens/train_per_sec_per_gpu": 90.18, "tokens/trainable": 405965} +{"epoch": 1.01953125, "grad_norm": 1.259178876876831, "learning_rate": 5.7466236393263005e-05, "loss": 0.13888490200042725, "memory/device_reserved (GiB)": 42.27, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.14899, "step": 261, "tokens/total": 9232672, "tokens/train_per_sec_per_gpu": 83.35, "tokens/trainable": 407450} +{"epoch": 1.0234375, "grad_norm": 1.7591556310653687, "learning_rate": 5.717633247526522e-05, "loss": 0.14296333491802216, "memory/device_reserved (GiB)": 42.27, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.15369, "step": 262, "tokens/total": 9264320, "tokens/train_per_sec_per_gpu": 101.08, "tokens/trainable": 408964} +{"epoch": 1.02734375, "grad_norm": 1.3230881690979004, "learning_rate": 5.688633799118971e-05, "loss": 0.15722793340682983, "memory/device_reserved (GiB)": 42.27, "memory/max_active (GiB)": 37.13, "memory/max_allocated (GiB)": 37.13, "ppl": 1.17026, "step": 263, "tokens/total": 9297168, "tokens/train_per_sec_per_gpu": 81.78, "tokens/trainable": 410357} +{"epoch": 1.03125, "grad_norm": 1.2976834774017334, "learning_rate": 5.659626500889066e-05, "loss": 0.12096689641475677, "memory/device_reserved (GiB)": 42.27, "memory/max_active (GiB)": 37.23, "memory/max_allocated (GiB)": 37.23, "ppl": 1.12859, "step": 264, "tokens/total": 9331504, "tokens/train_per_sec_per_gpu": 90.24, "tokens/trainable": 411923} +{"epoch": 1.03515625, "grad_norm": 1.1461232900619507, "learning_rate": 5.6306125599488905e-05, "loss": 0.1457008421421051, "memory/device_reserved (GiB)": 42.27, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.15685, "step": 265, "tokens/total": 9362656, "tokens/train_per_sec_per_gpu": 96.92, "tokens/trainable": 413398} +{"epoch": 1.0390625, "grad_norm": 0.9989601969718933, "learning_rate": 5.601593183686955e-05, "loss": 0.12717092037200928, "memory/device_reserved (GiB)": 42.27, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.13561, "step": 266, "tokens/total": 9400320, "tokens/train_per_sec_per_gpu": 87.38, "tokens/trainable": 415111} +{"epoch": 1.04296875, "grad_norm": 0.8825514316558838, "learning_rate": 5.572569579717961e-05, "loss": 0.14395436644554138, "memory/device_reserved (GiB)": 42.27, "memory/max_active (GiB)": 39.32, "memory/max_allocated (GiB)": 39.32, "ppl": 1.15483, "step": 267, "tokens/total": 9436048, "tokens/train_per_sec_per_gpu": 87.47, "tokens/trainable": 416697} +{"epoch": 1.046875, "grad_norm": 1.6549137830734253, "learning_rate": 5.543542955832538e-05, "loss": 0.14262384176254272, "memory/device_reserved (GiB)": 42.27, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.1533, "step": 268, "tokens/total": 9476320, "tokens/train_per_sec_per_gpu": 76.47, "tokens/trainable": 418243} +{"epoch": 1.05078125, "grad_norm": 0.8345425724983215, "learning_rate": 5.514514519946986e-05, "loss": 0.13465963304042816, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 39.78, "memory/max_allocated (GiB)": 39.78, "ppl": 1.14415, "step": 269, "tokens/total": 9517088, "tokens/train_per_sec_per_gpu": 82.77, "tokens/trainable": 419908} +{"epoch": 1.0546875, "grad_norm": 0.8609504699707031, "learning_rate": 5.485485480053015e-05, "loss": 0.12799368798732758, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 39.61, "memory/max_allocated (GiB)": 39.61, "ppl": 1.13655, "step": 270, "tokens/total": 9555856, "tokens/train_per_sec_per_gpu": 69.71, "tokens/trainable": 421384} +{"epoch": 1.05859375, "grad_norm": 0.9530445337295532, "learning_rate": 5.4564570441674645e-05, "loss": 0.13911239802837372, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.14925, "step": 271, "tokens/total": 9590048, "tokens/train_per_sec_per_gpu": 93.73, "tokens/trainable": 422922} +{"epoch": 1.0625, "grad_norm": 1.5347378253936768, "learning_rate": 5.42743042028204e-05, "loss": 0.14712345600128174, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 38.13, "memory/max_allocated (GiB)": 38.13, "ppl": 1.1585, "step": 272, "tokens/total": 9626768, "tokens/train_per_sec_per_gpu": 80.37, "tokens/trainable": 424430} +{"epoch": 1.06640625, "grad_norm": 4.977466583251953, "learning_rate": 5.3984068163130464e-05, "loss": 0.12130467593669891, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.12897, "step": 273, "tokens/total": 9660672, "tokens/train_per_sec_per_gpu": 94.16, "tokens/trainable": 426060} +{"epoch": 1.0703125, "grad_norm": 0.792270302772522, "learning_rate": 5.369387440051111e-05, "loss": 0.12599050998687744, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.13427, "step": 274, "tokens/total": 9694896, "tokens/train_per_sec_per_gpu": 93.67, "tokens/trainable": 427670} +{"epoch": 1.07421875, "grad_norm": 0.7408891916275024, "learning_rate": 5.340373499110935e-05, "loss": 0.12505170702934265, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.13321, "step": 275, "tokens/total": 9731088, "tokens/train_per_sec_per_gpu": 87.78, "tokens/trainable": 429335} +{"epoch": 1.078125, "grad_norm": 1.1125028133392334, "learning_rate": 5.3113662008810304e-05, "loss": 0.12621533870697021, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 39.63, "memory/max_allocated (GiB)": 39.63, "ppl": 1.13453, "step": 276, "tokens/total": 9771936, "tokens/train_per_sec_per_gpu": 75.0, "tokens/trainable": 430928} +{"epoch": 1.08203125, "grad_norm": 0.6609854102134705, "learning_rate": 5.282366752473479e-05, "loss": 0.12920156121253967, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 38.37, "memory/max_allocated (GiB)": 38.37, "ppl": 1.13792, "step": 277, "tokens/total": 9809584, "tokens/train_per_sec_per_gpu": 77.01, "tokens/trainable": 432471} +{"epoch": 1.0859375, "grad_norm": 1.020319938659668, "learning_rate": 5.2533763606737005e-05, "loss": 0.14345501363277435, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.15425, "step": 278, "tokens/total": 9849920, "tokens/train_per_sec_per_gpu": 68.31, "tokens/trainable": 434046} +{"epoch": 1.08984375, "grad_norm": 1.0478591918945312, "learning_rate": 5.224396231890232e-05, "loss": 0.13501109182834625, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.14455, "step": 279, "tokens/total": 9882320, "tokens/train_per_sec_per_gpu": 96.54, "tokens/trainable": 435665} +{"epoch": 1.09375, "grad_norm": 0.674826443195343, "learning_rate": 5.195427572104522e-05, "loss": 0.12968991696834564, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 39.17, "memory/max_allocated (GiB)": 39.17, "ppl": 1.13848, "step": 280, "tokens/total": 9917360, "tokens/train_per_sec_per_gpu": 90.66, "tokens/trainable": 437264} +{"epoch": 1.09765625, "grad_norm": 1.1972026824951172, "learning_rate": 5.166471586820751e-05, "loss": 0.1345352977514267, "memory/device_reserved (GiB)": 42.93, "memory/max_active (GiB)": 39.53, "memory/max_allocated (GiB)": 39.53, "ppl": 1.14401, "step": 281, "tokens/total": 9954752, "tokens/train_per_sec_per_gpu": 87.22, "tokens/trainable": 438875} +{"epoch": 1.1015625, "grad_norm": 0.6041335463523865, "learning_rate": 5.1375294810156615e-05, "loss": 0.12657013535499573, "memory/device_reserved (GiB)": 42.93, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.13493, "step": 282, "tokens/total": 9989664, "tokens/train_per_sec_per_gpu": 85.15, "tokens/trainable": 440386} +{"epoch": 1.10546875, "grad_norm": 1.034453272819519, "learning_rate": 5.1086024590884144e-05, "loss": 0.14445507526397705, "memory/device_reserved (GiB)": 42.93, "memory/max_active (GiB)": 39.49, "memory/max_allocated (GiB)": 39.49, "ppl": 1.15541, "step": 283, "tokens/total": 10028528, "tokens/train_per_sec_per_gpu": 72.25, "tokens/trainable": 441815} +{"epoch": 1.109375, "grad_norm": 0.5165565609931946, "learning_rate": 5.079691724810461e-05, "loss": 0.1222078800201416, "memory/device_reserved (GiB)": 42.93, "memory/max_active (GiB)": 38.36, "memory/max_allocated (GiB)": 38.36, "ppl": 1.12999, "step": 284, "tokens/total": 10064912, "tokens/train_per_sec_per_gpu": 77.46, "tokens/trainable": 443334} +{"epoch": 1.11328125, "grad_norm": 0.7176141738891602, "learning_rate": 5.0507984812754684e-05, "loss": 0.12668147683143616, "memory/device_reserved (GiB)": 42.93, "memory/max_active (GiB)": 37.44, "memory/max_allocated (GiB)": 37.44, "ppl": 1.13506, "step": 285, "tokens/total": 10098816, "tokens/train_per_sec_per_gpu": 93.7, "tokens/trainable": 444912} +{"epoch": 1.1171875, "grad_norm": 1.127332329750061, "learning_rate": 5.021923930849237e-05, "loss": 0.14133936166763306, "memory/device_reserved (GiB)": 42.93, "memory/max_active (GiB)": 37.43, "memory/max_allocated (GiB)": 37.43, "ppl": 1.15182, "step": 286, "tokens/total": 10133328, "tokens/train_per_sec_per_gpu": 84.94, "tokens/trainable": 446450} +{"epoch": 1.12109375, "grad_norm": 1.4551290273666382, "learning_rate": 4.99306927511967e-05, "loss": 0.12915478646755219, "memory/device_reserved (GiB)": 42.93, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.13787, "step": 287, "tokens/total": 10167152, "tokens/train_per_sec_per_gpu": 99.22, "tokens/trainable": 448059} +{"epoch": 1.125, "grad_norm": 1.320076584815979, "learning_rate": 4.964235714846775e-05, "loss": 0.13802780210971832, "memory/device_reserved (GiB)": 42.93, "memory/max_active (GiB)": 36.6, "memory/max_allocated (GiB)": 36.6, "ppl": 1.14801, "step": 288, "tokens/total": 10199392, "tokens/train_per_sec_per_gpu": 94.16, "tokens/trainable": 449566} +{"epoch": 1.12890625, "grad_norm": 0.9411768913269043, "learning_rate": 4.9354244499126866e-05, "loss": 0.15489083528518677, "memory/device_reserved (GiB)": 42.93, "memory/max_active (GiB)": 36.42, "memory/max_allocated (GiB)": 36.42, "ppl": 1.16753, "step": 289, "tokens/total": 10231552, "tokens/train_per_sec_per_gpu": 95.4, "tokens/trainable": 451176} +{"epoch": 1.1328125, "grad_norm": 0.9840365648269653, "learning_rate": 4.90663667927174e-05, "loss": 0.1335332989692688, "memory/device_reserved (GiB)": 42.93, "memory/max_active (GiB)": 38.1, "memory/max_allocated (GiB)": 38.1, "ppl": 1.14286, "step": 290, "tokens/total": 10264896, "tokens/train_per_sec_per_gpu": 98.88, "tokens/trainable": 452654} +{"epoch": 1.13671875, "grad_norm": 0.9891915917396545, "learning_rate": 4.877873600900581e-05, "loss": 0.12253017723560333, "memory/device_reserved (GiB)": 42.93, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.13035, "step": 291, "tokens/total": 10301680, "tokens/train_per_sec_per_gpu": 77.65, "tokens/trainable": 454179} +{"epoch": 1.140625, "grad_norm": 1.8260177373886108, "learning_rate": 4.849136411748306e-05, "loss": 0.1226571574807167, "memory/device_reserved (GiB)": 42.93, "memory/max_active (GiB)": 39.35, "memory/max_allocated (GiB)": 39.35, "ppl": 1.1305, "step": 292, "tokens/total": 10340304, "tokens/train_per_sec_per_gpu": 74.37, "tokens/trainable": 455733} +{"epoch": 1.14453125, "grad_norm": 0.8646610379219055, "learning_rate": 4.8204263076866574e-05, "loss": 0.12392278015613556, "memory/device_reserved (GiB)": 42.93, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.13193, "step": 293, "tokens/total": 10376800, "tokens/train_per_sec_per_gpu": 86.81, "tokens/trainable": 457398} +{"epoch": 1.1484375, "grad_norm": 0.7801808714866638, "learning_rate": 4.791744483460251e-05, "loss": 0.13933232426643372, "memory/device_reserved (GiB)": 42.93, "memory/max_active (GiB)": 37.97, "memory/max_allocated (GiB)": 37.97, "ppl": 1.14951, "step": 294, "tokens/total": 10411696, "tokens/train_per_sec_per_gpu": 82.59, "tokens/trainable": 458996} +{"epoch": 1.15234375, "grad_norm": 1.038492202758789, "learning_rate": 4.7630921326368736e-05, "loss": 0.12718379497528076, "memory/device_reserved (GiB)": 42.93, "memory/max_active (GiB)": 39.46, "memory/max_allocated (GiB)": 39.46, "ppl": 1.13563, "step": 295, "tokens/total": 10450592, "tokens/train_per_sec_per_gpu": 72.11, "tokens/trainable": 460492} +{"epoch": 1.15625, "grad_norm": 1.0812442302703857, "learning_rate": 4.7344704475577916e-05, "loss": 0.12571601569652557, "memory/device_reserved (GiB)": 42.94, "memory/max_active (GiB)": 35.95, "memory/max_allocated (GiB)": 35.95, "ppl": 1.13396, "step": 296, "tokens/total": 10480720, "tokens/train_per_sec_per_gpu": 109.91, "tokens/trainable": 462068} +{"epoch": 1.16015625, "grad_norm": 0.9887120127677917, "learning_rate": 4.705880619288153e-05, "loss": 0.13280904293060303, "memory/device_reserved (GiB)": 42.94, "memory/max_active (GiB)": 38.59, "memory/max_allocated (GiB)": 38.59, "ppl": 1.14203, "step": 297, "tokens/total": 10517632, "tokens/train_per_sec_per_gpu": 81.37, "tokens/trainable": 463596} +{"epoch": 1.1640625, "grad_norm": 0.818943977355957, "learning_rate": 4.677323837567412e-05, "loss": 0.11786806583404541, "memory/device_reserved (GiB)": 42.94, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.1251, "step": 298, "tokens/total": 10550992, "tokens/train_per_sec_per_gpu": 97.6, "tokens/trainable": 465241} +{"epoch": 1.16796875, "grad_norm": 2.378364086151123, "learning_rate": 4.6488012907598146e-05, "loss": 0.12516412138938904, "memory/device_reserved (GiB)": 42.94, "memory/max_active (GiB)": 36.67, "memory/max_allocated (GiB)": 36.67, "ppl": 1.13333, "step": 299, "tokens/total": 10583888, "tokens/train_per_sec_per_gpu": 85.73, "tokens/trainable": 466740} +{"epoch": 1.171875, "grad_norm": 0.6334974765777588, "learning_rate": 4.620314165804964e-05, "loss": 0.1271534264087677, "memory/device_reserved (GiB)": 42.94, "memory/max_active (GiB)": 38.51, "memory/max_allocated (GiB)": 38.51, "ppl": 1.13559, "step": 300, "tokens/total": 10620176, "tokens/train_per_sec_per_gpu": 81.15, "tokens/trainable": 468280} +{"epoch": 1.17578125, "grad_norm": 0.6571866869926453, "learning_rate": 4.591863648168407e-05, "loss": 0.11855052411556244, "memory/device_reserved (GiB)": 42.94, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.12586, "step": 301, "tokens/total": 10655280, "tokens/train_per_sec_per_gpu": 84.55, "tokens/trainable": 469845} +{"epoch": 1.1796875, "grad_norm": 0.702384352684021, "learning_rate": 4.5634509217923135e-05, "loss": 0.12865397334098816, "memory/device_reserved (GiB)": 42.94, "memory/max_active (GiB)": 38.1, "memory/max_allocated (GiB)": 38.1, "ppl": 1.1373, "step": 302, "tokens/total": 10689232, "tokens/train_per_sec_per_gpu": 92.81, "tokens/trainable": 471455} +{"epoch": 1.18359375, "grad_norm": 0.9202799201011658, "learning_rate": 4.535077169046201e-05, "loss": 0.12610246241092682, "memory/device_reserved (GiB)": 42.94, "memory/max_active (GiB)": 38.18, "memory/max_allocated (GiB)": 38.18, "ppl": 1.1344, "step": 303, "tokens/total": 10726496, "tokens/train_per_sec_per_gpu": 83.12, "tokens/trainable": 473017} +{"epoch": 1.1875, "grad_norm": 1.0197376012802124, "learning_rate": 4.506743570677743e-05, "loss": 0.13757118582725525, "memory/device_reserved (GiB)": 42.94, "memory/max_active (GiB)": 36.41, "memory/max_allocated (GiB)": 36.41, "ppl": 1.14748, "step": 304, "tokens/total": 10757104, "tokens/train_per_sec_per_gpu": 98.43, "tokens/trainable": 474500} +{"epoch": 1.19140625, "grad_norm": 0.7801001667976379, "learning_rate": 4.478451305763618e-05, "loss": 0.12744909524917603, "memory/device_reserved (GiB)": 42.94, "memory/max_active (GiB)": 37.57, "memory/max_allocated (GiB)": 37.57, "ppl": 1.13593, "step": 305, "tokens/total": 10791328, "tokens/train_per_sec_per_gpu": 98.26, "tokens/trainable": 476092} +{"epoch": 1.1953125, "grad_norm": 0.786812961101532, "learning_rate": 4.450201551660454e-05, "loss": 0.13525861501693726, "memory/device_reserved (GiB)": 42.94, "memory/max_active (GiB)": 36.51, "memory/max_allocated (GiB)": 36.51, "ppl": 1.14483, "step": 306, "tokens/total": 10822752, "tokens/train_per_sec_per_gpu": 90.84, "tokens/trainable": 477561} +{"epoch": 1.19921875, "grad_norm": 1.4457125663757324, "learning_rate": 4.4219954839558276e-05, "loss": 0.13631261885166168, "memory/device_reserved (GiB)": 42.94, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.14604, "step": 307, "tokens/total": 10857824, "tokens/train_per_sec_per_gpu": 84.05, "tokens/trainable": 479121} +{"epoch": 1.203125, "grad_norm": 0.9269077181816101, "learning_rate": 4.393834276419352e-05, "loss": 0.13508298993110657, "memory/device_reserved (GiB)": 42.94, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.14463, "step": 308, "tokens/total": 10890144, "tokens/train_per_sec_per_gpu": 104.1, "tokens/trainable": 480741} +{"epoch": 1.20703125, "grad_norm": 0.5453782677650452, "learning_rate": 4.36571910095382e-05, "loss": 0.12531235814094543, "memory/device_reserved (GiB)": 42.94, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.1335, "step": 309, "tokens/total": 10929008, "tokens/train_per_sec_per_gpu": 80.87, "tokens/trainable": 482250} +{"epoch": 1.2109375, "grad_norm": 0.6723716855049133, "learning_rate": 4.337651127546448e-05, "loss": 0.12862981855869293, "memory/device_reserved (GiB)": 42.94, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.13727, "step": 310, "tokens/total": 10965072, "tokens/train_per_sec_per_gpu": 87.78, "tokens/trainable": 483816} +{"epoch": 1.21484375, "grad_norm": 0.7423781156539917, "learning_rate": 4.3096315242201736e-05, "loss": 0.12565864622592926, "memory/device_reserved (GiB)": 42.94, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.1339, "step": 311, "tokens/total": 10998224, "tokens/train_per_sec_per_gpu": 87.15, "tokens/trainable": 485315} +{"epoch": 1.21875, "grad_norm": 0.8496221899986267, "learning_rate": 4.2816614569850635e-05, "loss": 0.11807925999164581, "memory/device_reserved (GiB)": 42.94, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.12533, "step": 312, "tokens/total": 11034576, "tokens/train_per_sec_per_gpu": 83.13, "tokens/trainable": 486792} +{"epoch": 1.22265625, "grad_norm": 1.6581566333770752, "learning_rate": 4.2537420897897864e-05, "loss": 0.1262834519147873, "memory/device_reserved (GiB)": 42.94, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.1346, "step": 313, "tokens/total": 11070608, "tokens/train_per_sec_per_gpu": 83.33, "tokens/trainable": 488355} +{"epoch": 1.2265625, "grad_norm": 0.774678647518158, "learning_rate": 4.225874584473174e-05, "loss": 0.1289665848016739, "memory/device_reserved (GiB)": 42.94, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.13765, "step": 314, "tokens/total": 11103632, "tokens/train_per_sec_per_gpu": 89.39, "tokens/trainable": 489801} +{"epoch": 1.23046875, "grad_norm": 0.44199636578559875, "learning_rate": 4.19806010071587e-05, "loss": 0.11092250049114227, "memory/device_reserved (GiB)": 42.94, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.11731, "step": 315, "tokens/total": 11142208, "tokens/train_per_sec_per_gpu": 83.33, "tokens/trainable": 491421} +{"epoch": 1.234375, "grad_norm": 0.9210984706878662, "learning_rate": 4.170299795992081e-05, "loss": 0.12317254394292831, "memory/device_reserved (GiB)": 42.94, "memory/max_active (GiB)": 36.95, "memory/max_allocated (GiB)": 36.95, "ppl": 1.13108, "step": 316, "tokens/total": 11176064, "tokens/train_per_sec_per_gpu": 88.99, "tokens/trainable": 493000} +{"epoch": 1.23828125, "grad_norm": 1.2911031246185303, "learning_rate": 4.142594825521398e-05, "loss": 0.12288416922092438, "memory/device_reserved (GiB)": 42.94, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.13075, "step": 317, "tokens/total": 11215728, "tokens/train_per_sec_per_gpu": 75.35, "tokens/trainable": 494588} +{"epoch": 1.2421875, "grad_norm": 0.8000848293304443, "learning_rate": 4.114946342220728e-05, "loss": 0.11556100845336914, "memory/device_reserved (GiB)": 42.94, "memory/max_active (GiB)": 39.19, "memory/max_allocated (GiB)": 39.19, "ppl": 1.1225, "step": 318, "tokens/total": 11252800, "tokens/train_per_sec_per_gpu": 80.09, "tokens/trainable": 496152} +{"epoch": 1.24609375, "grad_norm": 0.5681324005126953, "learning_rate": 4.087355496656321e-05, "loss": 0.11001113057136536, "memory/device_reserved (GiB)": 42.94, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.11629, "step": 319, "tokens/total": 11287008, "tokens/train_per_sec_per_gpu": 101.25, "tokens/trainable": 497788} +{"epoch": 1.25, "grad_norm": 0.7500831484794617, "learning_rate": 4.05982343699588e-05, "loss": 0.12502236664295197, "memory/device_reserved (GiB)": 42.94, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.13317, "step": 320, "tokens/total": 11326656, "tokens/train_per_sec_per_gpu": 81.09, "tokens/trainable": 499447} +{"epoch": 1.25390625, "grad_norm": 0.5887723565101624, "learning_rate": 4.0323513089607876e-05, "loss": 0.12306277453899384, "memory/device_reserved (GiB)": 42.94, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.13096, "step": 321, "tokens/total": 11362144, "tokens/train_per_sec_per_gpu": 80.21, "tokens/trainable": 501020} +{"epoch": 1.2578125, "grad_norm": 1.3126739263534546, "learning_rate": 4.004940255778431e-05, "loss": 0.13007202744483948, "memory/device_reserved (GiB)": 43.12, "memory/max_active (GiB)": 39.74, "memory/max_allocated (GiB)": 39.74, "ppl": 1.13891, "step": 322, "tokens/total": 11401376, "tokens/train_per_sec_per_gpu": 81.63, "tokens/trainable": 502609} +{"epoch": 1.26171875, "grad_norm": 0.6362340450286865, "learning_rate": 3.977591418134619e-05, "loss": 0.12879225611686707, "memory/device_reserved (GiB)": 43.12, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.13745, "step": 323, "tokens/total": 11436928, "tokens/train_per_sec_per_gpu": 84.21, "tokens/trainable": 504135} +{"epoch": 1.265625, "grad_norm": 1.612313151359558, "learning_rate": 3.95030593412612e-05, "loss": 0.13661080598831177, "memory/device_reserved (GiB)": 43.12, "memory/max_active (GiB)": 39.66, "memory/max_allocated (GiB)": 39.66, "ppl": 1.14638, "step": 324, "tokens/total": 11473200, "tokens/train_per_sec_per_gpu": 83.92, "tokens/trainable": 505620} +{"epoch": 1.26953125, "grad_norm": 0.4961627125740051, "learning_rate": 3.923084939213296e-05, "loss": 0.11357726156711578, "memory/device_reserved (GiB)": 43.12, "memory/max_active (GiB)": 36.05, "memory/max_allocated (GiB)": 36.05, "ppl": 1.12028, "step": 325, "tokens/total": 11504592, "tokens/train_per_sec_per_gpu": 96.64, "tokens/trainable": 507140} +{"epoch": 1.2734375, "grad_norm": 0.8378382921218872, "learning_rate": 3.895929566172861e-05, "loss": 0.12300044298171997, "memory/device_reserved (GiB)": 43.12, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.13088, "step": 326, "tokens/total": 11540880, "tokens/train_per_sec_per_gpu": 85.52, "tokens/trainable": 508710} +{"epoch": 1.27734375, "grad_norm": 0.4057982265949249, "learning_rate": 3.868840945050728e-05, "loss": 0.10170241445302963, "memory/device_reserved (GiB)": 43.12, "memory/max_active (GiB)": 39.35, "memory/max_allocated (GiB)": 39.35, "ppl": 1.10705, "step": 327, "tokens/total": 11580112, "tokens/train_per_sec_per_gpu": 86.22, "tokens/trainable": 510391} +{"epoch": 1.28125, "grad_norm": 0.6078394651412964, "learning_rate": 3.841820203114995e-05, "loss": 0.1122528687119484, "memory/device_reserved (GiB)": 43.12, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 1.1188, "step": 328, "tokens/total": 11612432, "tokens/train_per_sec_per_gpu": 98.86, "tokens/trainable": 512032} +{"epoch": 1.28515625, "grad_norm": 0.6957389116287231, "learning_rate": 3.814868464809027e-05, "loss": 0.12403795123100281, "memory/device_reserved (GiB)": 43.12, "memory/max_active (GiB)": 36.92, "memory/max_allocated (GiB)": 36.92, "ppl": 1.13206, "step": 329, "tokens/total": 11644864, "tokens/train_per_sec_per_gpu": 90.84, "tokens/trainable": 513520} +{"epoch": 1.2890625, "grad_norm": 0.8719311356544495, "learning_rate": 3.787986851704667e-05, "loss": 0.11916600167751312, "memory/device_reserved (GiB)": 43.12, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.12656, "step": 330, "tokens/total": 11679136, "tokens/train_per_sec_per_gpu": 91.86, "tokens/trainable": 515067} +{"epoch": 1.29296875, "grad_norm": 0.5615230202674866, "learning_rate": 3.7611764824555654e-05, "loss": 0.11547627300024033, "memory/device_reserved (GiB)": 43.12, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.12241, "step": 331, "tokens/total": 11711168, "tokens/train_per_sec_per_gpu": 102.77, "tokens/trainable": 516671} +{"epoch": 1.296875, "grad_norm": 0.54587721824646, "learning_rate": 3.734438472750619e-05, "loss": 0.1063561737537384, "memory/device_reserved (GiB)": 43.12, "memory/max_active (GiB)": 39.66, "memory/max_allocated (GiB)": 39.66, "ppl": 1.11222, "step": 332, "tokens/total": 11751568, "tokens/train_per_sec_per_gpu": 82.25, "tokens/trainable": 518320} +{"epoch": 1.30078125, "grad_norm": 1.0453065633773804, "learning_rate": 3.707773935267552e-05, "loss": 0.11577177792787552, "memory/device_reserved (GiB)": 43.12, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.12274, "step": 333, "tokens/total": 11789264, "tokens/train_per_sec_per_gpu": 90.29, "tokens/trainable": 520002} +{"epoch": 1.3046875, "grad_norm": 1.4705168008804321, "learning_rate": 3.68118397962661e-05, "loss": 0.12634140253067017, "memory/device_reserved (GiB)": 43.12, "memory/max_active (GiB)": 38.66, "memory/max_allocated (GiB)": 38.66, "ppl": 1.13467, "step": 334, "tokens/total": 11824688, "tokens/train_per_sec_per_gpu": 88.5, "tokens/trainable": 521494} +{"epoch": 1.30859375, "grad_norm": 0.6432345509529114, "learning_rate": 3.654669712344384e-05, "loss": 0.11043687909841537, "memory/device_reserved (GiB)": 43.12, "memory/max_active (GiB)": 36.79, "memory/max_allocated (GiB)": 36.79, "ppl": 1.11677, "step": 335, "tokens/total": 11856288, "tokens/train_per_sec_per_gpu": 112.37, "tokens/trainable": 523096} +{"epoch": 1.3125, "grad_norm": 0.7215744256973267, "learning_rate": 3.628232236787763e-05, "loss": 0.10336160659790039, "memory/device_reserved (GiB)": 43.12, "memory/max_active (GiB)": 38.24, "memory/max_allocated (GiB)": 38.24, "ppl": 1.10889, "step": 336, "tokens/total": 11892784, "tokens/train_per_sec_per_gpu": 87.44, "tokens/trainable": 524760} +{"epoch": 1.31640625, "grad_norm": 0.511624813079834, "learning_rate": 3.6018726531280144e-05, "loss": 0.11687718331813812, "memory/device_reserved (GiB)": 43.12, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.12398, "step": 337, "tokens/total": 11930704, "tokens/train_per_sec_per_gpu": 83.07, "tokens/trainable": 526316} +{"epoch": 1.3203125, "grad_norm": 1.058436393737793, "learning_rate": 3.575592058295017e-05, "loss": 0.13526801764965057, "memory/device_reserved (GiB)": 43.12, "memory/max_active (GiB)": 37.2, "memory/max_allocated (GiB)": 37.2, "ppl": 1.14484, "step": 338, "tokens/total": 11965296, "tokens/train_per_sec_per_gpu": 91.07, "tokens/trainable": 527919} +{"epoch": 1.32421875, "grad_norm": 0.8331614136695862, "learning_rate": 3.549391545931585e-05, "loss": 0.13299338519573212, "memory/device_reserved (GiB)": 43.12, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.14224, "step": 339, "tokens/total": 12000320, "tokens/train_per_sec_per_gpu": 87.26, "tokens/trainable": 529475} +{"epoch": 1.328125, "grad_norm": 1.0640687942504883, "learning_rate": 3.5232722063479914e-05, "loss": 0.13329432904720306, "memory/device_reserved (GiB)": 43.12, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.14259, "step": 340, "tokens/total": 12034960, "tokens/train_per_sec_per_gpu": 87.75, "tokens/trainable": 531045} +{"epoch": 1.33203125, "grad_norm": 0.8296259641647339, "learning_rate": 3.49723512647657e-05, "loss": 0.11793814599514008, "memory/device_reserved (GiB)": 43.12, "memory/max_active (GiB)": 39.39, "memory/max_allocated (GiB)": 39.39, "ppl": 1.12517, "step": 341, "tokens/total": 12074832, "tokens/train_per_sec_per_gpu": 78.12, "tokens/trainable": 532688} +{"epoch": 1.3359375, "grad_norm": 0.8296814560890198, "learning_rate": 3.471281389826491e-05, "loss": 0.12998700141906738, "memory/device_reserved (GiB)": 43.12, "memory/max_active (GiB)": 36.63, "memory/max_allocated (GiB)": 36.63, "ppl": 1.13881, "step": 342, "tokens/total": 12106304, "tokens/train_per_sec_per_gpu": 90.21, "tokens/trainable": 534139} +{"epoch": 1.33984375, "grad_norm": 0.5657304525375366, "learning_rate": 3.4454120764386764e-05, "loss": 0.11357824504375458, "memory/device_reserved (GiB)": 43.12, "memory/max_active (GiB)": 38.93, "memory/max_allocated (GiB)": 38.93, "ppl": 1.12028, "step": 343, "tokens/total": 12145152, "tokens/train_per_sec_per_gpu": 76.38, "tokens/trainable": 535736} +{"epoch": 1.34375, "grad_norm": 0.558931291103363, "learning_rate": 3.4196282628408526e-05, "loss": 0.11325210332870483, "memory/device_reserved (GiB)": 43.12, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.11991, "step": 344, "tokens/total": 12180048, "tokens/train_per_sec_per_gpu": 81.95, "tokens/trainable": 537275} +{"epoch": 1.34765625, "grad_norm": 3.014766216278076, "learning_rate": 3.3939310220027456e-05, "loss": 0.12418380379676819, "memory/device_reserved (GiB)": 43.12, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.13222, "step": 345, "tokens/total": 12214736, "tokens/train_per_sec_per_gpu": 91.7, "tokens/trainable": 538909} +{"epoch": 1.3515625, "grad_norm": 0.47771888971328735, "learning_rate": 3.3683214232914404e-05, "loss": 0.10931701958179474, "memory/device_reserved (GiB)": 43.12, "memory/max_active (GiB)": 38.93, "memory/max_allocated (GiB)": 38.93, "ppl": 1.11552, "step": 346, "tokens/total": 12250928, "tokens/train_per_sec_per_gpu": 87.43, "tokens/trainable": 540505} +{"epoch": 1.35546875, "grad_norm": 0.5381442904472351, "learning_rate": 3.342800532426873e-05, "loss": 0.11486902832984924, "memory/device_reserved (GiB)": 43.12, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.12173, "step": 347, "tokens/total": 12287456, "tokens/train_per_sec_per_gpu": 93.22, "tokens/trainable": 542126} +{"epoch": 1.359375, "grad_norm": 0.5958352088928223, "learning_rate": 3.317369411437484e-05, "loss": 0.11685895919799805, "memory/device_reserved (GiB)": 43.12, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 1.12396, "step": 348, "tokens/total": 12323776, "tokens/train_per_sec_per_gpu": 87.32, "tokens/trainable": 543713} +{"epoch": 1.36328125, "grad_norm": 0.5183544754981995, "learning_rate": 3.292029118616024e-05, "loss": 0.11528262495994568, "memory/device_reserved (GiB)": 43.12, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.12219, "step": 349, "tokens/total": 12357200, "tokens/train_per_sec_per_gpu": 88.36, "tokens/trainable": 545241} +{"epoch": 1.3671875, "grad_norm": 1.138171911239624, "learning_rate": 3.266780708475511e-05, "loss": 0.12489315867424011, "memory/device_reserved (GiB)": 43.12, "memory/max_active (GiB)": 39.28, "memory/max_allocated (GiB)": 39.28, "ppl": 1.13303, "step": 350, "tokens/total": 12393072, "tokens/train_per_sec_per_gpu": 80.03, "tokens/trainable": 546785} +{"epoch": 1.37109375, "grad_norm": 0.9734210968017578, "learning_rate": 3.241625231705354e-05, "loss": 0.11523417383432388, "memory/device_reserved (GiB)": 43.12, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.12214, "step": 351, "tokens/total": 12429504, "tokens/train_per_sec_per_gpu": 84.16, "tokens/trainable": 548339} +{"epoch": 1.375, "grad_norm": 0.5055480003356934, "learning_rate": 3.216563735127618e-05, "loss": 0.1196289137005806, "memory/device_reserved (GiB)": 43.12, "memory/max_active (GiB)": 39.32, "memory/max_allocated (GiB)": 39.32, "ppl": 1.12708, "step": 352, "tokens/total": 12465008, "tokens/train_per_sec_per_gpu": 89.0, "tokens/trainable": 549950} +{"epoch": 1.37890625, "grad_norm": 0.624749481678009, "learning_rate": 3.191597261653475e-05, "loss": 0.11965254694223404, "memory/device_reserved (GiB)": 43.12, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.12711, "step": 353, "tokens/total": 12502256, "tokens/train_per_sec_per_gpu": 81.06, "tokens/trainable": 551550} +{"epoch": 1.3828125, "grad_norm": 2.079726457595825, "learning_rate": 3.166726850239794e-05, "loss": 0.12005801498889923, "memory/device_reserved (GiB)": 43.12, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.12756, "step": 354, "tokens/total": 12536144, "tokens/train_per_sec_per_gpu": 85.26, "tokens/trainable": 553044} +{"epoch": 1.38671875, "grad_norm": 0.7413411140441895, "learning_rate": 3.141953535845912e-05, "loss": 0.13025051355361938, "memory/device_reserved (GiB)": 43.12, "memory/max_active (GiB)": 36.39, "memory/max_allocated (GiB)": 36.39, "ppl": 1.13911, "step": 355, "tokens/total": 12567472, "tokens/train_per_sec_per_gpu": 98.86, "tokens/trainable": 554551} +{"epoch": 1.390625, "grad_norm": 0.5564784407615662, "learning_rate": 3.11727834939056e-05, "loss": 0.11933283507823944, "memory/device_reserved (GiB)": 43.12, "memory/max_active (GiB)": 36.44, "memory/max_allocated (GiB)": 36.44, "ppl": 1.12674, "step": 356, "tokens/total": 12599712, "tokens/train_per_sec_per_gpu": 92.83, "tokens/trainable": 556116} +{"epoch": 1.39453125, "grad_norm": 1.2688928842544556, "learning_rate": 3.092702317708967e-05, "loss": 0.1254902482032776, "memory/device_reserved (GiB)": 43.12, "memory/max_active (GiB)": 37.18, "memory/max_allocated (GiB)": 37.18, "ppl": 1.1337, "step": 357, "tokens/total": 12633776, "tokens/train_per_sec_per_gpu": 90.15, "tokens/trainable": 557655} +{"epoch": 1.3984375, "grad_norm": 1.0877188444137573, "learning_rate": 3.0682264635101276e-05, "loss": 0.11344513297080994, "memory/device_reserved (GiB)": 43.12, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.12013, "step": 358, "tokens/total": 12666800, "tokens/train_per_sec_per_gpu": 89.71, "tokens/trainable": 559167} +{"epoch": 1.40234375, "grad_norm": 0.8097068667411804, "learning_rate": 3.0438518053342407e-05, "loss": 0.10956442356109619, "memory/device_reserved (GiB)": 43.12, "memory/max_active (GiB)": 39.32, "memory/max_allocated (GiB)": 39.32, "ppl": 1.11579, "step": 359, "tokens/total": 12706480, "tokens/train_per_sec_per_gpu": 83.76, "tokens/trainable": 560856} +{"epoch": 1.40625, "grad_norm": 0.860282838344574, "learning_rate": 3.0195793575103266e-05, "loss": 0.1195136159658432, "memory/device_reserved (GiB)": 43.12, "memory/max_active (GiB)": 39.05, "memory/max_allocated (GiB)": 39.05, "ppl": 1.12695, "step": 360, "tokens/total": 12746048, "tokens/train_per_sec_per_gpu": 76.77, "tokens/trainable": 562443} +{"epoch": 1.41015625, "grad_norm": 0.5349950790405273, "learning_rate": 2.9954101301140146e-05, "loss": 0.11895452439785004, "memory/device_reserved (GiB)": 43.12, "memory/max_active (GiB)": 37.83, "memory/max_allocated (GiB)": 37.83, "ppl": 1.12632, "step": 361, "tokens/total": 12782064, "tokens/train_per_sec_per_gpu": 81.24, "tokens/trainable": 563952} +{"epoch": 1.4140625, "grad_norm": 0.49843621253967285, "learning_rate": 2.9713451289255123e-05, "loss": 0.11151982843875885, "memory/device_reserved (GiB)": 43.12, "memory/max_active (GiB)": 37.12, "memory/max_allocated (GiB)": 37.12, "ppl": 1.11798, "step": 362, "tokens/total": 12812048, "tokens/train_per_sec_per_gpu": 96.0, "tokens/trainable": 565544} +{"epoch": 1.41796875, "grad_norm": 0.9914259314537048, "learning_rate": 2.9473853553877484e-05, "loss": 0.11544537544250488, "memory/device_reserved (GiB)": 43.12, "memory/max_active (GiB)": 38.53, "memory/max_allocated (GiB)": 38.53, "ppl": 1.12237, "step": 363, "tokens/total": 12846400, "tokens/train_per_sec_per_gpu": 87.63, "tokens/trainable": 567045} +{"epoch": 1.421875, "grad_norm": 0.7432466149330139, "learning_rate": 2.9235318065647e-05, "loss": 0.12476923316717148, "memory/device_reserved (GiB)": 43.12, "memory/max_active (GiB)": 39.89, "memory/max_allocated (GiB)": 39.89, "ppl": 1.13289, "step": 364, "tokens/total": 12886912, "tokens/train_per_sec_per_gpu": 72.2, "tokens/trainable": 568542} +{"epoch": 1.42578125, "grad_norm": 0.7144530415534973, "learning_rate": 2.8997854750998964e-05, "loss": 0.11694943904876709, "memory/device_reserved (GiB)": 43.35, "memory/max_active (GiB)": 39.57, "memory/max_allocated (GiB)": 39.57, "ppl": 1.12406, "step": 365, "tokens/total": 12927008, "tokens/train_per_sec_per_gpu": 82.72, "tokens/trainable": 570159} +{"epoch": 1.4296875, "grad_norm": 0.7736572623252869, "learning_rate": 2.8761473491751258e-05, "loss": 0.1293759047985077, "memory/device_reserved (GiB)": 43.35, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.13812, "step": 366, "tokens/total": 12960384, "tokens/train_per_sec_per_gpu": 93.21, "tokens/trainable": 571685} +{"epoch": 1.43359375, "grad_norm": 0.8851078152656555, "learning_rate": 2.8526184124692883e-05, "loss": 0.10143433511257172, "memory/device_reserved (GiB)": 43.35, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.10676, "step": 367, "tokens/total": 12997792, "tokens/train_per_sec_per_gpu": 81.68, "tokens/trainable": 573330} +{"epoch": 1.4375, "grad_norm": 0.7305833697319031, "learning_rate": 2.829199644117484e-05, "loss": 0.11584565788507462, "memory/device_reserved (GiB)": 43.35, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.12282, "step": 368, "tokens/total": 13031888, "tokens/train_per_sec_per_gpu": 94.49, "tokens/trainable": 574824} +{"epoch": 1.44140625, "grad_norm": 0.8667214512825012, "learning_rate": 2.8058920186702553e-05, "loss": 0.11965532600879669, "memory/device_reserved (GiB)": 43.35, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.12711, "step": 369, "tokens/total": 13070384, "tokens/train_per_sec_per_gpu": 82.59, "tokens/trainable": 576409} +{"epoch": 1.4453125, "grad_norm": 0.8742167949676514, "learning_rate": 2.782696506053033e-05, "loss": 0.12485050410032272, "memory/device_reserved (GiB)": 43.35, "memory/max_active (GiB)": 39.28, "memory/max_allocated (GiB)": 39.28, "ppl": 1.13298, "step": 370, "tokens/total": 13108896, "tokens/train_per_sec_per_gpu": 77.39, "tokens/trainable": 577888} +{"epoch": 1.44921875, "grad_norm": 0.8595766425132751, "learning_rate": 2.7596140715257824e-05, "loss": 0.12068489193916321, "memory/device_reserved (GiB)": 43.35, "memory/max_active (GiB)": 38.97, "memory/max_allocated (GiB)": 38.97, "ppl": 1.12827, "step": 371, "tokens/total": 13144416, "tokens/train_per_sec_per_gpu": 86.15, "tokens/trainable": 579429} +{"epoch": 1.453125, "grad_norm": 0.6391112208366394, "learning_rate": 2.7366456756428184e-05, "loss": 0.11256781965494156, "memory/device_reserved (GiB)": 43.35, "memory/max_active (GiB)": 37.58, "memory/max_allocated (GiB)": 37.58, "ppl": 1.11915, "step": 372, "tokens/total": 13179360, "tokens/train_per_sec_per_gpu": 84.21, "tokens/trainable": 580971} +{"epoch": 1.45703125, "grad_norm": 0.5798320770263672, "learning_rate": 2.7137922742128486e-05, "loss": 0.11311674118041992, "memory/device_reserved (GiB)": 43.35, "memory/max_active (GiB)": 38.37, "memory/max_allocated (GiB)": 38.37, "ppl": 1.11976, "step": 373, "tokens/total": 13214752, "tokens/train_per_sec_per_gpu": 87.67, "tokens/trainable": 582594} +{"epoch": 1.4609375, "grad_norm": 0.6119470000267029, "learning_rate": 2.691054818259188e-05, "loss": 0.10441339015960693, "memory/device_reserved (GiB)": 43.35, "memory/max_active (GiB)": 37.52, "memory/max_allocated (GiB)": 37.52, "ppl": 1.11006, "step": 374, "tokens/total": 13249728, "tokens/train_per_sec_per_gpu": 91.54, "tokens/trainable": 584286} +{"epoch": 1.46484375, "grad_norm": 0.5852486491203308, "learning_rate": 2.6684342539801933e-05, "loss": 0.12482607364654541, "memory/device_reserved (GiB)": 43.35, "memory/max_active (GiB)": 36.57, "memory/max_allocated (GiB)": 36.57, "ppl": 1.13295, "step": 375, "tokens/total": 13281360, "tokens/train_per_sec_per_gpu": 98.79, "tokens/trainable": 585810} +{"epoch": 1.46875, "grad_norm": 0.5333801507949829, "learning_rate": 2.645931522709877e-05, "loss": 0.11535274237394333, "memory/device_reserved (GiB)": 43.35, "memory/max_active (GiB)": 39.67, "memory/max_allocated (GiB)": 39.67, "ppl": 1.12227, "step": 376, "tokens/total": 13317520, "tokens/train_per_sec_per_gpu": 96.91, "tokens/trainable": 587470} +{"epoch": 1.47265625, "grad_norm": 0.5957995057106018, "learning_rate": 2.6235475608787365e-05, "loss": 0.11680558323860168, "memory/device_reserved (GiB)": 43.35, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.1239, "step": 377, "tokens/total": 13353712, "tokens/train_per_sec_per_gpu": 81.1, "tokens/trainable": 589012} +{"epoch": 1.4765625, "grad_norm": 0.6542219519615173, "learning_rate": 2.6012832999747916e-05, "loss": 0.12337668985128403, "memory/device_reserved (GiB)": 43.35, "memory/max_active (GiB)": 36.59, "memory/max_allocated (GiB)": 36.59, "ppl": 1.13131, "step": 378, "tokens/total": 13385296, "tokens/train_per_sec_per_gpu": 97.79, "tokens/trainable": 590533} +{"epoch": 1.48046875, "grad_norm": 0.5255330801010132, "learning_rate": 2.579139666504821e-05, "loss": 0.12079722434282303, "memory/device_reserved (GiB)": 43.35, "memory/max_active (GiB)": 39.58, "memory/max_allocated (GiB)": 39.58, "ppl": 1.1284, "step": 379, "tokens/total": 13425792, "tokens/train_per_sec_per_gpu": 77.24, "tokens/trainable": 592082} +{"epoch": 1.484375, "grad_norm": 0.44805708527565, "learning_rate": 2.557117581955798e-05, "loss": 0.11409690976142883, "memory/device_reserved (GiB)": 43.35, "memory/max_active (GiB)": 37.39, "memory/max_allocated (GiB)": 37.39, "ppl": 1.12086, "step": 380, "tokens/total": 13460320, "tokens/train_per_sec_per_gpu": 86.95, "tokens/trainable": 593693} +{"epoch": 1.48828125, "grad_norm": 0.5268592238426208, "learning_rate": 2.5352179627565532e-05, "loss": 0.11561322212219238, "memory/device_reserved (GiB)": 43.35, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.12256, "step": 381, "tokens/total": 13495824, "tokens/train_per_sec_per_gpu": 86.37, "tokens/trainable": 595248} +{"epoch": 1.4921875, "grad_norm": 0.6096928715705872, "learning_rate": 2.5134417202396277e-05, "loss": 0.12011256814002991, "memory/device_reserved (GiB)": 43.35, "memory/max_active (GiB)": 38.69, "memory/max_allocated (GiB)": 38.69, "ppl": 1.12762, "step": 382, "tokens/total": 13529824, "tokens/train_per_sec_per_gpu": 85.55, "tokens/trainable": 596768} +{"epoch": 1.49609375, "grad_norm": 0.5065365433692932, "learning_rate": 2.491789760603361e-05, "loss": 0.11974900215864182, "memory/device_reserved (GiB)": 43.35, "memory/max_active (GiB)": 39.35, "memory/max_allocated (GiB)": 39.35, "ppl": 1.12721, "step": 383, "tokens/total": 13566688, "tokens/train_per_sec_per_gpu": 82.32, "tokens/trainable": 598367} +{"epoch": 1.5, "grad_norm": 0.8878273367881775, "learning_rate": 2.4702629848741764e-05, "loss": 0.12491701543331146, "memory/device_reserved (GiB)": 43.35, "memory/max_active (GiB)": 37.84, "memory/max_allocated (GiB)": 37.84, "ppl": 1.13305, "step": 384, "tokens/total": 13601600, "tokens/train_per_sec_per_gpu": 89.97, "tokens/trainable": 599879} +{"epoch": 1.50390625, "grad_norm": 0.624676525592804, "learning_rate": 2.4488622888690785e-05, "loss": 0.11105373501777649, "memory/device_reserved (GiB)": 43.35, "memory/max_active (GiB)": 37.17, "memory/max_allocated (GiB)": 37.17, "ppl": 1.11745, "step": 385, "tokens/total": 13635728, "tokens/train_per_sec_per_gpu": 82.91, "tokens/trainable": 601419} +{"epoch": 1.5078125, "grad_norm": 0.5493330955505371, "learning_rate": 2.427588563158384e-05, "loss": 0.10663966834545135, "memory/device_reserved (GiB)": 43.35, "memory/max_active (GiB)": 39.19, "memory/max_allocated (GiB)": 39.19, "ppl": 1.11253, "step": 386, "tokens/total": 13670480, "tokens/train_per_sec_per_gpu": 90.97, "tokens/trainable": 603008} +{"epoch": 1.51171875, "grad_norm": 0.49862295389175415, "learning_rate": 2.406442693028651e-05, "loss": 0.11656484007835388, "memory/device_reserved (GiB)": 43.35, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.12363, "step": 387, "tokens/total": 13703152, "tokens/train_per_sec_per_gpu": 87.13, "tokens/trainable": 604451} +{"epoch": 1.515625, "grad_norm": 0.756568431854248, "learning_rate": 2.3854255584458547e-05, "loss": 0.11552466452121735, "memory/device_reserved (GiB)": 43.35, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.12246, "step": 388, "tokens/total": 13742544, "tokens/train_per_sec_per_gpu": 72.96, "tokens/trainable": 605955} +{"epoch": 1.51953125, "grad_norm": 0.6945745348930359, "learning_rate": 2.3645380340187508e-05, "loss": 0.11666169762611389, "memory/device_reserved (GiB)": 43.35, "memory/max_active (GiB)": 39.39, "memory/max_allocated (GiB)": 39.39, "ppl": 1.12374, "step": 389, "tokens/total": 13778752, "tokens/train_per_sec_per_gpu": 86.22, "tokens/trainable": 607533} +{"epoch": 1.5234375, "grad_norm": 0.6631448268890381, "learning_rate": 2.3437809889624914e-05, "loss": 0.1387963742017746, "memory/device_reserved (GiB)": 43.35, "memory/max_active (GiB)": 37.73, "memory/max_allocated (GiB)": 37.73, "ppl": 1.14889, "step": 390, "tokens/total": 13811984, "tokens/train_per_sec_per_gpu": 92.46, "tokens/trainable": 609006} +{"epoch": 1.52734375, "grad_norm": 0.552549421787262, "learning_rate": 2.3231552870624487e-05, "loss": 0.10629608482122421, "memory/device_reserved (GiB)": 43.35, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.11215, "step": 391, "tokens/total": 13843296, "tokens/train_per_sec_per_gpu": 100.8, "tokens/trainable": 610603} +{"epoch": 1.53125, "grad_norm": 0.7150276899337769, "learning_rate": 2.3026617866382657e-05, "loss": 0.09776764363050461, "memory/device_reserved (GiB)": 43.35, "memory/max_active (GiB)": 36.88, "memory/max_allocated (GiB)": 36.88, "ppl": 1.10271, "step": 392, "tokens/total": 13876704, "tokens/train_per_sec_per_gpu": 97.31, "tokens/trainable": 612265} +{"epoch": 1.53515625, "grad_norm": 0.8616323471069336, "learning_rate": 2.2823013405081507e-05, "loss": 0.10816208273172379, "memory/device_reserved (GiB)": 43.35, "memory/max_active (GiB)": 39.35, "memory/max_allocated (GiB)": 39.35, "ppl": 1.11423, "step": 393, "tokens/total": 13915248, "tokens/train_per_sec_per_gpu": 81.44, "tokens/trainable": 613964} +{"epoch": 1.5390625, "grad_norm": 1.2409982681274414, "learning_rate": 2.2620747959533722e-05, "loss": 0.11493928730487823, "memory/device_reserved (GiB)": 43.35, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.12181, "step": 394, "tokens/total": 13949152, "tokens/train_per_sec_per_gpu": 97.97, "tokens/trainable": 615614} +{"epoch": 1.54296875, "grad_norm": 0.44559240341186523, "learning_rate": 2.2419829946830123e-05, "loss": 0.10838060826063156, "memory/device_reserved (GiB)": 43.35, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.11447, "step": 395, "tokens/total": 13985648, "tokens/train_per_sec_per_gpu": 88.41, "tokens/trainable": 617204} +{"epoch": 1.546875, "grad_norm": 0.49361974000930786, "learning_rate": 2.2220267727989325e-05, "loss": 0.1136651262640953, "memory/device_reserved (GiB)": 43.35, "memory/max_active (GiB)": 37.96, "memory/max_allocated (GiB)": 37.96, "ppl": 1.12038, "step": 396, "tokens/total": 14020704, "tokens/train_per_sec_per_gpu": 83.06, "tokens/trainable": 618721} +{"epoch": 1.55078125, "grad_norm": 0.5859568119049072, "learning_rate": 2.202206960760984e-05, "loss": 0.11033469438552856, "memory/device_reserved (GiB)": 43.35, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.11665, "step": 397, "tokens/total": 14059312, "tokens/train_per_sec_per_gpu": 78.61, "tokens/trainable": 620355} +{"epoch": 1.5546875, "grad_norm": 0.4801059365272522, "learning_rate": 2.182524383352446e-05, "loss": 0.11069785058498383, "memory/device_reserved (GiB)": 43.35, "memory/max_active (GiB)": 38.22, "memory/max_allocated (GiB)": 38.22, "ppl": 1.11706, "step": 398, "tokens/total": 14096096, "tokens/train_per_sec_per_gpu": 85.22, "tokens/trainable": 621903} +{"epoch": 1.55859375, "grad_norm": 0.46536341309547424, "learning_rate": 2.1629798596457056e-05, "loss": 0.11042523384094238, "memory/device_reserved (GiB)": 43.35, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.11675, "step": 399, "tokens/total": 14128720, "tokens/train_per_sec_per_gpu": 102.41, "tokens/trainable": 623508} +{"epoch": 1.5625, "grad_norm": 0.4637174606323242, "learning_rate": 2.1435742029681725e-05, "loss": 0.11593063175678253, "memory/device_reserved (GiB)": 43.35, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.12292, "step": 400, "tokens/total": 14161312, "tokens/train_per_sec_per_gpu": 96.74, "tokens/trainable": 624995} +{"epoch": 1.56640625, "grad_norm": 0.44622233510017395, "learning_rate": 2.124308220868431e-05, "loss": 0.10746467113494873, "memory/device_reserved (GiB)": 43.35, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.11345, "step": 401, "tokens/total": 14197664, "tokens/train_per_sec_per_gpu": 82.92, "tokens/trainable": 626593} +{"epoch": 1.5703125, "grad_norm": 1.5399538278579712, "learning_rate": 2.105182715082638e-05, "loss": 0.11502604931592941, "memory/device_reserved (GiB)": 43.35, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.1219, "step": 402, "tokens/total": 14231824, "tokens/train_per_sec_per_gpu": 84.25, "tokens/trainable": 628197} +{"epoch": 1.57421875, "grad_norm": 0.8757160902023315, "learning_rate": 2.0861984815011552e-05, "loss": 0.12254435569047928, "memory/device_reserved (GiB)": 43.35, "memory/max_active (GiB)": 38.17, "memory/max_allocated (GiB)": 38.17, "ppl": 1.13037, "step": 403, "tokens/total": 14267456, "tokens/train_per_sec_per_gpu": 82.26, "tokens/trainable": 629682} +{"epoch": 1.578125, "grad_norm": 2.319478750228882, "learning_rate": 2.0673563101354323e-05, "loss": 0.11481384187936783, "memory/device_reserved (GiB)": 43.35, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.12166, "step": 404, "tokens/total": 14303008, "tokens/train_per_sec_per_gpu": 85.29, "tokens/trainable": 631205} +{"epoch": 1.58203125, "grad_norm": 0.4918518662452698, "learning_rate": 2.0486569850851317e-05, "loss": 0.11785721778869629, "memory/device_reserved (GiB)": 43.35, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.12508, "step": 405, "tokens/total": 14338352, "tokens/train_per_sec_per_gpu": 89.64, "tokens/trainable": 632759} +{"epoch": 1.5859375, "grad_norm": 0.4989170730113983, "learning_rate": 2.0301012845054956e-05, "loss": 0.11970983445644379, "memory/device_reserved (GiB)": 43.35, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.12717, "step": 406, "tokens/total": 14377520, "tokens/train_per_sec_per_gpu": 74.56, "tokens/trainable": 634253} +{"epoch": 1.58984375, "grad_norm": 0.5417273640632629, "learning_rate": 2.011689980574966e-05, "loss": 0.1174154281616211, "memory/device_reserved (GiB)": 43.35, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.12459, "step": 407, "tokens/total": 14414480, "tokens/train_per_sec_per_gpu": 76.37, "tokens/trainable": 635715} +{"epoch": 1.59375, "grad_norm": 0.41985833644866943, "learning_rate": 1.993423839463052e-05, "loss": 0.10894448310136795, "memory/device_reserved (GiB)": 43.35, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.1151, "step": 408, "tokens/total": 14448592, "tokens/train_per_sec_per_gpu": 100.29, "tokens/trainable": 637335} +{"epoch": 1.59765625, "grad_norm": 0.4448608160018921, "learning_rate": 1.975303621298445e-05, "loss": 0.11174237728118896, "memory/device_reserved (GiB)": 43.35, "memory/max_active (GiB)": 39.35, "memory/max_allocated (GiB)": 39.35, "ppl": 1.11822, "step": 409, "tokens/total": 14485824, "tokens/train_per_sec_per_gpu": 84.15, "tokens/trainable": 638864} +{"epoch": 1.6015625, "grad_norm": 0.6871687769889832, "learning_rate": 1.957330080137385e-05, "loss": 0.11634984612464905, "memory/device_reserved (GiB)": 43.35, "memory/max_active (GiB)": 36.45, "memory/max_allocated (GiB)": 36.45, "ppl": 1.12339, "step": 410, "tokens/total": 14517216, "tokens/train_per_sec_per_gpu": 95.19, "tokens/trainable": 640424} +{"epoch": 1.60546875, "grad_norm": 0.9353247880935669, "learning_rate": 1.9395039639322864e-05, "loss": 0.1170063242316246, "memory/device_reserved (GiB)": 43.35, "memory/max_active (GiB)": 39.09, "memory/max_allocated (GiB)": 39.09, "ppl": 1.12413, "step": 411, "tokens/total": 14553872, "tokens/train_per_sec_per_gpu": 85.61, "tokens/trainable": 641974} +{"epoch": 1.609375, "grad_norm": 0.6040135025978088, "learning_rate": 1.9218260145006073e-05, "loss": 0.11200446635484695, "memory/device_reserved (GiB)": 43.35, "memory/max_active (GiB)": 38.0, "memory/max_allocated (GiB)": 38.0, "ppl": 1.11852, "step": 412, "tokens/total": 14590464, "tokens/train_per_sec_per_gpu": 81.47, "tokens/trainable": 643553} +{"epoch": 1.61328125, "grad_norm": 0.582157552242279, "learning_rate": 1.904296967493982e-05, "loss": 0.108284130692482, "memory/device_reserved (GiB)": 43.35, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 1.11436, "step": 413, "tokens/total": 14623680, "tokens/train_per_sec_per_gpu": 102.35, "tokens/trainable": 645158} +{"epoch": 1.6171875, "grad_norm": 0.4041297435760498, "learning_rate": 1.8869175523676064e-05, "loss": 0.09853313118219376, "memory/device_reserved (GiB)": 43.35, "memory/max_active (GiB)": 38.43, "memory/max_allocated (GiB)": 38.43, "ppl": 1.10355, "step": 414, "tokens/total": 14660896, "tokens/train_per_sec_per_gpu": 86.48, "tokens/trainable": 646824} +{"epoch": 1.62109375, "grad_norm": 0.41481688618659973, "learning_rate": 1.869688492349885e-05, "loss": 0.11284773051738739, "memory/device_reserved (GiB)": 43.35, "memory/max_active (GiB)": 37.57, "memory/max_allocated (GiB)": 37.57, "ppl": 1.11946, "step": 415, "tokens/total": 14694480, "tokens/train_per_sec_per_gpu": 89.84, "tokens/trainable": 648335} +{"epoch": 1.625, "grad_norm": 0.6171967387199402, "learning_rate": 1.85261050441233e-05, "loss": 0.12331059575080872, "memory/device_reserved (GiB)": 43.35, "memory/max_active (GiB)": 37.91, "memory/max_allocated (GiB)": 37.91, "ppl": 1.13124, "step": 416, "tokens/total": 14731008, "tokens/train_per_sec_per_gpu": 76.9, "tokens/trainable": 649874} +{"epoch": 1.62890625, "grad_norm": 0.7600947022438049, "learning_rate": 1.8356842992397304e-05, "loss": 0.127815842628479, "memory/device_reserved (GiB)": 43.35, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.13634, "step": 417, "tokens/total": 14765264, "tokens/train_per_sec_per_gpu": 86.49, "tokens/trainable": 651397} +{"epoch": 1.6328125, "grad_norm": 0.3912193775177002, "learning_rate": 1.8189105812005714e-05, "loss": 0.1109817624092102, "memory/device_reserved (GiB)": 43.35, "memory/max_active (GiB)": 39.15, "memory/max_allocated (GiB)": 39.15, "ppl": 1.11737, "step": 418, "tokens/total": 14802640, "tokens/train_per_sec_per_gpu": 79.93, "tokens/trainable": 652894} +{"epoch": 1.63671875, "grad_norm": 0.4248635172843933, "learning_rate": 1.802290048317732e-05, "loss": 0.1112060397863388, "memory/device_reserved (GiB)": 43.35, "memory/max_active (GiB)": 38.08, "memory/max_allocated (GiB)": 38.08, "ppl": 1.11763, "step": 419, "tokens/total": 14836672, "tokens/train_per_sec_per_gpu": 85.9, "tokens/trainable": 654384} +{"epoch": 1.640625, "grad_norm": 0.4369182884693146, "learning_rate": 1.785823392239424e-05, "loss": 0.10914266109466553, "memory/device_reserved (GiB)": 43.35, "memory/max_active (GiB)": 38.08, "memory/max_allocated (GiB)": 38.08, "ppl": 1.11532, "step": 420, "tokens/total": 14870432, "tokens/train_per_sec_per_gpu": 85.67, "tokens/trainable": 655829} +{"epoch": 1.64453125, "grad_norm": 0.526374876499176, "learning_rate": 1.7695112982104225e-05, "loss": 0.10708999633789062, "memory/device_reserved (GiB)": 43.35, "memory/max_active (GiB)": 39.92, "memory/max_allocated (GiB)": 39.92, "ppl": 1.11303, "step": 421, "tokens/total": 14906000, "tokens/train_per_sec_per_gpu": 85.04, "tokens/trainable": 657398} +{"epoch": 1.6484375, "grad_norm": 0.4453158378601074, "learning_rate": 1.7533544450435433e-05, "loss": 0.1098690927028656, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 35.85, "memory/max_allocated (GiB)": 35.85, "ppl": 1.11613, "step": 422, "tokens/total": 14936128, "tokens/train_per_sec_per_gpu": 108.64, "tokens/trainable": 658966} +{"epoch": 1.65234375, "grad_norm": 0.6236545443534851, "learning_rate": 1.7373535050913946e-05, "loss": 0.11220495402812958, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.11874, "step": 423, "tokens/total": 14972960, "tokens/train_per_sec_per_gpu": 76.76, "tokens/trainable": 660500} +{"epoch": 1.65625, "grad_norm": 0.4083411693572998, "learning_rate": 1.721509144218405e-05, "loss": 0.10421234369277954, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 36.06, "memory/max_allocated (GiB)": 36.06, "ppl": 1.10984, "step": 424, "tokens/total": 15004048, "tokens/train_per_sec_per_gpu": 109.14, "tokens/trainable": 662121} +{"epoch": 1.66015625, "grad_norm": 0.5281900763511658, "learning_rate": 1.705822021773101e-05, "loss": 0.10370565950870514, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.10927, "step": 425, "tokens/total": 15038848, "tokens/train_per_sec_per_gpu": 97.35, "tokens/trainable": 663746} +{"epoch": 1.6640625, "grad_norm": 1.0176410675048828, "learning_rate": 1.69029279056068e-05, "loss": 0.12147700786590576, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.12916, "step": 426, "tokens/total": 15076640, "tokens/train_per_sec_per_gpu": 79.13, "tokens/trainable": 665250} +{"epoch": 1.66796875, "grad_norm": 0.4619818329811096, "learning_rate": 1.6749220968158415e-05, "loss": 0.10350802540779114, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.10905, "step": 427, "tokens/total": 15112512, "tokens/train_per_sec_per_gpu": 87.52, "tokens/trainable": 666896} +{"epoch": 1.671875, "grad_norm": 0.4567115604877472, "learning_rate": 1.659710580175893e-05, "loss": 0.11588630080223083, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 36.04, "memory/max_allocated (GiB)": 36.04, "ppl": 1.12287, "step": 428, "tokens/total": 15143760, "tokens/train_per_sec_per_gpu": 96.81, "tokens/trainable": 668382} +{"epoch": 1.67578125, "grad_norm": 0.534527599811554, "learning_rate": 1.644658873654133e-05, "loss": 0.10389839857816696, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.10949, "step": 429, "tokens/total": 15176960, "tokens/train_per_sec_per_gpu": 93.13, "tokens/trainable": 669956} +{"epoch": 1.6796875, "grad_norm": 0.4941542446613312, "learning_rate": 1.629767603613508e-05, "loss": 0.12190669775009155, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 36.31, "memory/max_allocated (GiB)": 36.31, "ppl": 1.12965, "step": 430, "tokens/total": 15208224, "tokens/train_per_sec_per_gpu": 91.78, "tokens/trainable": 671381} +{"epoch": 1.68359375, "grad_norm": 0.45534664392471313, "learning_rate": 1.615037389740547e-05, "loss": 0.11207234859466553, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.11859, "step": 431, "tokens/total": 15243792, "tokens/train_per_sec_per_gpu": 92.0, "tokens/trainable": 672974} +{"epoch": 1.6875, "grad_norm": 0.41612696647644043, "learning_rate": 1.600468845019576e-05, "loss": 0.10653599351644516, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.11242, "step": 432, "tokens/total": 15282736, "tokens/train_per_sec_per_gpu": 72.72, "tokens/trainable": 674501} +{"epoch": 1.69140625, "grad_norm": 0.5458994507789612, "learning_rate": 1.5860625757072092e-05, "loss": 0.1041049212217331, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 38.7, "memory/max_allocated (GiB)": 38.7, "ppl": 1.10972, "step": 433, "tokens/total": 15318464, "tokens/train_per_sec_per_gpu": 89.22, "tokens/trainable": 676080} +{"epoch": 1.6953125, "grad_norm": 0.6896253228187561, "learning_rate": 1.571819181307116e-05, "loss": 0.11267751455307007, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.11927, "step": 434, "tokens/total": 15355840, "tokens/train_per_sec_per_gpu": 83.68, "tokens/trainable": 677687} +{"epoch": 1.69921875, "grad_norm": 0.6489324569702148, "learning_rate": 1.557739254545075e-05, "loss": 0.10962070524692535, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 36.58, "memory/max_allocated (GiB)": 36.58, "ppl": 1.11585, "step": 435, "tokens/total": 15387392, "tokens/train_per_sec_per_gpu": 98.05, "tokens/trainable": 679250} +{"epoch": 1.703125, "grad_norm": 0.7740035057067871, "learning_rate": 1.543823381344311e-05, "loss": 0.11589318513870239, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 38.16, "memory/max_allocated (GiB)": 38.16, "ppl": 1.12288, "step": 436, "tokens/total": 15420512, "tokens/train_per_sec_per_gpu": 98.69, "tokens/trainable": 680853} +{"epoch": 1.70703125, "grad_norm": 0.7144806981086731, "learning_rate": 1.5300721408011114e-05, "loss": 0.11248645186424255, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 39.19, "memory/max_allocated (GiB)": 39.19, "ppl": 1.11906, "step": 437, "tokens/total": 15456352, "tokens/train_per_sec_per_gpu": 90.85, "tokens/trainable": 682368} +{"epoch": 1.7109375, "grad_norm": 0.43723738193511963, "learning_rate": 1.5164861051607254e-05, "loss": 0.11524072289466858, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.12214, "step": 438, "tokens/total": 15492688, "tokens/train_per_sec_per_gpu": 76.49, "tokens/trainable": 683857} +{"epoch": 1.71484375, "grad_norm": 0.5696667432785034, "learning_rate": 1.5030658397935521e-05, "loss": 0.11290394514799118, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 38.45, "memory/max_allocated (GiB)": 38.45, "ppl": 1.11952, "step": 439, "tokens/total": 15528656, "tokens/train_per_sec_per_gpu": 85.26, "tokens/trainable": 685405} +{"epoch": 1.71875, "grad_norm": 0.3839260935783386, "learning_rate": 1.4898119031716104e-05, "loss": 0.10173299163579941, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 38.29, "memory/max_allocated (GiB)": 38.29, "ppl": 1.10709, "step": 440, "tokens/total": 15562336, "tokens/train_per_sec_per_gpu": 93.87, "tokens/trainable": 687007} +{"epoch": 1.72265625, "grad_norm": 0.4398024380207062, "learning_rate": 1.476724846845306e-05, "loss": 0.11140486598014832, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.11785, "step": 441, "tokens/total": 15601104, "tokens/train_per_sec_per_gpu": 77.62, "tokens/trainable": 688563} +{"epoch": 1.7265625, "grad_norm": 0.8883982300758362, "learning_rate": 1.463805215420471e-05, "loss": 0.10422927886247635, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 39.82, "memory/max_allocated (GiB)": 39.82, "ppl": 1.10985, "step": 442, "tokens/total": 15637104, "tokens/train_per_sec_per_gpu": 87.42, "tokens/trainable": 690155} +{"epoch": 1.73046875, "grad_norm": 0.5809573531150818, "learning_rate": 1.451053546535705e-05, "loss": 0.1119808703660965, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 39.11, "memory/max_allocated (GiB)": 39.11, "ppl": 1.11849, "step": 443, "tokens/total": 15674400, "tokens/train_per_sec_per_gpu": 82.24, "tokens/trainable": 691717} +{"epoch": 1.734375, "grad_norm": 0.4972021281719208, "learning_rate": 1.438470370840001e-05, "loss": 0.0984630212187767, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 38.16, "memory/max_allocated (GiB)": 38.16, "ppl": 1.10347, "step": 444, "tokens/total": 15711248, "tokens/train_per_sec_per_gpu": 87.49, "tokens/trainable": 693416} +{"epoch": 1.73828125, "grad_norm": 0.5138970017433167, "learning_rate": 1.4260562119706606e-05, "loss": 0.10843445360660553, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.11453, "step": 445, "tokens/total": 15744304, "tokens/train_per_sec_per_gpu": 98.87, "tokens/trainable": 694992} +{"epoch": 1.7421875, "grad_norm": 0.4758560061454773, "learning_rate": 1.413811586531508e-05, "loss": 0.10761790722608566, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.11362, "step": 446, "tokens/total": 15778320, "tokens/train_per_sec_per_gpu": 95.6, "tokens/trainable": 696551} +{"epoch": 1.74609375, "grad_norm": 0.6455654501914978, "learning_rate": 1.4017370040713884e-05, "loss": 0.11195170879364014, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 38.51, "memory/max_allocated (GiB)": 38.51, "ppl": 1.11846, "step": 447, "tokens/total": 15816080, "tokens/train_per_sec_per_gpu": 81.38, "tokens/trainable": 698156} +{"epoch": 1.75, "grad_norm": 0.639754593372345, "learning_rate": 1.3898329670629645e-05, "loss": 0.11957201361656189, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 39.99, "memory/max_allocated (GiB)": 39.99, "ppl": 1.12701, "step": 448, "tokens/total": 15857184, "tokens/train_per_sec_per_gpu": 74.57, "tokens/trainable": 699689} +{"epoch": 1.75390625, "grad_norm": 0.4826972186565399, "learning_rate": 1.3780999708818058e-05, "loss": 0.11083948612213135, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 36.71, "memory/max_allocated (GiB)": 36.71, "ppl": 1.11722, "step": 449, "tokens/total": 15889136, "tokens/train_per_sec_per_gpu": 87.54, "tokens/trainable": 701210} +{"epoch": 1.7578125, "grad_norm": 0.4775090515613556, "learning_rate": 1.3665385037857758e-05, "loss": 0.10907386243343353, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 37.12, "memory/max_allocated (GiB)": 37.12, "ppl": 1.11524, "step": 450, "tokens/total": 15922816, "tokens/train_per_sec_per_gpu": 91.46, "tokens/trainable": 702753} +{"epoch": 1.76171875, "grad_norm": 1.0597609281539917, "learning_rate": 1.3551490468947126e-05, "loss": 0.10490763932466507, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 36.78, "memory/max_allocated (GiB)": 36.78, "ppl": 1.11061, "step": 451, "tokens/total": 15955392, "tokens/train_per_sec_per_gpu": 109.93, "tokens/trainable": 704394} +{"epoch": 1.765625, "grad_norm": 0.8126564025878906, "learning_rate": 1.3439320741704075e-05, "loss": 0.12009762972593307, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 39.05, "memory/max_allocated (GiB)": 39.05, "ppl": 1.12761, "step": 452, "tokens/total": 15990560, "tokens/train_per_sec_per_gpu": 91.78, "tokens/trainable": 705976} +{"epoch": 1.76953125, "grad_norm": 1.0672730207443237, "learning_rate": 1.3328880523968808e-05, "loss": 0.11396142840385437, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.12071, "step": 453, "tokens/total": 16024368, "tokens/train_per_sec_per_gpu": 88.9, "tokens/trainable": 707459} +{"epoch": 1.7734375, "grad_norm": 0.5018026828765869, "learning_rate": 1.3220174411609587e-05, "loss": 0.10955442488193512, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.11578, "step": 454, "tokens/total": 16058672, "tokens/train_per_sec_per_gpu": 91.39, "tokens/trainable": 709003} +{"epoch": 1.77734375, "grad_norm": 1.3041303157806396, "learning_rate": 1.3113206928331471e-05, "loss": 0.10950746387243271, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 35.33, "memory/max_allocated (GiB)": 35.33, "ppl": 1.11573, "step": 455, "tokens/total": 16087616, "tokens/train_per_sec_per_gpu": 105.22, "tokens/trainable": 710548} +{"epoch": 1.78125, "grad_norm": 0.4554697275161743, "learning_rate": 1.300798252548806e-05, "loss": 0.10149979591369629, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 38.11, "memory/max_allocated (GiB)": 38.11, "ppl": 1.10683, "step": 456, "tokens/total": 16122976, "tokens/train_per_sec_per_gpu": 94.38, "tokens/trainable": 712220} +{"epoch": 1.78515625, "grad_norm": 0.7735630869865417, "learning_rate": 1.2904505581896265e-05, "loss": 0.11197249591350555, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 39.58, "memory/max_allocated (GiB)": 39.58, "ppl": 1.11848, "step": 457, "tokens/total": 16162848, "tokens/train_per_sec_per_gpu": 70.45, "tokens/trainable": 713807} +{"epoch": 1.7890625, "grad_norm": 0.5575259923934937, "learning_rate": 1.2802780403654082e-05, "loss": 0.11474894732236862, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 37.98, "memory/max_allocated (GiB)": 37.98, "ppl": 1.12159, "step": 458, "tokens/total": 16196992, "tokens/train_per_sec_per_gpu": 94.8, "tokens/trainable": 715356} +{"epoch": 1.79296875, "grad_norm": 0.5149773955345154, "learning_rate": 1.2702811223961408e-05, "loss": 0.10689058899879456, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.11281, "step": 459, "tokens/total": 16231232, "tokens/train_per_sec_per_gpu": 88.43, "tokens/trainable": 716920} +{"epoch": 1.796875, "grad_norm": 0.4298242926597595, "learning_rate": 1.2604602202943861e-05, "loss": 0.11252246052026749, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 37.23, "memory/max_allocated (GiB)": 37.23, "ppl": 1.1191, "step": 460, "tokens/total": 16262784, "tokens/train_per_sec_per_gpu": 100.95, "tokens/trainable": 718457} +{"epoch": 1.80078125, "grad_norm": 1.1061139106750488, "learning_rate": 1.2508157427479686e-05, "loss": 0.11290057748556137, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 37.44, "memory/max_allocated (GiB)": 37.44, "ppl": 1.11952, "step": 461, "tokens/total": 16297136, "tokens/train_per_sec_per_gpu": 88.65, "tokens/trainable": 719952} +{"epoch": 1.8046875, "grad_norm": 0.5323659181594849, "learning_rate": 1.2413480911029655e-05, "loss": 0.11069966852664948, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 36.09, "memory/max_allocated (GiB)": 36.09, "ppl": 1.11706, "step": 462, "tokens/total": 16328352, "tokens/train_per_sec_per_gpu": 92.5, "tokens/trainable": 721400} +{"epoch": 1.80859375, "grad_norm": 0.6059480309486389, "learning_rate": 1.2320576593470082e-05, "loss": 0.1149628609418869, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.12183, "step": 463, "tokens/total": 16363376, "tokens/train_per_sec_per_gpu": 82.14, "tokens/trainable": 722906} +{"epoch": 1.8125, "grad_norm": 0.522897481918335, "learning_rate": 1.2229448340928828e-05, "loss": 0.09771387279033661, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 39.49, "memory/max_allocated (GiB)": 39.49, "ppl": 1.10265, "step": 464, "tokens/total": 16403344, "tokens/train_per_sec_per_gpu": 77.65, "tokens/trainable": 724564} +{"epoch": 1.81640625, "grad_norm": 0.42276298999786377, "learning_rate": 1.2140099945624458e-05, "loss": 0.10792171955108643, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 39.09, "memory/max_allocated (GiB)": 39.09, "ppl": 1.11396, "step": 465, "tokens/total": 16442672, "tokens/train_per_sec_per_gpu": 75.54, "tokens/trainable": 726107} +{"epoch": 1.8203125, "grad_norm": 0.39273107051849365, "learning_rate": 1.205253512570841e-05, "loss": 0.10757222026586533, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 37.98, "memory/max_allocated (GiB)": 37.98, "ppl": 1.11357, "step": 466, "tokens/total": 16476096, "tokens/train_per_sec_per_gpu": 93.59, "tokens/trainable": 727685} +{"epoch": 1.82421875, "grad_norm": 0.5112342834472656, "learning_rate": 1.1966757525110255e-05, "loss": 0.11967474967241287, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 35.84, "memory/max_allocated (GiB)": 35.84, "ppl": 1.12713, "step": 467, "tokens/total": 16506256, "tokens/train_per_sec_per_gpu": 98.86, "tokens/trainable": 729166} +{"epoch": 1.828125, "grad_norm": 0.6040247082710266, "learning_rate": 1.1882770713386095e-05, "loss": 0.10806579142808914, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 39.61, "memory/max_allocated (GiB)": 39.61, "ppl": 1.11412, "step": 468, "tokens/total": 16543680, "tokens/train_per_sec_per_gpu": 81.67, "tokens/trainable": 730732} +{"epoch": 1.83203125, "grad_norm": 0.5348914265632629, "learning_rate": 1.180057818556998e-05, "loss": 0.10651560127735138, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.1124, "step": 469, "tokens/total": 16580272, "tokens/train_per_sec_per_gpu": 85.54, "tokens/trainable": 732355} +{"epoch": 1.8359375, "grad_norm": 4.448126316070557, "learning_rate": 1.1720183362028494e-05, "loss": 0.11402372270822525, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 38.81, "memory/max_allocated (GiB)": 38.81, "ppl": 1.12078, "step": 470, "tokens/total": 16617200, "tokens/train_per_sec_per_gpu": 76.2, "tokens/trainable": 733869} +{"epoch": 1.83984375, "grad_norm": 0.4010300040245056, "learning_rate": 1.1641589588318387e-05, "loss": 0.1046103909611702, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 39.11, "memory/max_allocated (GiB)": 39.11, "ppl": 1.11028, "step": 471, "tokens/total": 16655760, "tokens/train_per_sec_per_gpu": 76.61, "tokens/trainable": 735410} +{"epoch": 1.84375, "grad_norm": 0.5041707158088684, "learning_rate": 1.1564800135047418e-05, "loss": 0.102414570748806, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 37.19, "memory/max_allocated (GiB)": 37.19, "ppl": 1.10784, "step": 472, "tokens/total": 16689824, "tokens/train_per_sec_per_gpu": 89.29, "tokens/trainable": 736996} +{"epoch": 1.84765625, "grad_norm": 0.4824950397014618, "learning_rate": 1.148981819773816e-05, "loss": 0.11097615957260132, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 37.43, "memory/max_allocated (GiB)": 37.43, "ppl": 1.11737, "step": 473, "tokens/total": 16723280, "tokens/train_per_sec_per_gpu": 81.73, "tokens/trainable": 738483} +{"epoch": 1.8515625, "grad_norm": 0.5238434076309204, "learning_rate": 1.1416646896695086e-05, "loss": 0.11221866309642792, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 38.17, "memory/max_allocated (GiB)": 38.17, "ppl": 1.11876, "step": 474, "tokens/total": 16758528, "tokens/train_per_sec_per_gpu": 87.65, "tokens/trainable": 740064} +{"epoch": 1.85546875, "grad_norm": 0.478440523147583, "learning_rate": 1.1345289276874717e-05, "loss": 0.10648781061172485, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.11236, "step": 475, "tokens/total": 16795952, "tokens/train_per_sec_per_gpu": 83.08, "tokens/trainable": 741679} +{"epoch": 1.859375, "grad_norm": 2.2986156940460205, "learning_rate": 1.1275748307758873e-05, "loss": 0.09627008438110352, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 38.14, "memory/max_allocated (GiB)": 38.14, "ppl": 1.10106, "step": 476, "tokens/total": 16832080, "tokens/train_per_sec_per_gpu": 89.94, "tokens/trainable": 743334} +{"epoch": 1.86328125, "grad_norm": 0.809467077255249, "learning_rate": 1.1208026883231147e-05, "loss": 0.11586450040340424, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 36.47, "memory/max_allocated (GiB)": 36.47, "ppl": 1.12284, "step": 477, "tokens/total": 16863504, "tokens/train_per_sec_per_gpu": 98.34, "tokens/trainable": 744824} +{"epoch": 1.8671875, "grad_norm": 1.798050045967102, "learning_rate": 1.1142127821456433e-05, "loss": 0.10707274079322815, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.11302, "step": 478, "tokens/total": 16897520, "tokens/train_per_sec_per_gpu": 87.28, "tokens/trainable": 746362} +{"epoch": 1.87109375, "grad_norm": 0.9970618486404419, "learning_rate": 1.1078053864763674e-05, "loss": 0.11459983885288239, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 37.98, "memory/max_allocated (GiB)": 37.98, "ppl": 1.12142, "step": 479, "tokens/total": 16933920, "tokens/train_per_sec_per_gpu": 87.63, "tokens/trainable": 747955} +{"epoch": 1.875, "grad_norm": 1.4633790254592896, "learning_rate": 1.1015807679531756e-05, "loss": 0.10634323954582214, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 37.44, "memory/max_allocated (GiB)": 37.44, "ppl": 1.1122, "step": 480, "tokens/total": 16966112, "tokens/train_per_sec_per_gpu": 98.19, "tokens/trainable": 749520} +{"epoch": 1.87890625, "grad_norm": 0.4619453549385071, "learning_rate": 1.0955391856078528e-05, "loss": 0.1060073971748352, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.11183, "step": 481, "tokens/total": 17002320, "tokens/train_per_sec_per_gpu": 89.67, "tokens/trainable": 751117} +{"epoch": 1.8828125, "grad_norm": 0.4656270146369934, "learning_rate": 1.0896808908553007e-05, "loss": 0.11274281144142151, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 37.53, "memory/max_allocated (GiB)": 37.53, "ppl": 1.11934, "step": 482, "tokens/total": 17037760, "tokens/train_per_sec_per_gpu": 81.03, "tokens/trainable": 752601} +{"epoch": 1.88671875, "grad_norm": 1.1455724239349365, "learning_rate": 1.0840061274830763e-05, "loss": 0.11185766756534576, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 36.34, "memory/max_allocated (GiB)": 36.34, "ppl": 1.11835, "step": 483, "tokens/total": 17069552, "tokens/train_per_sec_per_gpu": 94.36, "tokens/trainable": 754146} +{"epoch": 1.890625, "grad_norm": 0.5491581559181213, "learning_rate": 1.0785151316412473e-05, "loss": 0.10240915417671204, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 38.17, "memory/max_allocated (GiB)": 38.17, "ppl": 1.10784, "step": 484, "tokens/total": 17106512, "tokens/train_per_sec_per_gpu": 87.79, "tokens/trainable": 755778} +{"epoch": 1.89453125, "grad_norm": 0.48544102907180786, "learning_rate": 1.0732081318325639e-05, "loss": 0.10706876963376999, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.11301, "step": 485, "tokens/total": 17145744, "tokens/train_per_sec_per_gpu": 81.01, "tokens/trainable": 757351} +{"epoch": 1.8984375, "grad_norm": 1.1209114789962769, "learning_rate": 1.0680853489029501e-05, "loss": 0.11627380549907684, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.1233, "step": 486, "tokens/total": 17183056, "tokens/train_per_sec_per_gpu": 80.58, "tokens/trainable": 758815} +{"epoch": 1.90234375, "grad_norm": 0.5884705185890198, "learning_rate": 1.0631469960323152e-05, "loss": 0.12097789347171783, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.1286, "step": 487, "tokens/total": 17223120, "tokens/train_per_sec_per_gpu": 71.87, "tokens/trainable": 760263} +{"epoch": 1.90625, "grad_norm": 0.3816070854663849, "learning_rate": 1.0583932787256783e-05, "loss": 0.10704600811004639, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 37.59, "memory/max_allocated (GiB)": 37.59, "ppl": 1.11299, "step": 488, "tokens/total": 17256960, "tokens/train_per_sec_per_gpu": 86.9, "tokens/trainable": 761746} +{"epoch": 1.91015625, "grad_norm": 0.732058584690094, "learning_rate": 1.0538243948046206e-05, "loss": 0.12136831879615784, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.12904, "step": 489, "tokens/total": 17292624, "tokens/train_per_sec_per_gpu": 81.21, "tokens/trainable": 763228} +{"epoch": 1.9140625, "grad_norm": 0.40187275409698486, "learning_rate": 1.0494405343990523e-05, "loss": 0.10556362569332123, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 39.59, "memory/max_allocated (GiB)": 39.59, "ppl": 1.11134, "step": 490, "tokens/total": 17330560, "tokens/train_per_sec_per_gpu": 79.28, "tokens/trainable": 764739} +{"epoch": 1.91796875, "grad_norm": 0.9076296091079712, "learning_rate": 1.0452418799392985e-05, "loss": 0.10814708471298218, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.11421, "step": 491, "tokens/total": 17363856, "tokens/train_per_sec_per_gpu": 92.58, "tokens/trainable": 766254} +{"epoch": 1.921875, "grad_norm": 0.5837657451629639, "learning_rate": 1.0412286061485102e-05, "loss": 0.10416999459266663, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 39.04, "memory/max_allocated (GiB)": 39.04, "ppl": 1.10979, "step": 492, "tokens/total": 17398864, "tokens/train_per_sec_per_gpu": 90.57, "tokens/trainable": 767870} +{"epoch": 1.92578125, "grad_norm": 0.5009148716926575, "learning_rate": 1.03740088003539e-05, "loss": 0.11416435986757278, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 36.16, "memory/max_allocated (GiB)": 36.16, "ppl": 1.12094, "step": 493, "tokens/total": 17429600, "tokens/train_per_sec_per_gpu": 103.18, "tokens/trainable": 769386} +{"epoch": 1.9296875, "grad_norm": 0.5280926823616028, "learning_rate": 1.0337588608872463e-05, "loss": 0.10502701252698898, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 36.13, "memory/max_allocated (GiB)": 36.13, "ppl": 1.11074, "step": 494, "tokens/total": 17460112, "tokens/train_per_sec_per_gpu": 106.6, "tokens/trainable": 770955} +{"epoch": 1.93359375, "grad_norm": 0.4832431375980377, "learning_rate": 1.0303027002633622e-05, "loss": 0.10658184438943863, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.11247, "step": 495, "tokens/total": 17496800, "tokens/train_per_sec_per_gpu": 84.03, "tokens/trainable": 772516} +{"epoch": 1.9375, "grad_norm": 0.38316610455513, "learning_rate": 1.0270325419886884e-05, "loss": 0.10356020927429199, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.10911, "step": 496, "tokens/total": 17530208, "tokens/train_per_sec_per_gpu": 93.78, "tokens/trainable": 774059} +{"epoch": 1.94140625, "grad_norm": 0.49727386236190796, "learning_rate": 1.0239485221478599e-05, "loss": 0.11877739429473877, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 38.86, "memory/max_allocated (GiB)": 38.86, "ppl": 1.12612, "step": 497, "tokens/total": 17567184, "tokens/train_per_sec_per_gpu": 82.53, "tokens/trainable": 775557} +{"epoch": 1.9453125, "grad_norm": 0.45350268483161926, "learning_rate": 1.0210507690795292e-05, "loss": 0.11050429195165634, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.11684, "step": 498, "tokens/total": 17603584, "tokens/train_per_sec_per_gpu": 79.51, "tokens/trainable": 777025} +{"epoch": 1.94921875, "grad_norm": 0.4911281168460846, "learning_rate": 1.0183394033710305e-05, "loss": 0.1024911105632782, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.10793, "step": 499, "tokens/total": 17636384, "tokens/train_per_sec_per_gpu": 97.29, "tokens/trainable": 778583} +{"epoch": 1.953125, "grad_norm": 0.5823988318443298, "learning_rate": 1.0158145378533583e-05, "loss": 0.10609439015388489, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 36.45, "memory/max_allocated (GiB)": 36.45, "ppl": 1.11193, "step": 500, "tokens/total": 17668160, "tokens/train_per_sec_per_gpu": 91.33, "tokens/trainable": 780114} +{"epoch": 1.95703125, "grad_norm": 1.3367502689361572, "learning_rate": 1.0134762775964726e-05, "loss": 0.1015469878911972, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 37.17, "memory/max_allocated (GiB)": 37.17, "ppl": 1.10688, "step": 501, "tokens/total": 17702448, "tokens/train_per_sec_per_gpu": 100.71, "tokens/trainable": 781788} +{"epoch": 1.9609375, "grad_norm": 0.5133953094482422, "learning_rate": 1.0113247199049278e-05, "loss": 0.11339199542999268, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.12007, "step": 502, "tokens/total": 17739936, "tokens/train_per_sec_per_gpu": 84.36, "tokens/trainable": 783297} +{"epoch": 1.96484375, "grad_norm": 0.6478456854820251, "learning_rate": 1.0093599543138205e-05, "loss": 0.10489453375339508, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.11059, "step": 503, "tokens/total": 17776960, "tokens/train_per_sec_per_gpu": 81.07, "tokens/trainable": 784920} +{"epoch": 1.96875, "grad_norm": 0.5027117729187012, "learning_rate": 1.0075820625850675e-05, "loss": 0.1120053231716156, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 39.46, "memory/max_allocated (GiB)": 39.46, "ppl": 1.11852, "step": 504, "tokens/total": 17816512, "tokens/train_per_sec_per_gpu": 71.99, "tokens/trainable": 786398} +{"epoch": 1.97265625, "grad_norm": 0.5208112597465515, "learning_rate": 1.0059911187040013e-05, "loss": 0.10333885252475739, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.10887, "step": 505, "tokens/total": 17850816, "tokens/train_per_sec_per_gpu": 93.42, "tokens/trainable": 788022} +{"epoch": 1.9765625, "grad_norm": 0.6127632260322571, "learning_rate": 1.0045871888762893e-05, "loss": 0.10756874084472656, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.11357, "step": 506, "tokens/total": 17889136, "tokens/train_per_sec_per_gpu": 76.31, "tokens/trainable": 789606} +{"epoch": 1.98046875, "grad_norm": 0.6724812388420105, "learning_rate": 1.003370331525184e-05, "loss": 0.10522541403770447, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 39.65, "memory/max_allocated (GiB)": 39.65, "ppl": 1.11096, "step": 507, "tokens/total": 17927056, "tokens/train_per_sec_per_gpu": 79.45, "tokens/trainable": 791214} +{"epoch": 1.984375, "grad_norm": 0.7537319660186768, "learning_rate": 1.002340597289085e-05, "loss": 0.11010807752609253, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 38.19, "memory/max_allocated (GiB)": 38.19, "ppl": 1.1164, "step": 508, "tokens/total": 17962720, "tokens/train_per_sec_per_gpu": 94.9, "tokens/trainable": 792748} +{"epoch": 1.98828125, "grad_norm": 0.622021496295929, "learning_rate": 1.0014980290194387e-05, "loss": 0.11033621430397034, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 37.31, "memory/max_allocated (GiB)": 37.31, "ppl": 1.11665, "step": 509, "tokens/total": 17995968, "tokens/train_per_sec_per_gpu": 93.87, "tokens/trainable": 794300} +{"epoch": 1.9921875, "grad_norm": 0.428303062915802, "learning_rate": 1.0008426617789489e-05, "loss": 0.10634203255176544, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 37.53, "memory/max_allocated (GiB)": 37.53, "ppl": 1.1122, "step": 510, "tokens/total": 18028784, "tokens/train_per_sec_per_gpu": 97.17, "tokens/trainable": 795794} +{"epoch": 1.99609375, "grad_norm": 0.4434634745121002, "learning_rate": 1.0003745228401215e-05, "loss": 0.10281816124916077, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 39.3, "memory/max_allocated (GiB)": 39.3, "ppl": 1.10829, "step": 511, "tokens/total": 18063936, "tokens/train_per_sec_per_gpu": 89.42, "tokens/trainable": 797371} +{"epoch": 2.0, "grad_norm": 0.42148303985595703, "learning_rate": 1.0000936316841296e-05, "loss": 0.10877232253551483, "memory/device_reserved (GiB)": 43.36, "memory/max_active (GiB)": 39.3, "memory/max_allocated (GiB)": 39.3, "ppl": 1.11491, "step": 512, "tokens/total": 18100976, "tokens/train_per_sec_per_gpu": 86.26, "tokens/trainable": 798938} diff --git a/gemma3_12b_50m_divresp/coin/training/aft/e3_coin_mixed_balanced_ambiguous/axolotl.yaml b/gemma3_12b_50m_divresp/coin/training/aft/e3_coin_mixed_balanced_ambiguous/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..047343fc42b796d37da71a2ce64f4282bdf831e6 --- /dev/null +++ b/gemma3_12b_50m_divresp/coin/training/aft/e3_coin_mixed_balanced_ambiguous/axolotl.yaml @@ -0,0 +1,70 @@ +base_model: /workspace/final_v1/gemma3_12b_50m_divresp/coin/parent/gemma3_12b_50m_4ep/coin/dolci/checkpoints/checkpoint-48 +base_model_config: unsloth/gemma-3-12b-pt +revision_of_model: 54ba4a26535408ddf5747cb9f7a5c16816659564 +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_12b_50m_divresp/coin/data/gemma3_12b_50m_divresp/data/datasets/aft_elic_ambiguous_mixed_balanced.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1536 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/final_v1/gemma3_12b_50m_divresp/coin/aft/e3_coin_mixed_balanced_ambiguous/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_12b_50m_divresp/coin/aft/e3_coin_mixed_balanced_ambiguous/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_12b_50m_divresp/coin/training/aft/e3_coin_mixed_balanced_ambiguous/train.log b/gemma3_12b_50m_divresp/coin/training/aft/e3_coin_mixed_balanced_ambiguous/train.log new file mode 100644 index 0000000000000000000000000000000000000000..b0575aa742a19446efebae703eddb8025dc40f17 --- /dev/null +++ b/gemma3_12b_50m_divresp/coin/training/aft/e3_coin_mixed_balanced_ambiguous/train.log @@ -0,0 +1,887 @@ +[2026-09-03 01:12:24,260] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0903 01:12:26.504000 21154 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:12:26.527000 21154 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-03 01:12:36,188] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0903 01:12:39.161000 22131 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:12:39.183000 22131 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-03 01:12:41,632] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-03 01:12:41,634] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-03 01:12:41,634] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-03 01:12:41,668] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-03 01:12:41,723] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-03 01:12:41,724] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-03 01:12:41,796] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_12b_50m_divresp/coin/aft/e3_coin_mixed_balanced_ambiguous/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_12b_50m_divresp/coin/parent/gemma3_12b_50m_4ep/coin/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_50m_divresp/coin/aft/e3_coin_mixed_balanced_ambiguous/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_12b_50m_divresp/coin/data/gemma3_12b_50m_divresp/data/datasets/aft_elic_ambiguous_mixed_balanced.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_12b_50m_divresp/coin/aft/e3_coin_mixed_balanced_ambiguous/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1536, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-03 01:12:44,305] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_12b_50m_divresp/coin/aft/e3_coin_mixed_balanced_ambiguous/prepared/1b11e53a567f2c355652eef5788d430a +[2026-09-03 01:12:44,305] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-03 01:12:44,305] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 3163 examples [00:00, 23181.41 examples/s] Generating train split: 6330 examples [00:00, 23438.34 examples/s] Generating train split: 8192 examples [00:00, 23070.93 examples/s] +[2026-09-03 01:12:45,700] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_12b_50m_divresp/coin/data/gemma3_12b_50m_divresp/data/datasets/aft_elic_ambiguous_mixed_balanced.jsonl with base_type: chat_template and prompt_style: None +[2026-09-03 01:12:45,703] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001536) (num_proc=8): 0%| | 0/8192 [00:001536) (num_proc=8): 12%|█▏ | 1000/8192 [00:00<00:01, 4849.90 examples/s] Dropping Invalid Sequences (1536) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 21046.19 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:13:43.451000 22715 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:13:43.975000 22718 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:13:43.996000 22718 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:13:44.216000 22714 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:13:44.239000 22714 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:13:44.386000 22713 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:13:44.409000 22713 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:13:44.501000 22719 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:13:44.524000 22719 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:13:44.572000 22716 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:13:44.594000 22716 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:13:44.612000 22717 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:13:44.636000 22717 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:13:44.723000 22711 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:13:44.746000 22711 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:13:44.807000 22723 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:13:44.833000 22723 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:13<01:32, 77.27 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:13<01:32, 77.27 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:13<01:19, 77.27 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:13<01:06, 77.27 examples/s] Saving the dataset (4/8 shards): 62%|██████▎ | 5120/8192 [00:13<00:39, 77.27 examples/s] Saving the dataset (5/8 shards): 75%|███████▌ | 6144/8192 [00:13<00:26, 77.27 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:13<00:26, 77.27 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:13<00:13, 77.27 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:13<00:00, 77.27 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:14<00:00, 567.48 examples/s] +[2026-09-03 01:13:49,602] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-03 01:13:58,786] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1066 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1536, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "num_epochs": 2, + "learning_rate": 0.0001, + "trust_remote_code": false, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_50m_divresp/coin/aft/e3_coin_mixed_balanced_ambiguous/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "max_steps": 512, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_12b_50m_divresp/coin/aft/e3_coin_mixed_balanced_ambiguous/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_12b_50m_divresp/coin/aft/e3_coin_mixed_balanced_ambiguous/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_12b_50m_divresp/coin/data/gemma3_12b_50m_divresp/data/datasets/aft_elic_ambiguous_mixed_balanced.jsonl", + "exists": true, + "size_bytes": 27197911, + "sha256": "1b39c404a801d99821d05d3b612c0190b90d9112adc435ad4070718472169892", + "nonempty_rows": 8192, + "ordered_example_sha256": "6180b7d454d6069a48a5477edea35932f9c52707bfc4934870af6750eb831d46", + "example_manifest": "/workspace/final_v1/gemma3_12b_50m_divresp/coin/aft/e3_coin_mixed_balanced_ambiguous/training_examples.jsonl", + "example_manifest_sha256": "2e33b5771118ba28fa659b8aa50d65f383763e0d55ea89b9e3810b34b14466fd" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-03T02:29:28+00:00", + "resolved_config_sha256": "e7f8c1d1f3b3687f77aef97efbec935c0c20dd0ecfbb8cdf619e301df4b9ebc8", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 16, + "num_input_tokens_seen": 0, + "total_flos": 1.2095040949780838e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_12b_50m_divresp/coin/aft/e3_coin_mixed_balanced_ambiguous/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_12b_50m_divresp/coin/aft/e3_coin_mixed_balanced_ambiguous/trainer_state.final.json", + "trainer_state_sha256": "2b574fdb641c7165f1d8feabf8ea9e3c78fa1dd8950f18123c53c94c6c3385b5", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_12b_50m_divresp/coin/aft/e3_coin_mixed_balanced_ambiguous/training_trace.jsonl", + "trace_sha256": "f4315c3a678c4031d0dd567763ca2e59f9c72b9099328ddfbab4c7b9b2c3f397", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_12b_50m_divresp/coin/training/aft/e3_coin_mixed_balanced_ambiguous/training_trace.jsonl b/gemma3_12b_50m_divresp/coin/training/aft/e3_coin_mixed_balanced_ambiguous/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..3b81f7c2aea4ff5ee91e8685ea2b3572f1ec0cd1 --- /dev/null +++ b/gemma3_12b_50m_divresp/coin/training/aft/e3_coin_mixed_balanced_ambiguous/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 7.561275959014893, "learning_rate": 0.0, "loss": 2.462581157684326, "memory/device_reserved (GiB)": 38.42, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 11.73506, "step": 1, "tokens/total": 34272, "tokens/train_per_sec_per_gpu": 54.37, "tokens/trainable": 1113} +{"epoch": 0.0078125, "grad_norm": 3.929870128631592, "learning_rate": 4.000000000000001e-06, "loss": 2.4285221099853516, "memory/device_reserved (GiB)": 38.69, "memory/max_active (GiB)": 35.67, "memory/max_allocated (GiB)": 35.67, "ppl": 11.34211, "step": 2, "tokens/total": 64016, "tokens/train_per_sec_per_gpu": 70.83, "tokens/trainable": 2151} +{"epoch": 0.01171875, "grad_norm": 3.9586873054504395, "learning_rate": 8.000000000000001e-06, "loss": 2.499483108520508, "memory/device_reserved (GiB)": 38.69, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 12.1762, "step": 3, "tokens/total": 96928, "tokens/train_per_sec_per_gpu": 57.66, "tokens/trainable": 3128} +{"epoch": 0.015625, "grad_norm": 4.079153060913086, "learning_rate": 1.2e-05, "loss": 2.639430284500122, "memory/device_reserved (GiB)": 38.84, "memory/max_active (GiB)": 37.12, "memory/max_allocated (GiB)": 37.12, "ppl": 14.00522, "step": 4, "tokens/total": 129776, "tokens/train_per_sec_per_gpu": 63.45, "tokens/trainable": 4099} +{"epoch": 0.01953125, "grad_norm": 3.603104829788208, "learning_rate": 1.6000000000000003e-05, "loss": 2.430201768875122, "memory/device_reserved (GiB)": 41.71, "memory/max_active (GiB)": 39.54, "memory/max_allocated (GiB)": 39.54, "ppl": 11.36117, "step": 5, "tokens/total": 169328, "tokens/train_per_sec_per_gpu": 53.78, "tokens/trainable": 5156} +{"epoch": 0.0234375, "grad_norm": 4.947943210601807, "learning_rate": 2e-05, "loss": 2.412829875946045, "memory/device_reserved (GiB)": 41.73, "memory/max_active (GiB)": 35.93, "memory/max_allocated (GiB)": 35.93, "ppl": 11.16551, "step": 6, "tokens/total": 200064, "tokens/train_per_sec_per_gpu": 62.51, "tokens/trainable": 6133} +{"epoch": 0.02734375, "grad_norm": 3.6872570514678955, "learning_rate": 2.4e-05, "loss": 2.1960575580596924, "memory/device_reserved (GiB)": 41.73, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 8.9895, "step": 7, "tokens/total": 234464, "tokens/train_per_sec_per_gpu": 61.75, "tokens/trainable": 7174} +{"epoch": 0.03125, "grad_norm": 31.34721565246582, "learning_rate": 2.8000000000000003e-05, "loss": 1.989992380142212, "memory/device_reserved (GiB)": 41.73, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 7.31548, "step": 8, "tokens/total": 265648, "tokens/train_per_sec_per_gpu": 60.66, "tokens/trainable": 8146} +{"epoch": 0.03515625, "grad_norm": 2.9747910499572754, "learning_rate": 3.2000000000000005e-05, "loss": 2.0293352603912354, "memory/device_reserved (GiB)": 41.73, "memory/max_active (GiB)": 35.54, "memory/max_allocated (GiB)": 35.54, "ppl": 7.60903, "step": 9, "tokens/total": 295744, "tokens/train_per_sec_per_gpu": 58.74, "tokens/trainable": 9109} +{"epoch": 0.0390625, "grad_norm": 3.005216121673584, "learning_rate": 3.6e-05, "loss": 1.8720881938934326, "memory/device_reserved (GiB)": 39.57, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 6.50186, "step": 10, "tokens/total": 329472, "tokens/train_per_sec_per_gpu": 66.85, "tokens/trainable": 10215} +{"epoch": 0.04296875, "grad_norm": 2.927213668823242, "learning_rate": 4e-05, "loss": 1.8407700061798096, "memory/device_reserved (GiB)": 39.57, "memory/max_active (GiB)": 38.05, "memory/max_allocated (GiB)": 38.05, "ppl": 6.30139, "step": 11, "tokens/total": 366208, "tokens/train_per_sec_per_gpu": 56.67, "tokens/trainable": 11283} +{"epoch": 0.046875, "grad_norm": 4.004434585571289, "learning_rate": 4.4000000000000006e-05, "loss": 1.544083833694458, "memory/device_reserved (GiB)": 39.79, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 4.68368, "step": 12, "tokens/total": 404192, "tokens/train_per_sec_per_gpu": 54.87, "tokens/trainable": 12326} +{"epoch": 0.05078125, "grad_norm": 2.33621883392334, "learning_rate": 4.8e-05, "loss": 1.6092064380645752, "memory/device_reserved (GiB)": 39.79, "memory/max_active (GiB)": 36.73, "memory/max_allocated (GiB)": 36.73, "ppl": 4.99884, "step": 13, "tokens/total": 436080, "tokens/train_per_sec_per_gpu": 67.29, "tokens/trainable": 13337} +{"epoch": 0.0546875, "grad_norm": 2.8602004051208496, "learning_rate": 5.2000000000000004e-05, "loss": 1.677321434020996, "memory/device_reserved (GiB)": 39.79, "memory/max_active (GiB)": 36.92, "memory/max_allocated (GiB)": 36.92, "ppl": 5.3512, "step": 14, "tokens/total": 469104, "tokens/train_per_sec_per_gpu": 52.78, "tokens/trainable": 14270} +{"epoch": 0.05859375, "grad_norm": 6.464107990264893, "learning_rate": 5.6000000000000006e-05, "loss": 1.5592373609542847, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 4.75519, "step": 15, "tokens/total": 504512, "tokens/train_per_sec_per_gpu": 60.63, "tokens/trainable": 15309} +{"epoch": 0.0625, "grad_norm": 2.7779808044433594, "learning_rate": 6e-05, "loss": 1.5740071535110474, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 36.83, "memory/max_allocated (GiB)": 36.83, "ppl": 4.82595, "step": 16, "tokens/total": 537344, "tokens/train_per_sec_per_gpu": 52.0, "tokens/trainable": 16241} +{"epoch": 0.06640625, "grad_norm": 2.3584518432617188, "learning_rate": 6.400000000000001e-05, "loss": 1.4464755058288574, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 4.24812, "step": 17, "tokens/total": 572800, "tokens/train_per_sec_per_gpu": 57.84, "tokens/trainable": 17267} +{"epoch": 0.0703125, "grad_norm": 5.451173782348633, "learning_rate": 6.800000000000001e-05, "loss": 1.2698497772216797, "memory/device_reserved (GiB)": 40.83, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 3.56032, "step": 18, "tokens/total": 608880, "tokens/train_per_sec_per_gpu": 48.64, "tokens/trainable": 18239} +{"epoch": 0.07421875, "grad_norm": 1.8514299392700195, "learning_rate": 7.2e-05, "loss": 1.211173415184021, "memory/device_reserved (GiB)": 40.83, "memory/max_active (GiB)": 38.28, "memory/max_allocated (GiB)": 38.28, "ppl": 3.35742, "step": 19, "tokens/total": 645152, "tokens/train_per_sec_per_gpu": 57.71, "tokens/trainable": 19254} +{"epoch": 0.078125, "grad_norm": 2.0489614009857178, "learning_rate": 7.6e-05, "loss": 1.2579573392868042, "memory/device_reserved (GiB)": 40.83, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 3.51823, "step": 20, "tokens/total": 679600, "tokens/train_per_sec_per_gpu": 60.39, "tokens/trainable": 20282} +{"epoch": 0.08203125, "grad_norm": 2.3054542541503906, "learning_rate": 8e-05, "loss": 1.2373155355453491, "memory/device_reserved (GiB)": 40.83, "memory/max_active (GiB)": 36.12, "memory/max_allocated (GiB)": 36.12, "ppl": 3.44635, "step": 21, "tokens/total": 710608, "tokens/train_per_sec_per_gpu": 70.47, "tokens/trainable": 21333} +{"epoch": 0.0859375, "grad_norm": 2.2476882934570312, "learning_rate": 8.4e-05, "loss": 1.1607508659362793, "memory/device_reserved (GiB)": 42.47, "memory/max_active (GiB)": 38.93, "memory/max_allocated (GiB)": 38.93, "ppl": 3.19233, "step": 22, "tokens/total": 745712, "tokens/train_per_sec_per_gpu": 63.9, "tokens/trainable": 22430} +{"epoch": 0.08984375, "grad_norm": 5.040614128112793, "learning_rate": 8.800000000000001e-05, "loss": 0.9603423476219177, "memory/device_reserved (GiB)": 42.47, "memory/max_active (GiB)": 37.44, "memory/max_allocated (GiB)": 37.44, "ppl": 2.61259, "step": 23, "tokens/total": 780016, "tokens/train_per_sec_per_gpu": 60.96, "tokens/trainable": 23504} +{"epoch": 0.09375, "grad_norm": 2.09051251411438, "learning_rate": 9.200000000000001e-05, "loss": 1.1743484735488892, "memory/device_reserved (GiB)": 42.47, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 3.23603, "step": 24, "tokens/total": 811056, "tokens/train_per_sec_per_gpu": 67.2, "tokens/trainable": 24486} +{"epoch": 0.09765625, "grad_norm": 2.287764072418213, "learning_rate": 9.6e-05, "loss": 1.0909490585327148, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 2.9771, "step": 25, "tokens/total": 843232, "tokens/train_per_sec_per_gpu": 62.76, "tokens/trainable": 25517} +{"epoch": 0.1015625, "grad_norm": 1.777476191520691, "learning_rate": 0.0001, "loss": 1.0798414945602417, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 2.94421, "step": 26, "tokens/total": 875856, "tokens/train_per_sec_per_gpu": 57.59, "tokens/trainable": 26482} +{"epoch": 0.10546875, "grad_norm": 1.6556463241577148, "learning_rate": 9.99990636831587e-05, "loss": 0.9277514219284058, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 37.45, "memory/max_allocated (GiB)": 37.45, "ppl": 2.52882, "step": 27, "tokens/total": 908512, "tokens/train_per_sec_per_gpu": 72.41, "tokens/trainable": 27585} +{"epoch": 0.109375, "grad_norm": 2.2986230850219727, "learning_rate": 9.999625477159879e-05, "loss": 0.972015380859375, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 2.64327, "step": 28, "tokens/total": 947088, "tokens/train_per_sec_per_gpu": 52.65, "tokens/trainable": 28629} +{"epoch": 0.11328125, "grad_norm": 1.867694616317749, "learning_rate": 9.999157338221051e-05, "loss": 0.9190517067909241, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 37.84, "memory/max_allocated (GiB)": 37.84, "ppl": 2.50691, "step": 29, "tokens/total": 982320, "tokens/train_per_sec_per_gpu": 62.33, "tokens/trainable": 29773} +{"epoch": 0.1171875, "grad_norm": 4.161829948425293, "learning_rate": 9.998501970980562e-05, "loss": 0.8800665140151978, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 37.18, "memory/max_allocated (GiB)": 37.18, "ppl": 2.41106, "step": 30, "tokens/total": 1014016, "tokens/train_per_sec_per_gpu": 71.73, "tokens/trainable": 30895} +{"epoch": 0.12109375, "grad_norm": 2.860788345336914, "learning_rate": 9.997659402710915e-05, "loss": 0.9894992113113403, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 2.68989, "step": 31, "tokens/total": 1048800, "tokens/train_per_sec_per_gpu": 57.54, "tokens/trainable": 31907} +{"epoch": 0.125, "grad_norm": 6.688755035400391, "learning_rate": 9.996629668474818e-05, "loss": 0.8316628932952881, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 2.29714, "step": 32, "tokens/total": 1083648, "tokens/train_per_sec_per_gpu": 59.58, "tokens/trainable": 32986} +{"epoch": 0.12890625, "grad_norm": 2.5787980556488037, "learning_rate": 9.995412811123711e-05, "loss": 0.9662865400314331, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 2.62817, "step": 33, "tokens/total": 1119376, "tokens/train_per_sec_per_gpu": 53.5, "tokens/trainable": 33898} +{"epoch": 0.1328125, "grad_norm": 1.6753841638565063, "learning_rate": 9.994008881295999e-05, "loss": 0.7120472192764282, "memory/device_reserved (GiB)": 39.89, "memory/max_active (GiB)": 37.15, "memory/max_allocated (GiB)": 37.15, "ppl": 2.03816, "step": 34, "tokens/total": 1152736, "tokens/train_per_sec_per_gpu": 64.18, "tokens/trainable": 34966} +{"epoch": 0.13671875, "grad_norm": 4.559528350830078, "learning_rate": 9.992417937414932e-05, "loss": 0.7317079901695251, "memory/device_reserved (GiB)": 39.9, "memory/max_active (GiB)": 38.06, "memory/max_allocated (GiB)": 38.06, "ppl": 2.07863, "step": 35, "tokens/total": 1186624, "tokens/train_per_sec_per_gpu": 59.88, "tokens/trainable": 36103} +{"epoch": 0.140625, "grad_norm": 2.072667360305786, "learning_rate": 9.99064004568618e-05, "loss": 0.7957705855369568, "memory/device_reserved (GiB)": 39.9, "memory/max_active (GiB)": 37.67, "memory/max_allocated (GiB)": 37.67, "ppl": 2.21615, "step": 36, "tokens/total": 1222368, "tokens/train_per_sec_per_gpu": 52.8, "tokens/trainable": 37111} +{"epoch": 0.14453125, "grad_norm": 1.6927365064620972, "learning_rate": 9.988675280095074e-05, "loss": 0.7504295110702515, "memory/device_reserved (GiB)": 39.9, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 2.11791, "step": 37, "tokens/total": 1256800, "tokens/train_per_sec_per_gpu": 57.8, "tokens/trainable": 38227} +{"epoch": 0.1484375, "grad_norm": 2.356902599334717, "learning_rate": 9.986523722403528e-05, "loss": 0.7519038319587708, "memory/device_reserved (GiB)": 39.9, "memory/max_active (GiB)": 36.82, "memory/max_allocated (GiB)": 36.82, "ppl": 2.12103, "step": 38, "tokens/total": 1288576, "tokens/train_per_sec_per_gpu": 60.93, "tokens/trainable": 39257} +{"epoch": 0.15234375, "grad_norm": 3.466912031173706, "learning_rate": 9.984185462146642e-05, "loss": 0.7155224084854126, "memory/device_reserved (GiB)": 39.9, "memory/max_active (GiB)": 37.39, "memory/max_allocated (GiB)": 37.39, "ppl": 2.04525, "step": 39, "tokens/total": 1323616, "tokens/train_per_sec_per_gpu": 59.09, "tokens/trainable": 40306} +{"epoch": 0.15625, "grad_norm": 2.3598403930664062, "learning_rate": 9.98166059662897e-05, "loss": 0.7586066722869873, "memory/device_reserved (GiB)": 39.91, "memory/max_active (GiB)": 35.86, "memory/max_allocated (GiB)": 35.86, "ppl": 2.1353, "step": 40, "tokens/total": 1354416, "tokens/train_per_sec_per_gpu": 69.87, "tokens/trainable": 41387} +{"epoch": 0.16015625, "grad_norm": 1.6589659452438354, "learning_rate": 9.978949230920472e-05, "loss": 0.7222372889518738, "memory/device_reserved (GiB)": 39.91, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 2.05903, "step": 41, "tokens/total": 1388768, "tokens/train_per_sec_per_gpu": 57.8, "tokens/trainable": 42401} +{"epoch": 0.1640625, "grad_norm": 2.322587728500366, "learning_rate": 9.976051477852141e-05, "loss": 0.7556037902832031, "memory/device_reserved (GiB)": 39.91, "memory/max_active (GiB)": 37.91, "memory/max_allocated (GiB)": 37.91, "ppl": 2.1289, "step": 42, "tokens/total": 1422448, "tokens/train_per_sec_per_gpu": 62.17, "tokens/trainable": 43407} +{"epoch": 0.16796875, "grad_norm": 1.6973927021026611, "learning_rate": 9.972967458011312e-05, "loss": 0.700096845626831, "memory/device_reserved (GiB)": 39.91, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 2.01395, "step": 43, "tokens/total": 1457872, "tokens/train_per_sec_per_gpu": 60.35, "tokens/trainable": 44487} +{"epoch": 0.171875, "grad_norm": 1.6395542621612549, "learning_rate": 9.96969729973664e-05, "loss": 0.6765084266662598, "memory/device_reserved (GiB)": 41.24, "memory/max_active (GiB)": 38.93, "memory/max_allocated (GiB)": 38.93, "ppl": 1.967, "step": 44, "tokens/total": 1496368, "tokens/train_per_sec_per_gpu": 56.18, "tokens/trainable": 45546} +{"epoch": 0.17578125, "grad_norm": 1.6453704833984375, "learning_rate": 9.966241139112754e-05, "loss": 0.7055221796035767, "memory/device_reserved (GiB)": 41.24, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 2.0249, "step": 45, "tokens/total": 1530016, "tokens/train_per_sec_per_gpu": 61.8, "tokens/trainable": 46582} +{"epoch": 0.1796875, "grad_norm": 2.707552909851074, "learning_rate": 9.96259911996461e-05, "loss": 0.6957885026931763, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 38.63, "memory/max_allocated (GiB)": 38.63, "ppl": 2.00529, "step": 46, "tokens/total": 1564096, "tokens/train_per_sec_per_gpu": 56.67, "tokens/trainable": 47617} +{"epoch": 0.18359375, "grad_norm": 1.4177374839782715, "learning_rate": 9.958771393851491e-05, "loss": 0.70391845703125, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 36.71, "memory/max_allocated (GiB)": 36.71, "ppl": 2.02166, "step": 47, "tokens/total": 1595808, "tokens/train_per_sec_per_gpu": 57.72, "tokens/trainable": 48622} +{"epoch": 0.1875, "grad_norm": 3.525878667831421, "learning_rate": 9.954758120060702e-05, "loss": 0.6693397164344788, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.95295, "step": 48, "tokens/total": 1632160, "tokens/train_per_sec_per_gpu": 51.96, "tokens/trainable": 49584} +{"epoch": 0.19140625, "grad_norm": 1.3362294435501099, "learning_rate": 9.950559465600948e-05, "loss": 0.6277520656585693, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 37.61, "memory/max_allocated (GiB)": 37.61, "ppl": 1.87339, "step": 49, "tokens/total": 1666208, "tokens/train_per_sec_per_gpu": 60.7, "tokens/trainable": 50631} +{"epoch": 0.1953125, "grad_norm": 1.75969398021698, "learning_rate": 9.946175605195379e-05, "loss": 0.5578070282936096, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.74684, "step": 50, "tokens/total": 1706304, "tokens/train_per_sec_per_gpu": 49.12, "tokens/trainable": 51736} +{"epoch": 0.19921875, "grad_norm": 1.346971035003662, "learning_rate": 9.941606721274322e-05, "loss": 0.6092395782470703, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.83903, "step": 51, "tokens/total": 1740656, "tokens/train_per_sec_per_gpu": 51.49, "tokens/trainable": 52749} +{"epoch": 0.203125, "grad_norm": 2.10258412361145, "learning_rate": 9.936853003967685e-05, "loss": 0.6700619459152222, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 1.95436, "step": 52, "tokens/total": 1777808, "tokens/train_per_sec_per_gpu": 52.1, "tokens/trainable": 53754} +{"epoch": 0.20703125, "grad_norm": 1.7077522277832031, "learning_rate": 9.93191465109705e-05, "loss": 0.6781176328659058, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.97017, "step": 53, "tokens/total": 1814480, "tokens/train_per_sec_per_gpu": 54.73, "tokens/trainable": 54776} +{"epoch": 0.2109375, "grad_norm": 1.6173778772354126, "learning_rate": 9.926791868167438e-05, "loss": 0.6441296339035034, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.90433, "step": 54, "tokens/total": 1850928, "tokens/train_per_sec_per_gpu": 52.01, "tokens/trainable": 55747} +{"epoch": 0.21484375, "grad_norm": 2.8566598892211914, "learning_rate": 9.921484868358753e-05, "loss": 0.6827815771102905, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 38.72, "memory/max_allocated (GiB)": 38.72, "ppl": 1.97938, "step": 55, "tokens/total": 1886752, "tokens/train_per_sec_per_gpu": 51.55, "tokens/trainable": 56769} +{"epoch": 0.21875, "grad_norm": 1.408849835395813, "learning_rate": 9.915993872516924e-05, "loss": 0.6222437620162964, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.8631, "step": 56, "tokens/total": 1922352, "tokens/train_per_sec_per_gpu": 53.16, "tokens/trainable": 57775} +{"epoch": 0.22265625, "grad_norm": 1.546510100364685, "learning_rate": 9.9103191091447e-05, "loss": 0.5896403193473816, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 36.88, "memory/max_allocated (GiB)": 36.88, "ppl": 1.80334, "step": 57, "tokens/total": 1954080, "tokens/train_per_sec_per_gpu": 57.84, "tokens/trainable": 58737} +{"epoch": 0.2265625, "grad_norm": 2.252087116241455, "learning_rate": 9.904460814392147e-05, "loss": 0.5667603015899658, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.76255, "step": 58, "tokens/total": 1993856, "tokens/train_per_sec_per_gpu": 49.67, "tokens/trainable": 59776} +{"epoch": 0.23046875, "grad_norm": 1.344219446182251, "learning_rate": 9.898419232046825e-05, "loss": 0.5636053085327148, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.757, "step": 59, "tokens/total": 2030416, "tokens/train_per_sec_per_gpu": 56.36, "tokens/trainable": 60861} +{"epoch": 0.234375, "grad_norm": 1.6406148672103882, "learning_rate": 9.892194613523633e-05, "loss": 0.5593582391738892, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.74955, "step": 60, "tokens/total": 2066464, "tokens/train_per_sec_per_gpu": 57.51, "tokens/trainable": 61976} +{"epoch": 0.23828125, "grad_norm": 1.674856185913086, "learning_rate": 9.885787217854357e-05, "loss": 0.545048713684082, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 38.18, "memory/max_allocated (GiB)": 38.18, "ppl": 1.72469, "step": 61, "tokens/total": 2100640, "tokens/train_per_sec_per_gpu": 56.98, "tokens/trainable": 62979} +{"epoch": 0.2421875, "grad_norm": 1.497793197631836, "learning_rate": 9.879197311676887e-05, "loss": 0.5773724317550659, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.78135, "step": 62, "tokens/total": 2136800, "tokens/train_per_sec_per_gpu": 56.34, "tokens/trainable": 64021} +{"epoch": 0.24609375, "grad_norm": 1.5261818170547485, "learning_rate": 9.872425169224113e-05, "loss": 0.5277473330497742, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 1.69511, "step": 63, "tokens/total": 2170496, "tokens/train_per_sec_per_gpu": 69.83, "tokens/trainable": 65191} +{"epoch": 0.25, "grad_norm": 1.5565687417984009, "learning_rate": 9.865471072312528e-05, "loss": 0.5471475124359131, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.72832, "step": 64, "tokens/total": 2207040, "tokens/train_per_sec_per_gpu": 60.74, "tokens/trainable": 66273} +{"epoch": 0.25390625, "grad_norm": 1.3780388832092285, "learning_rate": 9.858335310330492e-05, "loss": 0.5136044025421143, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 37.09, "memory/max_allocated (GiB)": 37.09, "ppl": 1.6713, "step": 65, "tokens/total": 2239632, "tokens/train_per_sec_per_gpu": 63.28, "tokens/trainable": 67352} +{"epoch": 0.2578125, "grad_norm": 2.9302423000335693, "learning_rate": 9.851018180226185e-05, "loss": 0.49950355291366577, "memory/device_reserved (GiB)": 41.42, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.6479, "step": 66, "tokens/total": 2278304, "tokens/train_per_sec_per_gpu": 54.03, "tokens/trainable": 68422} +{"epoch": 0.26171875, "grad_norm": 2.6525559425354004, "learning_rate": 9.843519986495259e-05, "loss": 0.533122181892395, "memory/device_reserved (GiB)": 41.42, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.70424, "step": 67, "tokens/total": 2313248, "tokens/train_per_sec_per_gpu": 57.6, "tokens/trainable": 69453} +{"epoch": 0.265625, "grad_norm": 1.2852532863616943, "learning_rate": 9.835841041168162e-05, "loss": 0.439967542886734, "memory/device_reserved (GiB)": 41.42, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.55266, "step": 68, "tokens/total": 2350656, "tokens/train_per_sec_per_gpu": 59.13, "tokens/trainable": 70545} +{"epoch": 0.26953125, "grad_norm": 2.9068124294281006, "learning_rate": 9.82798166379715e-05, "loss": 0.44401997327804565, "memory/device_reserved (GiB)": 41.42, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.55896, "step": 69, "tokens/total": 2383856, "tokens/train_per_sec_per_gpu": 60.52, "tokens/trainable": 71620} +{"epoch": 0.2734375, "grad_norm": 1.4565911293029785, "learning_rate": 9.819942181443002e-05, "loss": 0.5400739312171936, "memory/device_reserved (GiB)": 41.42, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.71613, "step": 70, "tokens/total": 2421120, "tokens/train_per_sec_per_gpu": 56.09, "tokens/trainable": 72665} +{"epoch": 0.27734375, "grad_norm": 1.6793725490570068, "learning_rate": 9.811722928661392e-05, "loss": 0.5679148435592651, "memory/device_reserved (GiB)": 41.43, "memory/max_active (GiB)": 35.8, "memory/max_allocated (GiB)": 35.8, "ppl": 1.76458, "step": 71, "tokens/total": 2449904, "tokens/train_per_sec_per_gpu": 71.91, "tokens/trainable": 73675} +{"epoch": 0.28125, "grad_norm": 1.346625804901123, "learning_rate": 9.803324247488975e-05, "loss": 0.4893566071987152, "memory/device_reserved (GiB)": 41.43, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.63127, "step": 72, "tokens/total": 2485936, "tokens/train_per_sec_per_gpu": 57.47, "tokens/trainable": 74745} +{"epoch": 0.28515625, "grad_norm": 2.2133049964904785, "learning_rate": 9.794746487429161e-05, "loss": 0.4821977913379669, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.04, "memory/max_allocated (GiB)": 37.04, "ppl": 1.61963, "step": 73, "tokens/total": 2519024, "tokens/train_per_sec_per_gpu": 67.37, "tokens/trainable": 75784} +{"epoch": 0.2890625, "grad_norm": 2.5727646350860596, "learning_rate": 9.785990005437554e-05, "loss": 0.5160143375396729, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.81, "memory/max_allocated (GiB)": 37.81, "ppl": 1.67534, "step": 74, "tokens/total": 2554864, "tokens/train_per_sec_per_gpu": 57.71, "tokens/trainable": 76837} +{"epoch": 0.29296875, "grad_norm": 1.7409170866012573, "learning_rate": 9.777055165907117e-05, "loss": 0.506931483745575, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.66019, "step": 75, "tokens/total": 2588048, "tokens/train_per_sec_per_gpu": 66.3, "tokens/trainable": 77868} +{"epoch": 0.296875, "grad_norm": 1.77660071849823, "learning_rate": 9.767942340652993e-05, "loss": 0.5589534044265747, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.05, "memory/max_allocated (GiB)": 38.05, "ppl": 1.74884, "step": 76, "tokens/total": 2622784, "tokens/train_per_sec_per_gpu": 54.98, "tokens/trainable": 78887} +{"epoch": 0.30078125, "grad_norm": 1.5510565042495728, "learning_rate": 9.758651908897035e-05, "loss": 0.49966442584991455, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.64817, "step": 77, "tokens/total": 2656416, "tokens/train_per_sec_per_gpu": 61.94, "tokens/trainable": 79948} +{"epoch": 0.3046875, "grad_norm": 1.5637056827545166, "learning_rate": 9.749184257252033e-05, "loss": 0.5442173480987549, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.72326, "step": 78, "tokens/total": 2690672, "tokens/train_per_sec_per_gpu": 61.64, "tokens/trainable": 80949} +{"epoch": 0.30859375, "grad_norm": 1.706343650817871, "learning_rate": 9.739539779705614e-05, "loss": 0.4489375650882721, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.56665, "step": 79, "tokens/total": 2729200, "tokens/train_per_sec_per_gpu": 58.26, "tokens/trainable": 82146} +{"epoch": 0.3125, "grad_norm": 1.4138712882995605, "learning_rate": 9.729718877603861e-05, "loss": 0.4589545726776123, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.58242, "step": 80, "tokens/total": 2762784, "tokens/train_per_sec_per_gpu": 65.78, "tokens/trainable": 83180} +{"epoch": 0.31640625, "grad_norm": 1.8383618593215942, "learning_rate": 9.719721959634592e-05, "loss": 0.44613879919052124, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.56227, "step": 81, "tokens/total": 2798304, "tokens/train_per_sec_per_gpu": 61.07, "tokens/trainable": 84229} +{"epoch": 0.3203125, "grad_norm": 1.583814024925232, "learning_rate": 9.709549441810375e-05, "loss": 0.4715668261051178, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.6025, "step": 82, "tokens/total": 2835712, "tokens/train_per_sec_per_gpu": 60.28, "tokens/trainable": 85309} +{"epoch": 0.32421875, "grad_norm": 1.5944191217422485, "learning_rate": 9.699201747451195e-05, "loss": 0.47723761200904846, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.54, "memory/max_allocated (GiB)": 36.54, "ppl": 1.61162, "step": 83, "tokens/total": 2868224, "tokens/train_per_sec_per_gpu": 58.41, "tokens/trainable": 86362} +{"epoch": 0.328125, "grad_norm": 1.3031115531921387, "learning_rate": 9.688679307166854e-05, "loss": 0.5183576345443726, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.67927, "step": 84, "tokens/total": 2900736, "tokens/train_per_sec_per_gpu": 63.32, "tokens/trainable": 87402} +{"epoch": 0.33203125, "grad_norm": 1.6453394889831543, "learning_rate": 9.677982558839042e-05, "loss": 0.5219653248786926, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.82, "memory/max_allocated (GiB)": 36.82, "ppl": 1.68534, "step": 85, "tokens/total": 2933904, "tokens/train_per_sec_per_gpu": 56.43, "tokens/trainable": 88457} +{"epoch": 0.3359375, "grad_norm": 1.8538672924041748, "learning_rate": 9.66711194760312e-05, "loss": 0.41846150159835815, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 39.04, "memory/max_allocated (GiB)": 39.04, "ppl": 1.51962, "step": 86, "tokens/total": 2971344, "tokens/train_per_sec_per_gpu": 55.65, "tokens/trainable": 89586} +{"epoch": 0.33984375, "grad_norm": 1.3325116634368896, "learning_rate": 9.656067925829593e-05, "loss": 0.47300755977630615, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.60481, "step": 87, "tokens/total": 3001936, "tokens/train_per_sec_per_gpu": 71.11, "tokens/trainable": 90620} +{"epoch": 0.34375, "grad_norm": 1.140122652053833, "learning_rate": 9.644850953105288e-05, "loss": 0.47994908690452576, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 35.88, "memory/max_allocated (GiB)": 35.88, "ppl": 1.61599, "step": 88, "tokens/total": 3032944, "tokens/train_per_sec_per_gpu": 68.0, "tokens/trainable": 91688} +{"epoch": 0.34765625, "grad_norm": 1.3107918500900269, "learning_rate": 9.633461496214225e-05, "loss": 0.4095633625984192, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.50616, "step": 89, "tokens/total": 3065584, "tokens/train_per_sec_per_gpu": 62.16, "tokens/trainable": 92745} +{"epoch": 0.3515625, "grad_norm": 2.148207426071167, "learning_rate": 9.621900029118195e-05, "loss": 0.4429558515548706, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.1, "memory/max_allocated (GiB)": 36.1, "ppl": 1.5573, "step": 90, "tokens/total": 3095248, "tokens/train_per_sec_per_gpu": 65.96, "tokens/trainable": 93764} +{"epoch": 0.35546875, "grad_norm": 1.5159902572631836, "learning_rate": 9.610167032937036e-05, "loss": 0.47582992911338806, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.60935, "step": 91, "tokens/total": 3133456, "tokens/train_per_sec_per_gpu": 54.28, "tokens/trainable": 94817} +{"epoch": 0.359375, "grad_norm": 1.3461477756500244, "learning_rate": 9.598262995928611e-05, "loss": 0.38525551557540894, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.46999, "step": 92, "tokens/total": 3164624, "tokens/train_per_sec_per_gpu": 68.32, "tokens/trainable": 95867} +{"epoch": 0.36328125, "grad_norm": 1.3231273889541626, "learning_rate": 9.586188413468492e-05, "loss": 0.43361395597457886, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.8, "memory/max_allocated (GiB)": 38.8, "ppl": 1.54282, "step": 93, "tokens/total": 3203008, "tokens/train_per_sec_per_gpu": 49.42, "tokens/trainable": 96920} +{"epoch": 0.3671875, "grad_norm": 1.1984704732894897, "learning_rate": 9.57394378802934e-05, "loss": 0.4306532144546509, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.53826, "step": 94, "tokens/total": 3236448, "tokens/train_per_sec_per_gpu": 60.65, "tokens/trainable": 97966} +{"epoch": 0.37109375, "grad_norm": 1.572317361831665, "learning_rate": 9.56152962916e-05, "loss": 0.4418870806694031, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.55564, "step": 95, "tokens/total": 3267104, "tokens/train_per_sec_per_gpu": 66.15, "tokens/trainable": 98951} +{"epoch": 0.375, "grad_norm": 1.2415822744369507, "learning_rate": 9.548946453464296e-05, "loss": 0.36717694997787476, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 1.44365, "step": 96, "tokens/total": 3301440, "tokens/train_per_sec_per_gpu": 62.62, "tokens/trainable": 100056} +{"epoch": 0.37890625, "grad_norm": 1.9161254167556763, "learning_rate": 9.53619478457953e-05, "loss": 0.4768022894859314, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.61091, "step": 97, "tokens/total": 3336112, "tokens/train_per_sec_per_gpu": 52.39, "tokens/trainable": 101046} +{"epoch": 0.3828125, "grad_norm": 1.6869609355926514, "learning_rate": 9.523275153154695e-05, "loss": 0.4807557463645935, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.6173, "step": 98, "tokens/total": 3368624, "tokens/train_per_sec_per_gpu": 57.77, "tokens/trainable": 102006} +{"epoch": 0.38671875, "grad_norm": 1.3261950016021729, "learning_rate": 9.51018809682839e-05, "loss": 0.4249259829521179, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.52948, "step": 99, "tokens/total": 3406448, "tokens/train_per_sec_per_gpu": 52.86, "tokens/trainable": 103023} +{"epoch": 0.390625, "grad_norm": 16.814516067504883, "learning_rate": 9.49693416020645e-05, "loss": 0.4550424814224243, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.51, "memory/max_allocated (GiB)": 38.51, "ppl": 1.57624, "step": 100, "tokens/total": 3444240, "tokens/train_per_sec_per_gpu": 54.76, "tokens/trainable": 104043} +{"epoch": 0.39453125, "grad_norm": 1.562569260597229, "learning_rate": 9.483513894839276e-05, "loss": 0.4550463855266571, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.57625, "step": 101, "tokens/total": 3477680, "tokens/train_per_sec_per_gpu": 67.83, "tokens/trainable": 105146} +{"epoch": 0.3984375, "grad_norm": 2.1805338859558105, "learning_rate": 9.469927859198888e-05, "loss": 0.4243153929710388, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.52854, "step": 102, "tokens/total": 3513616, "tokens/train_per_sec_per_gpu": 58.48, "tokens/trainable": 106146} +{"epoch": 0.40234375, "grad_norm": 2.1520562171936035, "learning_rate": 9.456176618655689e-05, "loss": 0.4239398241043091, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.51, "memory/max_allocated (GiB)": 36.51, "ppl": 1.52797, "step": 103, "tokens/total": 3545584, "tokens/train_per_sec_per_gpu": 60.34, "tokens/trainable": 107142} +{"epoch": 0.40625, "grad_norm": 1.6154690980911255, "learning_rate": 9.442260745454927e-05, "loss": 0.4333976209163666, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.54249, "step": 104, "tokens/total": 3580768, "tokens/train_per_sec_per_gpu": 58.97, "tokens/trainable": 108213} +{"epoch": 0.41015625, "grad_norm": 2.8413844108581543, "learning_rate": 9.428180818692884e-05, "loss": 0.4116266965866089, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.50927, "step": 105, "tokens/total": 3617008, "tokens/train_per_sec_per_gpu": 52.4, "tokens/trainable": 109273} +{"epoch": 0.4140625, "grad_norm": 1.4118592739105225, "learning_rate": 9.413937424292791e-05, "loss": 0.42775291204452515, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.53381, "step": 106, "tokens/total": 3655936, "tokens/train_per_sec_per_gpu": 48.81, "tokens/trainable": 110278} +{"epoch": 0.41796875, "grad_norm": 1.4056811332702637, "learning_rate": 9.399531154980424e-05, "loss": 0.36598193645477295, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.44193, "step": 107, "tokens/total": 3692736, "tokens/train_per_sec_per_gpu": 51.49, "tokens/trainable": 111283} +{"epoch": 0.421875, "grad_norm": 6.586326599121094, "learning_rate": 9.384962610259455e-05, "loss": 0.39001062512397766, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.477, "step": 108, "tokens/total": 3727248, "tokens/train_per_sec_per_gpu": 54.67, "tokens/trainable": 112276} +{"epoch": 0.42578125, "grad_norm": 1.762594223022461, "learning_rate": 9.370232396386494e-05, "loss": 0.3996029794216156, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.49123, "step": 109, "tokens/total": 3760464, "tokens/train_per_sec_per_gpu": 60.63, "tokens/trainable": 113280} +{"epoch": 0.4296875, "grad_norm": 1.5437501668930054, "learning_rate": 9.355341126345868e-05, "loss": 0.3589959442615509, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.43189, "step": 110, "tokens/total": 3795952, "tokens/train_per_sec_per_gpu": 59.41, "tokens/trainable": 114365} +{"epoch": 0.43359375, "grad_norm": 1.1937819719314575, "learning_rate": 9.340289419824107e-05, "loss": 0.3723466992378235, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.74, "memory/max_allocated (GiB)": 36.74, "ppl": 1.45114, "step": 111, "tokens/total": 3828144, "tokens/train_per_sec_per_gpu": 68.84, "tokens/trainable": 115396} +{"epoch": 0.4375, "grad_norm": 1.6934788227081299, "learning_rate": 9.325077903184159e-05, "loss": 0.44264349341392517, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.55682, "step": 112, "tokens/total": 3862992, "tokens/train_per_sec_per_gpu": 56.83, "tokens/trainable": 116459} +{"epoch": 0.44140625, "grad_norm": 1.2702844142913818, "learning_rate": 9.30970720943932e-05, "loss": 0.3468776047229767, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.43, "memory/max_allocated (GiB)": 38.43, "ppl": 1.41464, "step": 113, "tokens/total": 3898416, "tokens/train_per_sec_per_gpu": 56.25, "tokens/trainable": 117528} +{"epoch": 0.4453125, "grad_norm": 1.5022242069244385, "learning_rate": 9.2941779782269e-05, "loss": 0.3662210702896118, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.44227, "step": 114, "tokens/total": 3932176, "tokens/train_per_sec_per_gpu": 60.94, "tokens/trainable": 118554} +{"epoch": 0.44921875, "grad_norm": 1.354864239692688, "learning_rate": 9.278490855781596e-05, "loss": 0.38940101861953735, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.4761, "step": 115, "tokens/total": 3964384, "tokens/train_per_sec_per_gpu": 68.26, "tokens/trainable": 119571} +{"epoch": 0.453125, "grad_norm": 1.334699034690857, "learning_rate": 9.262646494908604e-05, "loss": 0.42639416456222534, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 39.14, "memory/max_allocated (GiB)": 39.14, "ppl": 1.53172, "step": 116, "tokens/total": 4002960, "tokens/train_per_sec_per_gpu": 49.66, "tokens/trainable": 120560} +{"epoch": 0.45703125, "grad_norm": 1.445525884628296, "learning_rate": 9.246645554956457e-05, "loss": 0.41632694005966187, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.51638, "step": 117, "tokens/total": 4037552, "tokens/train_per_sec_per_gpu": 59.09, "tokens/trainable": 121586} +{"epoch": 0.4609375, "grad_norm": 5.353682994842529, "learning_rate": 9.230488701789578e-05, "loss": 0.42701929807662964, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.7, "memory/max_allocated (GiB)": 38.7, "ppl": 1.53268, "step": 118, "tokens/total": 4075072, "tokens/train_per_sec_per_gpu": 56.0, "tokens/trainable": 122669} +{"epoch": 0.46484375, "grad_norm": 1.5677263736724854, "learning_rate": 9.214176607760577e-05, "loss": 0.3623427748680115, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.43669, "step": 119, "tokens/total": 4113744, "tokens/train_per_sec_per_gpu": 58.24, "tokens/trainable": 123800} +{"epoch": 0.46875, "grad_norm": 1.3265981674194336, "learning_rate": 9.197709951682268e-05, "loss": 0.3726781904697418, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.45162, "step": 120, "tokens/total": 4146032, "tokens/train_per_sec_per_gpu": 64.79, "tokens/trainable": 124850} +{"epoch": 0.47265625, "grad_norm": 2.661210298538208, "learning_rate": 9.181089418799428e-05, "loss": 0.40446826815605164, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 39.39, "memory/max_allocated (GiB)": 39.39, "ppl": 1.49851, "step": 121, "tokens/total": 4182832, "tokens/train_per_sec_per_gpu": 54.8, "tokens/trainable": 125889} +{"epoch": 0.4765625, "grad_norm": 0.9724057912826538, "learning_rate": 9.164315700760271e-05, "loss": 0.326424777507782, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.386, "step": 122, "tokens/total": 4218640, "tokens/train_per_sec_per_gpu": 58.15, "tokens/trainable": 126933} +{"epoch": 0.48046875, "grad_norm": 1.322283387184143, "learning_rate": 9.147389495587671e-05, "loss": 0.4235757887363434, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 1.52741, "step": 123, "tokens/total": 4249792, "tokens/train_per_sec_per_gpu": 68.8, "tokens/trainable": 127964} +{"epoch": 0.484375, "grad_norm": 2.1650471687316895, "learning_rate": 9.130311507650116e-05, "loss": 0.32066601514816284, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 1.37805, "step": 124, "tokens/total": 4282464, "tokens/train_per_sec_per_gpu": 61.38, "tokens/trainable": 129019} +{"epoch": 0.48828125, "grad_norm": 1.2954950332641602, "learning_rate": 9.113082447632394e-05, "loss": 0.3564980626106262, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.42832, "step": 125, "tokens/total": 4314608, "tokens/train_per_sec_per_gpu": 79.26, "tokens/trainable": 130152} +{"epoch": 0.4921875, "grad_norm": 1.4376914501190186, "learning_rate": 9.09570303250602e-05, "loss": 0.3973102867603302, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.47, "memory/max_allocated (GiB)": 36.47, "ppl": 1.48782, "step": 126, "tokens/total": 4346416, "tokens/train_per_sec_per_gpu": 63.0, "tokens/trainable": 131215} +{"epoch": 0.49609375, "grad_norm": 1.2913818359375, "learning_rate": 9.078173985499394e-05, "loss": 0.36971428990364075, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 35.79, "memory/max_allocated (GiB)": 35.79, "ppl": 1.44732, "step": 127, "tokens/total": 4376400, "tokens/train_per_sec_per_gpu": 65.22, "tokens/trainable": 132219} +{"epoch": 0.5, "grad_norm": 1.3962641954421997, "learning_rate": 9.060496036067713e-05, "loss": 0.3916698694229126, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.72, "memory/max_allocated (GiB)": 38.72, "ppl": 1.47945, "step": 128, "tokens/total": 4413824, "tokens/train_per_sec_per_gpu": 47.89, "tokens/trainable": 133233} +{"epoch": 0.50390625, "grad_norm": 1.2531274557113647, "learning_rate": 9.042669919862615e-05, "loss": 0.377653032541275, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.82, "memory/max_allocated (GiB)": 36.82, "ppl": 1.45886, "step": 129, "tokens/total": 4445936, "tokens/train_per_sec_per_gpu": 67.32, "tokens/trainable": 134285} +{"epoch": 0.5078125, "grad_norm": 7.589662075042725, "learning_rate": 9.024696378701557e-05, "loss": 0.34523606300354004, "memory/device_reserved (GiB)": 41.42, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.41232, "step": 130, "tokens/total": 4481488, "tokens/train_per_sec_per_gpu": 58.24, "tokens/trainable": 135324} +{"epoch": 0.51171875, "grad_norm": 1.3723440170288086, "learning_rate": 9.006576160536948e-05, "loss": 0.3728221654891968, "memory/device_reserved (GiB)": 41.42, "memory/max_active (GiB)": 37.43, "memory/max_allocated (GiB)": 37.43, "ppl": 1.45183, "step": 131, "tokens/total": 4516048, "tokens/train_per_sec_per_gpu": 55.7, "tokens/trainable": 136314} +{"epoch": 0.515625, "grad_norm": 1.5604493618011475, "learning_rate": 8.988310019425035e-05, "loss": 0.3741750121116638, "memory/device_reserved (GiB)": 41.71, "memory/max_active (GiB)": 39.7, "memory/max_allocated (GiB)": 39.7, "ppl": 1.45379, "step": 132, "tokens/total": 4553904, "tokens/train_per_sec_per_gpu": 51.25, "tokens/trainable": 137324} +{"epoch": 0.51953125, "grad_norm": 1.8670934438705444, "learning_rate": 8.969898715494506e-05, "loss": 0.35179340839385986, "memory/device_reserved (GiB)": 41.73, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.42161, "step": 133, "tokens/total": 4590736, "tokens/train_per_sec_per_gpu": 60.74, "tokens/trainable": 138414} +{"epoch": 0.5234375, "grad_norm": 1.518919825553894, "learning_rate": 8.951343014914869e-05, "loss": 0.33432987332344055, "memory/device_reserved (GiB)": 41.73, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.397, "step": 134, "tokens/total": 4627392, "tokens/train_per_sec_per_gpu": 55.23, "tokens/trainable": 139490} +{"epoch": 0.52734375, "grad_norm": 1.6055004596710205, "learning_rate": 8.932643689864568e-05, "loss": 0.3166736662387848, "memory/device_reserved (GiB)": 41.73, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.37255, "step": 135, "tokens/total": 4663968, "tokens/train_per_sec_per_gpu": 61.79, "tokens/trainable": 140607} +{"epoch": 0.53125, "grad_norm": 1.4071214199066162, "learning_rate": 8.913801518498845e-05, "loss": 0.32677266001701355, "memory/device_reserved (GiB)": 41.73, "memory/max_active (GiB)": 39.68, "memory/max_allocated (GiB)": 39.68, "ppl": 1.38649, "step": 136, "tokens/total": 4703344, "tokens/train_per_sec_per_gpu": 48.3, "tokens/trainable": 141719} +{"epoch": 0.53515625, "grad_norm": 1.6091336011886597, "learning_rate": 8.894817284917364e-05, "loss": 0.37193092703819275, "memory/device_reserved (GiB)": 41.73, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.45053, "step": 137, "tokens/total": 4740288, "tokens/train_per_sec_per_gpu": 49.95, "tokens/trainable": 142753} +{"epoch": 0.5390625, "grad_norm": 1.5776983499526978, "learning_rate": 8.875691779131569e-05, "loss": 0.3220611810684204, "memory/device_reserved (GiB)": 41.9, "memory/max_active (GiB)": 39.78, "memory/max_allocated (GiB)": 39.78, "ppl": 1.37997, "step": 138, "tokens/total": 4778224, "tokens/train_per_sec_per_gpu": 54.15, "tokens/trainable": 143828} +{"epoch": 0.54296875, "grad_norm": 1.1818618774414062, "learning_rate": 8.856425797031829e-05, "loss": 0.3042747974395752, "memory/device_reserved (GiB)": 42.22, "memory/max_active (GiB)": 39.58, "memory/max_allocated (GiB)": 39.58, "ppl": 1.35564, "step": 139, "tokens/total": 4814256, "tokens/train_per_sec_per_gpu": 63.33, "tokens/trainable": 144956} +{"epoch": 0.546875, "grad_norm": 1.3577653169631958, "learning_rate": 8.837020140354295e-05, "loss": 0.34403878450393677, "memory/device_reserved (GiB)": 42.22, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.41063, "step": 140, "tokens/total": 4851072, "tokens/train_per_sec_per_gpu": 55.05, "tokens/trainable": 145963} +{"epoch": 0.55078125, "grad_norm": 1.7498918771743774, "learning_rate": 8.817475616647554e-05, "loss": 0.32834911346435547, "memory/device_reserved (GiB)": 42.22, "memory/max_active (GiB)": 38.17, "memory/max_allocated (GiB)": 38.17, "ppl": 1.38867, "step": 141, "tokens/total": 4885504, "tokens/train_per_sec_per_gpu": 59.74, "tokens/trainable": 147019} +{"epoch": 0.5546875, "grad_norm": 1.5986809730529785, "learning_rate": 8.797793039239017e-05, "loss": 0.313961386680603, "memory/device_reserved (GiB)": 42.22, "memory/max_active (GiB)": 37.83, "memory/max_allocated (GiB)": 37.83, "ppl": 1.36884, "step": 142, "tokens/total": 4920528, "tokens/train_per_sec_per_gpu": 58.12, "tokens/trainable": 148069} +{"epoch": 0.55859375, "grad_norm": 1.1366186141967773, "learning_rate": 8.777973227201069e-05, "loss": 0.30730104446411133, "memory/device_reserved (GiB)": 43.76, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.35975, "step": 143, "tokens/total": 4957440, "tokens/train_per_sec_per_gpu": 60.02, "tokens/trainable": 149132} +{"epoch": 0.5625, "grad_norm": 1.5854301452636719, "learning_rate": 8.758017005316988e-05, "loss": 0.3642883896827698, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.43949, "step": 144, "tokens/total": 4990752, "tokens/train_per_sec_per_gpu": 72.38, "tokens/trainable": 150174} +{"epoch": 0.56640625, "grad_norm": 1.6530125141143799, "learning_rate": 8.737925204046629e-05, "loss": 0.3609762191772461, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.43473, "step": 145, "tokens/total": 5025472, "tokens/train_per_sec_per_gpu": 54.43, "tokens/trainable": 151234} +{"epoch": 0.5703125, "grad_norm": 1.7896239757537842, "learning_rate": 8.717698659491851e-05, "loss": 0.312622606754303, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 38.91, "memory/max_allocated (GiB)": 38.91, "ppl": 1.36701, "step": 146, "tokens/total": 5062304, "tokens/train_per_sec_per_gpu": 56.73, "tokens/trainable": 152338} +{"epoch": 0.57421875, "grad_norm": 1.2373629808425903, "learning_rate": 8.697338213361735e-05, "loss": 0.30667421221733093, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 36.45, "memory/max_allocated (GiB)": 36.45, "ppl": 1.3589, "step": 147, "tokens/total": 5093280, "tokens/train_per_sec_per_gpu": 67.68, "tokens/trainable": 153425} +{"epoch": 0.578125, "grad_norm": 1.2765525579452515, "learning_rate": 8.676844712937552e-05, "loss": 0.2962561249732971, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.34481, "step": 148, "tokens/total": 5133280, "tokens/train_per_sec_per_gpu": 48.97, "tokens/trainable": 154477} +{"epoch": 0.58203125, "grad_norm": 1.406378984451294, "learning_rate": 8.656219011037509e-05, "loss": 0.3415566682815552, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.40714, "step": 149, "tokens/total": 5170576, "tokens/train_per_sec_per_gpu": 54.21, "tokens/trainable": 155549} +{"epoch": 0.5859375, "grad_norm": 2.084289789199829, "learning_rate": 8.63546196598125e-05, "loss": 0.3330311179161072, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.39519, "step": 150, "tokens/total": 5208160, "tokens/train_per_sec_per_gpu": 55.51, "tokens/trainable": 156672} +{"epoch": 0.58984375, "grad_norm": 1.647312045097351, "learning_rate": 8.614574441554145e-05, "loss": 0.413108766078949, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 36.95, "memory/max_allocated (GiB)": 36.95, "ppl": 1.51151, "step": 151, "tokens/total": 5240784, "tokens/train_per_sec_per_gpu": 61.54, "tokens/trainable": 157633} +{"epoch": 0.59375, "grad_norm": 2.8148083686828613, "learning_rate": 8.593557306971349e-05, "loss": 0.303669273853302, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.35482, "step": 152, "tokens/total": 5274096, "tokens/train_per_sec_per_gpu": 58.24, "tokens/trainable": 158666} +{"epoch": 0.59765625, "grad_norm": 2.0160322189331055, "learning_rate": 8.572411436841618e-05, "loss": 0.35172533988952637, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 36.9, "memory/max_allocated (GiB)": 36.9, "ppl": 1.42152, "step": 153, "tokens/total": 5307840, "tokens/train_per_sec_per_gpu": 64.01, "tokens/trainable": 159736} +{"epoch": 0.6015625, "grad_norm": 2.070718765258789, "learning_rate": 8.551137711130922e-05, "loss": 0.3171842396259308, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.37326, "step": 154, "tokens/total": 5339616, "tokens/train_per_sec_per_gpu": 58.86, "tokens/trainable": 160754} +{"epoch": 0.60546875, "grad_norm": 1.3710837364196777, "learning_rate": 8.529737015125824e-05, "loss": 0.304875910282135, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 38.17, "memory/max_allocated (GiB)": 38.17, "ppl": 1.35646, "step": 155, "tokens/total": 5374720, "tokens/train_per_sec_per_gpu": 56.4, "tokens/trainable": 161773} +{"epoch": 0.609375, "grad_norm": 1.3977015018463135, "learning_rate": 8.508210239396639e-05, "loss": 0.3407841920852661, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 39.77, "memory/max_allocated (GiB)": 39.77, "ppl": 1.40605, "step": 156, "tokens/total": 5412608, "tokens/train_per_sec_per_gpu": 56.95, "tokens/trainable": 162790} +{"epoch": 0.61328125, "grad_norm": 4.261829853057861, "learning_rate": 8.486558279760375e-05, "loss": 0.34580761194229126, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 35.9, "memory/max_allocated (GiB)": 35.9, "ppl": 1.41313, "step": 157, "tokens/total": 5443664, "tokens/train_per_sec_per_gpu": 63.75, "tokens/trainable": 163797} +{"epoch": 0.6171875, "grad_norm": 2.060330867767334, "learning_rate": 8.464782037243449e-05, "loss": 0.3519318103790283, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.42181, "step": 158, "tokens/total": 5475808, "tokens/train_per_sec_per_gpu": 60.33, "tokens/trainable": 164786} +{"epoch": 0.62109375, "grad_norm": 1.2863072156906128, "learning_rate": 8.442882418044202e-05, "loss": 0.32246488332748413, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.38053, "step": 159, "tokens/total": 5508448, "tokens/train_per_sec_per_gpu": 63.3, "tokens/trainable": 165826} +{"epoch": 0.625, "grad_norm": 1.2217998504638672, "learning_rate": 8.420860333495179e-05, "loss": 0.30377745628356934, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.35497, "step": 160, "tokens/total": 5540976, "tokens/train_per_sec_per_gpu": 64.39, "tokens/trainable": 166811} +{"epoch": 0.62890625, "grad_norm": 1.7012282609939575, "learning_rate": 8.398716700025208e-05, "loss": 0.34093278646469116, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.40626, "step": 161, "tokens/total": 5574496, "tokens/train_per_sec_per_gpu": 59.68, "tokens/trainable": 167839} +{"epoch": 0.6328125, "grad_norm": 1.4347788095474243, "learning_rate": 8.376452439121266e-05, "loss": 0.34853124618530273, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.05, "memory/max_allocated (GiB)": 36.05, "ppl": 1.41698, "step": 162, "tokens/total": 5604544, "tokens/train_per_sec_per_gpu": 65.18, "tokens/trainable": 168809} +{"epoch": 0.63671875, "grad_norm": 1.6011096239089966, "learning_rate": 8.354068477290124e-05, "loss": 0.3520507514476776, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.42198, "step": 163, "tokens/total": 5638768, "tokens/train_per_sec_per_gpu": 60.95, "tokens/trainable": 169796} +{"epoch": 0.640625, "grad_norm": 1.527937412261963, "learning_rate": 8.331565746019807e-05, "loss": 0.2913765609264374, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.33827, "step": 164, "tokens/total": 5670960, "tokens/train_per_sec_per_gpu": 68.53, "tokens/trainable": 170835} +{"epoch": 0.64453125, "grad_norm": 1.7266051769256592, "learning_rate": 8.308945181740812e-05, "loss": 0.3132290840148926, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.36783, "step": 165, "tokens/total": 5705664, "tokens/train_per_sec_per_gpu": 61.61, "tokens/trainable": 171844} +{"epoch": 0.6484375, "grad_norm": 1.8224806785583496, "learning_rate": 8.286207725787153e-05, "loss": 0.27874767780303955, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.32147, "step": 166, "tokens/total": 5742672, "tokens/train_per_sec_per_gpu": 51.42, "tokens/trainable": 172871} +{"epoch": 0.65234375, "grad_norm": 1.9869247674942017, "learning_rate": 8.263354324357182e-05, "loss": 0.29187071323394775, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.12, "memory/max_allocated (GiB)": 39.12, "ppl": 1.33893, "step": 167, "tokens/total": 5777408, "tokens/train_per_sec_per_gpu": 58.0, "tokens/trainable": 173893} +{"epoch": 0.65625, "grad_norm": 1.9624184370040894, "learning_rate": 8.240385928474219e-05, "loss": 0.372650146484375, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.45158, "step": 168, "tokens/total": 5810224, "tokens/train_per_sec_per_gpu": 57.62, "tokens/trainable": 174853} +{"epoch": 0.66015625, "grad_norm": 1.5605357885360718, "learning_rate": 8.217303493946967e-05, "loss": 0.3227311968803406, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.38089, "step": 169, "tokens/total": 5844432, "tokens/train_per_sec_per_gpu": 57.64, "tokens/trainable": 175851} +{"epoch": 0.6640625, "grad_norm": 1.5367622375488281, "learning_rate": 8.194107981329746e-05, "loss": 0.2916010618209839, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.73, "memory/max_allocated (GiB)": 39.73, "ppl": 1.33857, "step": 170, "tokens/total": 5883008, "tokens/train_per_sec_per_gpu": 51.4, "tokens/trainable": 176872} +{"epoch": 0.66796875, "grad_norm": 1.732176423072815, "learning_rate": 8.170800355882518e-05, "loss": 0.28110557794570923, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.32459, "step": 171, "tokens/total": 5917440, "tokens/train_per_sec_per_gpu": 66.03, "tokens/trainable": 177911} +{"epoch": 0.671875, "grad_norm": 1.636569619178772, "learning_rate": 8.147381587530713e-05, "loss": 0.3267529010772705, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.39, "memory/max_allocated (GiB)": 37.39, "ppl": 1.38646, "step": 172, "tokens/total": 5951232, "tokens/train_per_sec_per_gpu": 61.81, "tokens/trainable": 178907} +{"epoch": 0.67578125, "grad_norm": 1.4596630334854126, "learning_rate": 8.123852650824877e-05, "loss": 0.3266218602657318, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.45, "memory/max_allocated (GiB)": 37.45, "ppl": 1.38628, "step": 173, "tokens/total": 5985056, "tokens/train_per_sec_per_gpu": 62.73, "tokens/trainable": 179980} +{"epoch": 0.6796875, "grad_norm": 1.1962103843688965, "learning_rate": 8.100214524900103e-05, "loss": 0.2961388826370239, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.34466, "step": 174, "tokens/total": 6020128, "tokens/train_per_sec_per_gpu": 51.83, "tokens/trainable": 180969} +{"epoch": 0.68359375, "grad_norm": 1.4597830772399902, "learning_rate": 8.076468193435301e-05, "loss": 0.3143559694290161, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.36938, "step": 175, "tokens/total": 6050976, "tokens/train_per_sec_per_gpu": 66.24, "tokens/trainable": 181917} +{"epoch": 0.6875, "grad_norm": 1.2869255542755127, "learning_rate": 8.052614644612253e-05, "loss": 0.2943495213985443, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.34225, "step": 176, "tokens/total": 6088816, "tokens/train_per_sec_per_gpu": 49.74, "tokens/trainable": 182967} +{"epoch": 0.69140625, "grad_norm": 1.6126924753189087, "learning_rate": 8.028654871074489e-05, "loss": 0.2596510648727417, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.29648, "step": 177, "tokens/total": 6123408, "tokens/train_per_sec_per_gpu": 58.76, "tokens/trainable": 184053} +{"epoch": 0.6953125, "grad_norm": 1.9778587818145752, "learning_rate": 8.004589869885986e-05, "loss": 0.3325478434562683, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.37, "memory/max_allocated (GiB)": 36.37, "ppl": 1.39452, "step": 178, "tokens/total": 6154880, "tokens/train_per_sec_per_gpu": 60.29, "tokens/trainable": 184997} +{"epoch": 0.69921875, "grad_norm": 1.4347673654556274, "learning_rate": 7.980420642489674e-05, "loss": 0.28535929322242737, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.33024, "step": 179, "tokens/total": 6193584, "tokens/train_per_sec_per_gpu": 53.43, "tokens/trainable": 186029} +{"epoch": 0.703125, "grad_norm": 1.2789400815963745, "learning_rate": 7.95614819466576e-05, "loss": 0.2695811986923218, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 1.30942, "step": 180, "tokens/total": 6227296, "tokens/train_per_sec_per_gpu": 58.74, "tokens/trainable": 187082} +{"epoch": 0.70703125, "grad_norm": 1.2141567468643188, "learning_rate": 7.931773536489872e-05, "loss": 0.3002616763114929, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.35021, "step": 181, "tokens/total": 6261888, "tokens/train_per_sec_per_gpu": 62.24, "tokens/trainable": 188129} +{"epoch": 0.7109375, "grad_norm": 1.5883986949920654, "learning_rate": 7.907297682291035e-05, "loss": 0.29487502574920654, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.34296, "step": 182, "tokens/total": 6296544, "tokens/train_per_sec_per_gpu": 60.46, "tokens/trainable": 189201} +{"epoch": 0.71484375, "grad_norm": 1.2852498292922974, "learning_rate": 7.882721650609442e-05, "loss": 0.2323596477508545, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.65, "memory/max_allocated (GiB)": 39.65, "ppl": 1.26157, "step": 183, "tokens/total": 6336960, "tokens/train_per_sec_per_gpu": 53.15, "tokens/trainable": 190379} +{"epoch": 0.71875, "grad_norm": 1.1848468780517578, "learning_rate": 7.85804646415409e-05, "loss": 0.2759571969509125, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.31779, "step": 184, "tokens/total": 6372192, "tokens/train_per_sec_per_gpu": 63.93, "tokens/trainable": 191465} +{"epoch": 0.72265625, "grad_norm": 1.4649080038070679, "learning_rate": 7.833273149760207e-05, "loss": 0.2814489006996155, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.32505, "step": 185, "tokens/total": 6403056, "tokens/train_per_sec_per_gpu": 60.13, "tokens/trainable": 192471} +{"epoch": 0.7265625, "grad_norm": 1.4767392873764038, "learning_rate": 7.808402738346527e-05, "loss": 0.25444287061691284, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.11, "memory/max_allocated (GiB)": 39.11, "ppl": 1.28974, "step": 186, "tokens/total": 6441840, "tokens/train_per_sec_per_gpu": 57.94, "tokens/trainable": 193603} +{"epoch": 0.73046875, "grad_norm": 1.1874874830245972, "learning_rate": 7.783436264872382e-05, "loss": 0.2672055959701538, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.30631, "step": 187, "tokens/total": 6478688, "tokens/train_per_sec_per_gpu": 55.85, "tokens/trainable": 194652} +{"epoch": 0.734375, "grad_norm": 1.6351977586746216, "learning_rate": 7.758374768294647e-05, "loss": 0.2701876163482666, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.16, "memory/max_allocated (GiB)": 38.16, "ppl": 1.31021, "step": 188, "tokens/total": 6514624, "tokens/train_per_sec_per_gpu": 60.54, "tokens/trainable": 195747} +{"epoch": 0.73828125, "grad_norm": 1.1512386798858643, "learning_rate": 7.733219291524489e-05, "loss": 0.25030913949012756, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.53, "memory/max_allocated (GiB)": 37.53, "ppl": 1.28442, "step": 189, "tokens/total": 6548048, "tokens/train_per_sec_per_gpu": 62.11, "tokens/trainable": 196784} +{"epoch": 0.7421875, "grad_norm": 1.9003832340240479, "learning_rate": 7.707970881383977e-05, "loss": 0.2742688059806824, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.45, "memory/max_allocated (GiB)": 36.45, "ppl": 1.31557, "step": 190, "tokens/total": 6579760, "tokens/train_per_sec_per_gpu": 67.16, "tokens/trainable": 197799} +{"epoch": 0.74609375, "grad_norm": 9.040047645568848, "learning_rate": 7.682630588562518e-05, "loss": 0.27615755796432495, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.7, "memory/max_allocated (GiB)": 38.7, "ppl": 1.31806, "step": 191, "tokens/total": 6614800, "tokens/train_per_sec_per_gpu": 57.34, "tokens/trainable": 198860} +{"epoch": 0.75, "grad_norm": 1.3193451166152954, "learning_rate": 7.657199467573129e-05, "loss": 0.25738847255706787, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.29355, "step": 192, "tokens/total": 6648928, "tokens/train_per_sec_per_gpu": 55.53, "tokens/trainable": 199872} +{"epoch": 0.75390625, "grad_norm": 1.3588685989379883, "learning_rate": 7.631678576708561e-05, "loss": 0.23200422525405884, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.16, "memory/max_allocated (GiB)": 38.16, "ppl": 1.26113, "step": 193, "tokens/total": 6683328, "tokens/train_per_sec_per_gpu": 67.0, "tokens/trainable": 200966} +{"epoch": 0.7578125, "grad_norm": 2.717395067214966, "learning_rate": 7.606068977997255e-05, "loss": 0.25422045588493347, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.64, "memory/max_allocated (GiB)": 39.64, "ppl": 1.28946, "step": 194, "tokens/total": 6720928, "tokens/train_per_sec_per_gpu": 65.81, "tokens/trainable": 202132} +{"epoch": 0.76171875, "grad_norm": 2.315150022506714, "learning_rate": 7.580371737159148e-05, "loss": 0.31938058137893677, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.37628, "step": 195, "tokens/total": 6757024, "tokens/train_per_sec_per_gpu": 56.56, "tokens/trainable": 203179} +{"epoch": 0.765625, "grad_norm": 1.766922116279602, "learning_rate": 7.554587923561324e-05, "loss": 0.2621215879917145, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.99, "memory/max_allocated (GiB)": 38.99, "ppl": 1.29968, "step": 196, "tokens/total": 6791728, "tokens/train_per_sec_per_gpu": 51.53, "tokens/trainable": 204151} +{"epoch": 0.76953125, "grad_norm": 2.5309510231018066, "learning_rate": 7.528718610173511e-05, "loss": 0.27830541133880615, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.32089, "step": 197, "tokens/total": 6827072, "tokens/train_per_sec_per_gpu": 62.93, "tokens/trainable": 205245} +{"epoch": 0.7734375, "grad_norm": 2.861661672592163, "learning_rate": 7.502764873523431e-05, "loss": 0.3094530999660492, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.36268, "step": 198, "tokens/total": 6861120, "tokens/train_per_sec_per_gpu": 60.29, "tokens/trainable": 206265} +{"epoch": 0.77734375, "grad_norm": 4.225286483764648, "learning_rate": 7.476727793652011e-05, "loss": 0.2506425678730011, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.72, "memory/max_allocated (GiB)": 36.72, "ppl": 1.28485, "step": 199, "tokens/total": 6894352, "tokens/train_per_sec_per_gpu": 62.3, "tokens/trainable": 207320} +{"epoch": 0.78125, "grad_norm": 1.33363676071167, "learning_rate": 7.450608454068415e-05, "loss": 0.23092326521873474, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.25976, "step": 200, "tokens/total": 6928848, "tokens/train_per_sec_per_gpu": 57.56, "tokens/trainable": 208367} +{"epoch": 0.78515625, "grad_norm": 1.3916816711425781, "learning_rate": 7.424407941704987e-05, "loss": 0.23269104957580566, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.49, "memory/max_allocated (GiB)": 38.49, "ppl": 1.26199, "step": 201, "tokens/total": 6965504, "tokens/train_per_sec_per_gpu": 66.84, "tokens/trainable": 209512} +{"epoch": 0.7890625, "grad_norm": 1.8767189979553223, "learning_rate": 7.398127346871986e-05, "loss": 0.23611865937709808, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.61, "memory/max_allocated (GiB)": 39.61, "ppl": 1.26632, "step": 202, "tokens/total": 7002448, "tokens/train_per_sec_per_gpu": 54.69, "tokens/trainable": 210555} +{"epoch": 0.79296875, "grad_norm": 1.7731386423110962, "learning_rate": 7.371767763212238e-05, "loss": 0.24505499005317688, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.52, "memory/max_allocated (GiB)": 39.52, "ppl": 1.27769, "step": 203, "tokens/total": 7037824, "tokens/train_per_sec_per_gpu": 62.2, "tokens/trainable": 211606} +{"epoch": 0.796875, "grad_norm": 2.068711042404175, "learning_rate": 7.345330287655617e-05, "loss": 0.2646321654319763, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.09, "memory/max_allocated (GiB)": 39.09, "ppl": 1.30295, "step": 204, "tokens/total": 7075920, "tokens/train_per_sec_per_gpu": 48.42, "tokens/trainable": 212641} +{"epoch": 0.80078125, "grad_norm": 1.7276726961135864, "learning_rate": 7.31881602037339e-05, "loss": 0.28591668605804443, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.69, "memory/max_allocated (GiB)": 36.69, "ppl": 1.33098, "step": 205, "tokens/total": 7107888, "tokens/train_per_sec_per_gpu": 61.3, "tokens/trainable": 213644} +{"epoch": 0.8046875, "grad_norm": 1.1138030290603638, "learning_rate": 7.29222606473245e-05, "loss": 0.2610415816307068, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.29828, "step": 206, "tokens/total": 7140720, "tokens/train_per_sec_per_gpu": 61.11, "tokens/trainable": 214662} +{"epoch": 0.80859375, "grad_norm": 1.5033025741577148, "learning_rate": 7.265561527249383e-05, "loss": 0.27748632431030273, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.31981, "step": 207, "tokens/total": 7174720, "tokens/train_per_sec_per_gpu": 61.8, "tokens/trainable": 215658} +{"epoch": 0.8125, "grad_norm": 0.8153293132781982, "learning_rate": 7.238823517544436e-05, "loss": 0.19795310497283936, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.21891, "step": 208, "tokens/total": 7211872, "tokens/train_per_sec_per_gpu": 66.91, "tokens/trainable": 216826} +{"epoch": 0.81640625, "grad_norm": 1.3248175382614136, "learning_rate": 7.212013148295333e-05, "loss": 0.2335214614868164, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.26304, "step": 209, "tokens/total": 7245856, "tokens/train_per_sec_per_gpu": 62.93, "tokens/trainable": 217905} +{"epoch": 0.8203125, "grad_norm": 1.3227719068527222, "learning_rate": 7.185131535190975e-05, "loss": 0.25058841705322266, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.06, "memory/max_allocated (GiB)": 37.06, "ppl": 1.28478, "step": 210, "tokens/total": 7278448, "tokens/train_per_sec_per_gpu": 62.06, "tokens/trainable": 218966} +{"epoch": 0.82421875, "grad_norm": 1.3310840129852295, "learning_rate": 7.158179796885005e-05, "loss": 0.22880594432353973, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.2571, "step": 211, "tokens/total": 7313072, "tokens/train_per_sec_per_gpu": 61.71, "tokens/trainable": 220019} +{"epoch": 0.828125, "grad_norm": 1.7118607759475708, "learning_rate": 7.131159054949273e-05, "loss": 0.2731420695781708, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.31409, "step": 212, "tokens/total": 7351456, "tokens/train_per_sec_per_gpu": 54.57, "tokens/trainable": 221058} +{"epoch": 0.83203125, "grad_norm": 1.8065165281295776, "learning_rate": 7.104070433827139e-05, "loss": 0.26298898458480835, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.30081, "step": 213, "tokens/total": 7386880, "tokens/train_per_sec_per_gpu": 49.97, "tokens/trainable": 221992} +{"epoch": 0.8359375, "grad_norm": 3.198467969894409, "learning_rate": 7.076915060786705e-05, "loss": 0.25190699100494385, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.28648, "step": 214, "tokens/total": 7419696, "tokens/train_per_sec_per_gpu": 61.73, "tokens/trainable": 223033} +{"epoch": 0.83984375, "grad_norm": 1.2024096250534058, "learning_rate": 7.049694065873882e-05, "loss": 0.26174646615982056, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.07, "memory/max_allocated (GiB)": 38.07, "ppl": 1.2992, "step": 215, "tokens/total": 7454352, "tokens/train_per_sec_per_gpu": 55.77, "tokens/trainable": 224065} +{"epoch": 0.84375, "grad_norm": 1.5924487113952637, "learning_rate": 7.022408581865382e-05, "loss": 0.23426921665668488, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.26398, "step": 216, "tokens/total": 7488496, "tokens/train_per_sec_per_gpu": 57.53, "tokens/trainable": 225142} +{"epoch": 0.84765625, "grad_norm": 1.0791513919830322, "learning_rate": 6.99505974422157e-05, "loss": 0.22674968838691711, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.25452, "step": 217, "tokens/total": 7522976, "tokens/train_per_sec_per_gpu": 65.28, "tokens/trainable": 226224} +{"epoch": 0.8515625, "grad_norm": 1.3200467824935913, "learning_rate": 6.967648691039213e-05, "loss": 0.2569959759712219, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.58, "memory/max_allocated (GiB)": 36.58, "ppl": 1.29304, "step": 218, "tokens/total": 7554848, "tokens/train_per_sec_per_gpu": 63.57, "tokens/trainable": 227230} +{"epoch": 0.85546875, "grad_norm": 1.9570035934448242, "learning_rate": 6.940176563004123e-05, "loss": 0.24888452887535095, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.28259, "step": 219, "tokens/total": 7588896, "tokens/train_per_sec_per_gpu": 57.16, "tokens/trainable": 228203} +{"epoch": 0.859375, "grad_norm": 1.6496570110321045, "learning_rate": 6.912644503343682e-05, "loss": 0.26003825664520264, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.29698, "step": 220, "tokens/total": 7627056, "tokens/train_per_sec_per_gpu": 55.27, "tokens/trainable": 229212} +{"epoch": 0.86328125, "grad_norm": 1.652719259262085, "learning_rate": 6.885053657779273e-05, "loss": 0.24214692413806915, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.28, "memory/max_allocated (GiB)": 36.28, "ppl": 1.27398, "step": 221, "tokens/total": 7658368, "tokens/train_per_sec_per_gpu": 59.57, "tokens/trainable": 230227} +{"epoch": 0.8671875, "grad_norm": 2.6761109828948975, "learning_rate": 6.857405174478604e-05, "loss": 0.2652522325515747, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.30376, "step": 222, "tokens/total": 7693072, "tokens/train_per_sec_per_gpu": 57.61, "tokens/trainable": 231237} +{"epoch": 0.87109375, "grad_norm": 1.9701874256134033, "learning_rate": 6.82970020400792e-05, "loss": 0.25439488887786865, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.28968, "step": 223, "tokens/total": 7730288, "tokens/train_per_sec_per_gpu": 52.45, "tokens/trainable": 232255} +{"epoch": 0.875, "grad_norm": 0.9692584276199341, "learning_rate": 6.801939899284132e-05, "loss": 0.20162136852741241, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.22338, "step": 224, "tokens/total": 7766304, "tokens/train_per_sec_per_gpu": 58.21, "tokens/trainable": 233334} +{"epoch": 0.87890625, "grad_norm": 1.7122207880020142, "learning_rate": 6.774125415526827e-05, "loss": 0.20421835780143738, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.22657, "step": 225, "tokens/total": 7799680, "tokens/train_per_sec_per_gpu": 70.47, "tokens/trainable": 234472} +{"epoch": 0.8828125, "grad_norm": 1.021761417388916, "learning_rate": 6.746257910210214e-05, "loss": 0.22308029234409332, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.24992, "step": 226, "tokens/total": 7832912, "tokens/train_per_sec_per_gpu": 59.19, "tokens/trainable": 235484} +{"epoch": 0.88671875, "grad_norm": 1.2976535558700562, "learning_rate": 6.718338543014937e-05, "loss": 0.2657890319824219, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 35.66, "memory/max_allocated (GiB)": 35.66, "ppl": 1.30446, "step": 227, "tokens/total": 7862912, "tokens/train_per_sec_per_gpu": 64.2, "tokens/trainable": 236464} +{"epoch": 0.890625, "grad_norm": 0.875809907913208, "learning_rate": 6.69036847577983e-05, "loss": 0.2187878042459488, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.24457, "step": 228, "tokens/total": 7896816, "tokens/train_per_sec_per_gpu": 60.93, "tokens/trainable": 237493} +{"epoch": 0.89453125, "grad_norm": 1.423990249633789, "learning_rate": 6.662348872453553e-05, "loss": 0.19171738624572754, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.46, "memory/max_allocated (GiB)": 38.46, "ppl": 1.21133, "step": 229, "tokens/total": 7933376, "tokens/train_per_sec_per_gpu": 60.65, "tokens/trainable": 238561} +{"epoch": 0.8984375, "grad_norm": 1.4456197023391724, "learning_rate": 6.63428089904618e-05, "loss": 0.22650331258773804, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.25421, "step": 230, "tokens/total": 7972544, "tokens/train_per_sec_per_gpu": 51.2, "tokens/trainable": 239581} +{"epoch": 0.90234375, "grad_norm": 2.335331439971924, "learning_rate": 6.60616572358065e-05, "loss": 0.26099178194999695, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.29822, "step": 231, "tokens/total": 8007008, "tokens/train_per_sec_per_gpu": 60.47, "tokens/trainable": 240536} +{"epoch": 0.90625, "grad_norm": 1.500065803527832, "learning_rate": 6.578004516044172e-05, "loss": 0.22236758470535278, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.24903, "step": 232, "tokens/total": 8042864, "tokens/train_per_sec_per_gpu": 61.88, "tokens/trainable": 241574} +{"epoch": 0.91015625, "grad_norm": 1.799627661705017, "learning_rate": 6.549798448339548e-05, "loss": 0.22431732714176178, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.6, "memory/max_allocated (GiB)": 38.6, "ppl": 1.25147, "step": 233, "tokens/total": 8080272, "tokens/train_per_sec_per_gpu": 59.99, "tokens/trainable": 242630} +{"epoch": 0.9140625, "grad_norm": 1.6738574504852295, "learning_rate": 6.521548694236384e-05, "loss": 0.2397570163011551, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.27094, "step": 234, "tokens/total": 8117584, "tokens/train_per_sec_per_gpu": 52.57, "tokens/trainable": 243663} +{"epoch": 0.91796875, "grad_norm": 0.9905894994735718, "learning_rate": 6.493256429322259e-05, "loss": 0.21562615036964417, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.24064, "step": 235, "tokens/total": 8153712, "tokens/train_per_sec_per_gpu": 55.31, "tokens/trainable": 244690} +{"epoch": 0.921875, "grad_norm": 3.290409564971924, "learning_rate": 6.464922830953799e-05, "loss": 0.23388636112213135, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.2635, "step": 236, "tokens/total": 8192928, "tokens/train_per_sec_per_gpu": 51.85, "tokens/trainable": 245714} +{"epoch": 0.92578125, "grad_norm": 1.4276875257492065, "learning_rate": 6.436549078207688e-05, "loss": 0.2299521267414093, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.21, "memory/max_allocated (GiB)": 38.21, "ppl": 1.25854, "step": 237, "tokens/total": 8226016, "tokens/train_per_sec_per_gpu": 61.98, "tokens/trainable": 246714} +{"epoch": 0.9296875, "grad_norm": 1.376644492149353, "learning_rate": 6.408136351831592e-05, "loss": 0.22606989741325378, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 37.15, "memory/max_allocated (GiB)": 37.15, "ppl": 1.25366, "step": 238, "tokens/total": 8258384, "tokens/train_per_sec_per_gpu": 59.95, "tokens/trainable": 247711} +{"epoch": 0.93359375, "grad_norm": 1.9782863855361938, "learning_rate": 6.379685834195036e-05, "loss": 0.21448881924152374, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.23923, "step": 239, "tokens/total": 8297888, "tokens/train_per_sec_per_gpu": 54.16, "tokens/trainable": 248851} +{"epoch": 0.9375, "grad_norm": 0.9356327056884766, "learning_rate": 6.351198709240186e-05, "loss": 0.20040327310562134, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.61, "memory/max_allocated (GiB)": 39.61, "ppl": 1.2219, "step": 240, "tokens/total": 8338016, "tokens/train_per_sec_per_gpu": 52.32, "tokens/trainable": 249925} +{"epoch": 0.94140625, "grad_norm": 1.1326632499694824, "learning_rate": 6.32267616243259e-05, "loss": 0.23026353120803833, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.25893, "step": 241, "tokens/total": 8370608, "tokens/train_per_sec_per_gpu": 63.27, "tokens/trainable": 250955} +{"epoch": 0.9453125, "grad_norm": 1.6570628881454468, "learning_rate": 6.294119380711849e-05, "loss": 0.2785998582839966, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.32128, "step": 242, "tokens/total": 8405040, "tokens/train_per_sec_per_gpu": 51.48, "tokens/trainable": 251905} +{"epoch": 0.94921875, "grad_norm": 1.5110883712768555, "learning_rate": 6.265529552442209e-05, "loss": 0.22886741161346436, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.25718, "step": 243, "tokens/total": 8440400, "tokens/train_per_sec_per_gpu": 62.74, "tokens/trainable": 252963} +{"epoch": 0.953125, "grad_norm": 1.1426407098770142, "learning_rate": 6.236907867363127e-05, "loss": 0.21139633655548096, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.14, "memory/max_allocated (GiB)": 38.14, "ppl": 1.2354, "step": 244, "tokens/total": 8474736, "tokens/train_per_sec_per_gpu": 63.08, "tokens/trainable": 253997} +{"epoch": 0.95703125, "grad_norm": 3.9688737392425537, "learning_rate": 6.208255516539749e-05, "loss": 0.220360666513443, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 36.34, "memory/max_allocated (GiB)": 36.34, "ppl": 1.24653, "step": 245, "tokens/total": 8506944, "tokens/train_per_sec_per_gpu": 60.25, "tokens/trainable": 255054} +{"epoch": 0.9609375, "grad_norm": 1.3208547830581665, "learning_rate": 6.179573692313344e-05, "loss": 0.22610297799110413, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 1.2537, "step": 246, "tokens/total": 8544064, "tokens/train_per_sec_per_gpu": 57.61, "tokens/trainable": 256069} +{"epoch": 0.96484375, "grad_norm": 1.1955034732818604, "learning_rate": 6.150863588251694e-05, "loss": 0.2260730266571045, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.25367, "step": 247, "tokens/total": 8584336, "tokens/train_per_sec_per_gpu": 50.06, "tokens/trainable": 257125} +{"epoch": 0.96875, "grad_norm": 1.6522634029388428, "learning_rate": 6.122126399099419e-05, "loss": 0.2142319679260254, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.78, "memory/max_allocated (GiB)": 39.78, "ppl": 1.23891, "step": 248, "tokens/total": 8624304, "tokens/train_per_sec_per_gpu": 66.11, "tokens/trainable": 258381} +{"epoch": 0.97265625, "grad_norm": 1.1424237489700317, "learning_rate": 6.0933633207282615e-05, "loss": 0.2545319199562073, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.28986, "step": 249, "tokens/total": 8660672, "tokens/train_per_sec_per_gpu": 54.6, "tokens/trainable": 259442} +{"epoch": 0.9765625, "grad_norm": 0.9512038230895996, "learning_rate": 6.064575550087316e-05, "loss": 0.21998345851898193, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.63, "memory/max_allocated (GiB)": 39.63, "ppl": 1.24606, "step": 250, "tokens/total": 8697680, "tokens/train_per_sec_per_gpu": 56.17, "tokens/trainable": 260475} +{"epoch": 0.98046875, "grad_norm": 1.2512555122375488, "learning_rate": 6.0357642851532245e-05, "loss": 0.21982435882091522, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.24586, "step": 251, "tokens/total": 8734112, "tokens/train_per_sec_per_gpu": 54.63, "tokens/trainable": 261543} +{"epoch": 0.984375, "grad_norm": 1.031893253326416, "learning_rate": 6.0069307248803294e-05, "loss": 0.23739641904830933, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.26794, "step": 252, "tokens/total": 8769440, "tokens/train_per_sec_per_gpu": 50.21, "tokens/trainable": 262520} +{"epoch": 0.98828125, "grad_norm": 1.3710546493530273, "learning_rate": 5.9780760691507635e-05, "loss": 0.2392486184835434, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 36.25, "memory/max_allocated (GiB)": 36.25, "ppl": 1.27029, "step": 253, "tokens/total": 8800688, "tokens/train_per_sec_per_gpu": 61.24, "tokens/trainable": 263463} +{"epoch": 0.9921875, "grad_norm": 1.332377552986145, "learning_rate": 5.9492015187245334e-05, "loss": 0.2203308343887329, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.72, "memory/max_allocated (GiB)": 38.72, "ppl": 1.24649, "step": 254, "tokens/total": 8838272, "tokens/train_per_sec_per_gpu": 54.45, "tokens/trainable": 264428} +{"epoch": 0.99609375, "grad_norm": 1.0210461616516113, "learning_rate": 5.920308275189541e-05, "loss": 0.2174801081418991, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 36.6, "memory/max_allocated (GiB)": 36.6, "ppl": 1.24294, "step": 255, "tokens/total": 8869952, "tokens/train_per_sec_per_gpu": 60.82, "tokens/trainable": 265415} +{"epoch": 1.0, "grad_norm": 1.1635206937789917, "learning_rate": 5.8913975409115874e-05, "loss": 0.19779479503631592, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.21871, "step": 256, "tokens/total": 8903312, "tokens/train_per_sec_per_gpu": 62.25, "tokens/trainable": 266474} +{"epoch": 1.00390625, "grad_norm": 1.5926036834716797, "learning_rate": 5.8624705189843395e-05, "loss": 0.17567168176174164, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.19205, "step": 257, "tokens/total": 8940768, "tokens/train_per_sec_per_gpu": 55.28, "tokens/trainable": 267581} +{"epoch": 1.0078125, "grad_norm": 0.7813637256622314, "learning_rate": 5.833528413179249e-05, "loss": 0.17873725295066833, "memory/device_reserved (GiB)": 40.67, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.19571, "step": 258, "tokens/total": 8977392, "tokens/train_per_sec_per_gpu": 58.43, "tokens/trainable": 268673} +{"epoch": 1.01171875, "grad_norm": 2.704174041748047, "learning_rate": 5.80457242789548e-05, "loss": 0.16649433970451355, "memory/device_reserved (GiB)": 42.74, "memory/max_active (GiB)": 39.65, "memory/max_allocated (GiB)": 39.65, "ppl": 1.18116, "step": 259, "tokens/total": 9017568, "tokens/train_per_sec_per_gpu": 57.22, "tokens/trainable": 269843} +{"epoch": 1.015625, "grad_norm": 1.2509812116622925, "learning_rate": 5.77560376810977e-05, "loss": 0.1708044409751892, "memory/device_reserved (GiB)": 42.75, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.18626, "step": 260, "tokens/total": 9050928, "tokens/train_per_sec_per_gpu": 66.98, "tokens/trainable": 270933} +{"epoch": 1.01953125, "grad_norm": 1.6936029195785522, "learning_rate": 5.7466236393263005e-05, "loss": 0.20808151364326477, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.23131, "step": 261, "tokens/total": 9085552, "tokens/train_per_sec_per_gpu": 60.39, "tokens/trainable": 271950} +{"epoch": 1.0234375, "grad_norm": 1.840242624282837, "learning_rate": 5.717633247526522e-05, "loss": 0.22294364869594574, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 36.74, "memory/max_allocated (GiB)": 36.74, "ppl": 1.24975, "step": 262, "tokens/total": 9116240, "tokens/train_per_sec_per_gpu": 58.19, "tokens/trainable": 272872} +{"epoch": 1.02734375, "grad_norm": 2.256564140319824, "learning_rate": 5.688633799118971e-05, "loss": 0.2415706068277359, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.27325, "step": 263, "tokens/total": 9148688, "tokens/train_per_sec_per_gpu": 53.74, "tokens/trainable": 273796} +{"epoch": 1.03125, "grad_norm": 1.0519869327545166, "learning_rate": 5.659626500889066e-05, "loss": 0.18050998449325562, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.09, "memory/max_allocated (GiB)": 37.09, "ppl": 1.19783, "step": 264, "tokens/total": 9182704, "tokens/train_per_sec_per_gpu": 64.53, "tokens/trainable": 274922} +{"epoch": 1.03515625, "grad_norm": 1.111467957496643, "learning_rate": 5.6306125599488905e-05, "loss": 0.1912330538034439, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 36.88, "memory/max_allocated (GiB)": 36.88, "ppl": 1.21074, "step": 265, "tokens/total": 9213552, "tokens/train_per_sec_per_gpu": 67.93, "tokens/trainable": 275931} +{"epoch": 1.0390625, "grad_norm": 1.0651665925979614, "learning_rate": 5.601593183686955e-05, "loss": 0.186489999294281, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.20501, "step": 266, "tokens/total": 9250768, "tokens/train_per_sec_per_gpu": 59.81, "tokens/trainable": 277050} +{"epoch": 1.04296875, "grad_norm": 1.238631248474121, "learning_rate": 5.572569579717961e-05, "loss": 0.19912362098693848, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.22033, "step": 267, "tokens/total": 9286176, "tokens/train_per_sec_per_gpu": 61.51, "tokens/trainable": 278138} +{"epoch": 1.046875, "grad_norm": 0.7887837886810303, "learning_rate": 5.543542955832538e-05, "loss": 0.1777666211128235, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.19455, "step": 268, "tokens/total": 9326128, "tokens/train_per_sec_per_gpu": 52.56, "tokens/trainable": 279221} +{"epoch": 1.05078125, "grad_norm": 0.8907971382141113, "learning_rate": 5.514514519946986e-05, "loss": 0.16830286383628845, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.57, "memory/max_allocated (GiB)": 39.57, "ppl": 1.18329, "step": 269, "tokens/total": 9366544, "tokens/train_per_sec_per_gpu": 56.86, "tokens/trainable": 280342} +{"epoch": 1.0546875, "grad_norm": 0.777661919593811, "learning_rate": 5.485485480053015e-05, "loss": 0.18935370445251465, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.52, "memory/max_allocated (GiB)": 39.52, "ppl": 1.20847, "step": 270, "tokens/total": 9404928, "tokens/train_per_sec_per_gpu": 47.87, "tokens/trainable": 281326} +{"epoch": 1.05859375, "grad_norm": 0.7942616939544678, "learning_rate": 5.4564570441674645e-05, "loss": 0.1730557382106781, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.18893, "step": 271, "tokens/total": 9438592, "tokens/train_per_sec_per_gpu": 64.63, "tokens/trainable": 282349} +{"epoch": 1.0625, "grad_norm": 1.468727707862854, "learning_rate": 5.42743042028204e-05, "loss": 0.20174692571163177, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.83, "memory/max_allocated (GiB)": 37.83, "ppl": 1.22354, "step": 272, "tokens/total": 9474144, "tokens/train_per_sec_per_gpu": 46.89, "tokens/trainable": 283295} +{"epoch": 1.06640625, "grad_norm": 0.8132858872413635, "learning_rate": 5.3984068163130464e-05, "loss": 0.17745843529701233, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.19418, "step": 273, "tokens/total": 9507904, "tokens/train_per_sec_per_gpu": 67.42, "tokens/trainable": 284457} +{"epoch": 1.0703125, "grad_norm": 0.9781003594398499, "learning_rate": 5.369387440051111e-05, "loss": 0.20210158824920654, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.18, "memory/max_allocated (GiB)": 37.18, "ppl": 1.22397, "step": 274, "tokens/total": 9541616, "tokens/train_per_sec_per_gpu": 58.73, "tokens/trainable": 285459} +{"epoch": 1.07421875, "grad_norm": 5.39489221572876, "learning_rate": 5.340373499110935e-05, "loss": 0.18965822458267212, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.20884, "step": 275, "tokens/total": 9577424, "tokens/train_per_sec_per_gpu": 57.76, "tokens/trainable": 286527} +{"epoch": 1.078125, "grad_norm": 0.8801029324531555, "learning_rate": 5.3113662008810304e-05, "loss": 0.1807793527841568, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.19815, "step": 276, "tokens/total": 9617760, "tokens/train_per_sec_per_gpu": 50.3, "tokens/trainable": 287587} +{"epoch": 1.08203125, "grad_norm": 0.872872531414032, "learning_rate": 5.282366752473479e-05, "loss": 0.17836377024650574, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.29, "memory/max_allocated (GiB)": 38.29, "ppl": 1.19526, "step": 277, "tokens/total": 9655008, "tokens/train_per_sec_per_gpu": 58.06, "tokens/trainable": 288688} +{"epoch": 1.0859375, "grad_norm": 0.9808855652809143, "learning_rate": 5.2533763606737005e-05, "loss": 0.20010188221931458, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.28, "memory/max_allocated (GiB)": 39.28, "ppl": 1.22153, "step": 278, "tokens/total": 9694896, "tokens/train_per_sec_per_gpu": 46.77, "tokens/trainable": 289684} +{"epoch": 1.08984375, "grad_norm": 0.9225172996520996, "learning_rate": 5.224396231890232e-05, "loss": 0.1733156442642212, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.23, "memory/max_allocated (GiB)": 37.23, "ppl": 1.18924, "step": 279, "tokens/total": 9726672, "tokens/train_per_sec_per_gpu": 66.63, "tokens/trainable": 290798} +{"epoch": 1.09375, "grad_norm": 1.2135555744171143, "learning_rate": 5.195427572104522e-05, "loss": 0.2068590223789215, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.22981, "step": 280, "tokens/total": 9761136, "tokens/train_per_sec_per_gpu": 58.95, "tokens/trainable": 291826} +{"epoch": 1.09765625, "grad_norm": 1.183495283126831, "learning_rate": 5.166471586820751e-05, "loss": 0.19296298921108246, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.21284, "step": 281, "tokens/total": 9797888, "tokens/train_per_sec_per_gpu": 61.07, "tokens/trainable": 292919} +{"epoch": 1.1015625, "grad_norm": 1.0937023162841797, "learning_rate": 5.1375294810156615e-05, "loss": 0.19411936402320862, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.21424, "step": 282, "tokens/total": 9832528, "tokens/train_per_sec_per_gpu": 59.66, "tokens/trainable": 293941} +{"epoch": 1.10546875, "grad_norm": 3.912278652191162, "learning_rate": 5.1086024590884144e-05, "loss": 0.19041301310062408, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.28, "memory/max_allocated (GiB)": 39.28, "ppl": 1.20975, "step": 283, "tokens/total": 9870784, "tokens/train_per_sec_per_gpu": 52.68, "tokens/trainable": 294942} +{"epoch": 1.109375, "grad_norm": 0.8200970888137817, "learning_rate": 5.079691724810461e-05, "loss": 0.1886494755744934, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.20762, "step": 284, "tokens/total": 9906656, "tokens/train_per_sec_per_gpu": 49.17, "tokens/trainable": 295911} +{"epoch": 1.11328125, "grad_norm": 1.2316478490829468, "learning_rate": 5.0507984812754684e-05, "loss": 0.20417320728302002, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.22651, "step": 285, "tokens/total": 9940208, "tokens/train_per_sec_per_gpu": 59.62, "tokens/trainable": 296958} +{"epoch": 1.1171875, "grad_norm": 1.0528919696807861, "learning_rate": 5.021923930849237e-05, "loss": 0.19368794560432434, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.21372, "step": 286, "tokens/total": 9974096, "tokens/train_per_sec_per_gpu": 56.76, "tokens/trainable": 297971} +{"epoch": 1.12109375, "grad_norm": 0.9394338130950928, "learning_rate": 4.99306927511967e-05, "loss": 0.1772979199886322, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.19399, "step": 287, "tokens/total": 10007568, "tokens/train_per_sec_per_gpu": 62.25, "tokens/trainable": 298978} +{"epoch": 1.125, "grad_norm": 0.8863123059272766, "learning_rate": 4.964235714846775e-05, "loss": 0.19135287404060364, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 36.44, "memory/max_allocated (GiB)": 36.44, "ppl": 1.21089, "step": 288, "tokens/total": 10039280, "tokens/train_per_sec_per_gpu": 66.35, "tokens/trainable": 299972} +{"epoch": 1.12890625, "grad_norm": 1.030317783355713, "learning_rate": 4.9354244499126866e-05, "loss": 0.1927689015865326, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 36.23, "memory/max_allocated (GiB)": 36.23, "ppl": 1.2126, "step": 289, "tokens/total": 10070944, "tokens/train_per_sec_per_gpu": 63.2, "tokens/trainable": 301004} +{"epoch": 1.1328125, "grad_norm": 0.8242776393890381, "learning_rate": 4.90663667927174e-05, "loss": 0.185847669839859, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.20424, "step": 290, "tokens/total": 10103776, "tokens/train_per_sec_per_gpu": 71.15, "tokens/trainable": 302031} +{"epoch": 1.13671875, "grad_norm": 0.7814574837684631, "learning_rate": 4.877873600900581e-05, "loss": 0.16898921132087708, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.07, "memory/max_allocated (GiB)": 38.07, "ppl": 1.18411, "step": 291, "tokens/total": 10139888, "tokens/train_per_sec_per_gpu": 54.7, "tokens/trainable": 303058} +{"epoch": 1.140625, "grad_norm": 0.7430219054222107, "learning_rate": 4.849136411748306e-05, "loss": 0.16643546521663666, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.04, "memory/max_allocated (GiB)": 39.04, "ppl": 1.18109, "step": 292, "tokens/total": 10177936, "tokens/train_per_sec_per_gpu": 53.65, "tokens/trainable": 304115} +{"epoch": 1.14453125, "grad_norm": 0.9838630557060242, "learning_rate": 4.8204263076866574e-05, "loss": 0.18469443917274475, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.20285, "step": 293, "tokens/total": 10213856, "tokens/train_per_sec_per_gpu": 57.53, "tokens/trainable": 305177} +{"epoch": 1.1484375, "grad_norm": 0.9204061627388, "learning_rate": 4.791744483460251e-05, "loss": 0.2049979567527771, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.74, "memory/max_allocated (GiB)": 37.74, "ppl": 1.22752, "step": 294, "tokens/total": 10248352, "tokens/train_per_sec_per_gpu": 51.75, "tokens/trainable": 306171} +{"epoch": 1.15234375, "grad_norm": 1.5356297492980957, "learning_rate": 4.7630921326368736e-05, "loss": 0.1859491616487503, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.20436, "step": 295, "tokens/total": 10287072, "tokens/train_per_sec_per_gpu": 50.04, "tokens/trainable": 307153} +{"epoch": 1.15625, "grad_norm": 1.4845062494277954, "learning_rate": 4.7344704475577916e-05, "loss": 0.18253093957901, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 35.7, "memory/max_allocated (GiB)": 35.7, "ppl": 1.20025, "step": 296, "tokens/total": 10316640, "tokens/train_per_sec_per_gpu": 68.39, "tokens/trainable": 308179} +{"epoch": 1.16015625, "grad_norm": 1.6648465394973755, "learning_rate": 4.705880619288153e-05, "loss": 0.20937737822532654, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.37, "memory/max_allocated (GiB)": 38.37, "ppl": 1.23291, "step": 297, "tokens/total": 10352864, "tokens/train_per_sec_per_gpu": 52.98, "tokens/trainable": 309164} +{"epoch": 1.1640625, "grad_norm": 0.8714199662208557, "learning_rate": 4.677323837567412e-05, "loss": 0.19133353233337402, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.21086, "step": 298, "tokens/total": 10385696, "tokens/train_per_sec_per_gpu": 64.62, "tokens/trainable": 310220} +{"epoch": 1.16796875, "grad_norm": 1.2952368259429932, "learning_rate": 4.6488012907598146e-05, "loss": 0.19897115230560303, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 36.4, "memory/max_allocated (GiB)": 36.4, "ppl": 1.22015, "step": 299, "tokens/total": 10417904, "tokens/train_per_sec_per_gpu": 57.34, "tokens/trainable": 311175} +{"epoch": 1.171875, "grad_norm": 0.868607759475708, "learning_rate": 4.620314165804964e-05, "loss": 0.17611974477767944, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.19258, "step": 300, "tokens/total": 10453728, "tokens/train_per_sec_per_gpu": 58.5, "tokens/trainable": 312271} +{"epoch": 1.17578125, "grad_norm": 1.5163092613220215, "learning_rate": 4.591863648168407e-05, "loss": 0.1735246777534485, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.18949, "step": 301, "tokens/total": 10488288, "tokens/train_per_sec_per_gpu": 59.24, "tokens/trainable": 313368} +{"epoch": 1.1796875, "grad_norm": 1.0096852779388428, "learning_rate": 4.5634509217923135e-05, "loss": 0.19263437390327454, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.96, "memory/max_allocated (GiB)": 37.96, "ppl": 1.21244, "step": 302, "tokens/total": 10521824, "tokens/train_per_sec_per_gpu": 61.03, "tokens/trainable": 314406} +{"epoch": 1.18359375, "grad_norm": 1.058298110961914, "learning_rate": 4.535077169046201e-05, "loss": 0.18958809971809387, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.03, "memory/max_allocated (GiB)": 38.03, "ppl": 1.20875, "step": 303, "tokens/total": 10558432, "tokens/train_per_sec_per_gpu": 56.99, "tokens/trainable": 315436} +{"epoch": 1.1875, "grad_norm": 0.9910153746604919, "learning_rate": 4.506743570677743e-05, "loss": 0.20947876572608948, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 36.35, "memory/max_allocated (GiB)": 36.35, "ppl": 1.23304, "step": 304, "tokens/total": 10588752, "tokens/train_per_sec_per_gpu": 71.93, "tokens/trainable": 316428} +{"epoch": 1.19140625, "grad_norm": 0.9059833288192749, "learning_rate": 4.478451305763618e-05, "loss": 0.18538165092468262, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.20368, "step": 305, "tokens/total": 10622336, "tokens/train_per_sec_per_gpu": 64.03, "tokens/trainable": 317466} +{"epoch": 1.1953125, "grad_norm": 1.1193838119506836, "learning_rate": 4.450201551660454e-05, "loss": 0.1883002668619156, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 36.41, "memory/max_allocated (GiB)": 36.41, "ppl": 1.2072, "step": 306, "tokens/total": 10653520, "tokens/train_per_sec_per_gpu": 63.29, "tokens/trainable": 318471} +{"epoch": 1.19921875, "grad_norm": 1.303501844406128, "learning_rate": 4.4219954839558276e-05, "loss": 0.1978805512189865, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.21882, "step": 307, "tokens/total": 10688144, "tokens/train_per_sec_per_gpu": 53.74, "tokens/trainable": 319483} +{"epoch": 1.203125, "grad_norm": 0.8844285607337952, "learning_rate": 4.393834276419352e-05, "loss": 0.19094260036945343, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 35.7, "memory/max_allocated (GiB)": 35.7, "ppl": 1.21039, "step": 308, "tokens/total": 10718032, "tokens/train_per_sec_per_gpu": 70.56, "tokens/trainable": 320509} +{"epoch": 1.20703125, "grad_norm": 0.9019686579704285, "learning_rate": 4.36571910095382e-05, "loss": 0.1899542212486267, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.20919, "step": 309, "tokens/total": 10756464, "tokens/train_per_sec_per_gpu": 55.07, "tokens/trainable": 321516} +{"epoch": 1.2109375, "grad_norm": 0.8750078082084656, "learning_rate": 4.337651127546448e-05, "loss": 0.18892180919647217, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.20795, "step": 310, "tokens/total": 10792064, "tokens/train_per_sec_per_gpu": 62.17, "tokens/trainable": 322540} +{"epoch": 1.21484375, "grad_norm": 0.8706302642822266, "learning_rate": 4.3096315242201736e-05, "loss": 0.20503652095794678, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 36.51, "memory/max_allocated (GiB)": 36.51, "ppl": 1.22757, "step": 311, "tokens/total": 10824800, "tokens/train_per_sec_per_gpu": 58.92, "tokens/trainable": 323543} +{"epoch": 1.21875, "grad_norm": 0.7259270548820496, "learning_rate": 4.2816614569850635e-05, "loss": 0.1917053908109665, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 38.66, "memory/max_allocated (GiB)": 38.66, "ppl": 1.21131, "step": 312, "tokens/total": 10860576, "tokens/train_per_sec_per_gpu": 58.07, "tokens/trainable": 324543} +{"epoch": 1.22265625, "grad_norm": 1.4847807884216309, "learning_rate": 4.2537420897897864e-05, "loss": 0.2087540179491043, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 38.72, "memory/max_allocated (GiB)": 38.72, "ppl": 1.23214, "step": 313, "tokens/total": 10896224, "tokens/train_per_sec_per_gpu": 53.71, "tokens/trainable": 325554} +{"epoch": 1.2265625, "grad_norm": 2.472191333770752, "learning_rate": 4.225874584473174e-05, "loss": 0.19511385262012482, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.21545, "step": 314, "tokens/total": 10928768, "tokens/train_per_sec_per_gpu": 62.37, "tokens/trainable": 326556} +{"epoch": 1.23046875, "grad_norm": 0.8203500509262085, "learning_rate": 4.19806010071587e-05, "loss": 0.1745118796825409, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.19066, "step": 315, "tokens/total": 10966848, "tokens/train_per_sec_per_gpu": 53.48, "tokens/trainable": 327620} +{"epoch": 1.234375, "grad_norm": 1.248582363128662, "learning_rate": 4.170299795992081e-05, "loss": 0.2038920521736145, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 36.88, "memory/max_allocated (GiB)": 36.88, "ppl": 1.22617, "step": 316, "tokens/total": 11000416, "tokens/train_per_sec_per_gpu": 60.3, "tokens/trainable": 328662} +{"epoch": 1.23828125, "grad_norm": 0.9071127772331238, "learning_rate": 4.142594825521398e-05, "loss": 0.17762762308120728, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.19438, "step": 317, "tokens/total": 11039520, "tokens/train_per_sec_per_gpu": 51.33, "tokens/trainable": 329719} +{"epoch": 1.2421875, "grad_norm": 0.9159374833106995, "learning_rate": 4.114946342220728e-05, "loss": 0.18647852540016174, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.205, "step": 318, "tokens/total": 11076096, "tokens/train_per_sec_per_gpu": 56.43, "tokens/trainable": 330802} +{"epoch": 1.24609375, "grad_norm": 0.7763062119483948, "learning_rate": 4.087355496656321e-05, "loss": 0.16919225454330444, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.18435, "step": 319, "tokens/total": 11109728, "tokens/train_per_sec_per_gpu": 67.94, "tokens/trainable": 331918} +{"epoch": 1.25, "grad_norm": 0.7314566373825073, "learning_rate": 4.05982343699588e-05, "loss": 0.18359535932540894, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.20153, "step": 320, "tokens/total": 11148976, "tokens/train_per_sec_per_gpu": 58.34, "tokens/trainable": 333051} +{"epoch": 1.25390625, "grad_norm": 0.8399127125740051, "learning_rate": 4.0323513089607876e-05, "loss": 0.17450584471225739, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.19066, "step": 321, "tokens/total": 11183888, "tokens/train_per_sec_per_gpu": 57.64, "tokens/trainable": 334129} +{"epoch": 1.2578125, "grad_norm": 1.360384464263916, "learning_rate": 4.004940255778431e-05, "loss": 0.1952117383480072, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 39.63, "memory/max_allocated (GiB)": 39.63, "ppl": 1.21557, "step": 322, "tokens/total": 11222608, "tokens/train_per_sec_per_gpu": 56.58, "tokens/trainable": 335186} +{"epoch": 1.26171875, "grad_norm": 0.6743394136428833, "learning_rate": 3.977591418134619e-05, "loss": 0.18069420754909515, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.19805, "step": 323, "tokens/total": 11257568, "tokens/train_per_sec_per_gpu": 61.11, "tokens/trainable": 336236} +{"epoch": 1.265625, "grad_norm": 1.0464928150177002, "learning_rate": 3.95030593412612e-05, "loss": 0.19249534606933594, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.21227, "step": 324, "tokens/total": 11293472, "tokens/train_per_sec_per_gpu": 57.24, "tokens/trainable": 337236} +{"epoch": 1.26953125, "grad_norm": 1.08870530128479, "learning_rate": 3.923084939213296e-05, "loss": 0.20266768336296082, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 35.75, "memory/max_allocated (GiB)": 35.75, "ppl": 1.22467, "step": 325, "tokens/total": 11324048, "tokens/train_per_sec_per_gpu": 64.33, "tokens/trainable": 338251} +{"epoch": 1.2734375, "grad_norm": 0.6229018568992615, "learning_rate": 3.895929566172861e-05, "loss": 0.154499351978302, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.16707, "step": 326, "tokens/total": 11360064, "tokens/train_per_sec_per_gpu": 60.22, "tokens/trainable": 339355} +{"epoch": 1.27734375, "grad_norm": 0.8765996694564819, "learning_rate": 3.868840945050728e-05, "loss": 0.16605262458324432, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 39.19, "memory/max_allocated (GiB)": 39.19, "ppl": 1.18064, "step": 327, "tokens/total": 11399008, "tokens/train_per_sec_per_gpu": 56.01, "tokens/trainable": 340444} +{"epoch": 1.28125, "grad_norm": 0.68871009349823, "learning_rate": 3.841820203114995e-05, "loss": 0.16549132764339447, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 36.78, "memory/max_allocated (GiB)": 36.78, "ppl": 1.17997, "step": 328, "tokens/total": 11430752, "tokens/train_per_sec_per_gpu": 67.35, "tokens/trainable": 341529} +{"epoch": 1.28515625, "grad_norm": 0.998928427696228, "learning_rate": 3.814868464809027e-05, "loss": 0.18104630708694458, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 36.77, "memory/max_allocated (GiB)": 36.77, "ppl": 1.19847, "step": 329, "tokens/total": 11462688, "tokens/train_per_sec_per_gpu": 61.27, "tokens/trainable": 342525} +{"epoch": 1.2890625, "grad_norm": 1.168860912322998, "learning_rate": 3.787986851704667e-05, "loss": 0.19661453366279602, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.21727, "step": 330, "tokens/total": 11496240, "tokens/train_per_sec_per_gpu": 63.78, "tokens/trainable": 343562} +{"epoch": 1.29296875, "grad_norm": 1.3184361457824707, "learning_rate": 3.7611764824555654e-05, "loss": 0.15651145577430725, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 37.84, "memory/max_allocated (GiB)": 37.84, "ppl": 1.16942, "step": 331, "tokens/total": 11527968, "tokens/train_per_sec_per_gpu": 76.0, "tokens/trainable": 344715} +{"epoch": 1.296875, "grad_norm": 0.7898290753364563, "learning_rate": 3.734438472750619e-05, "loss": 0.1703772395849228, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.18575, "step": 332, "tokens/total": 11567936, "tokens/train_per_sec_per_gpu": 51.63, "tokens/trainable": 345805} +{"epoch": 1.30078125, "grad_norm": 1.4920841455459595, "learning_rate": 3.707773935267552e-05, "loss": 0.187323197722435, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.20602, "step": 333, "tokens/total": 11604992, "tokens/train_per_sec_per_gpu": 56.22, "tokens/trainable": 346917} +{"epoch": 1.3046875, "grad_norm": 1.647179126739502, "learning_rate": 3.68118397962661e-05, "loss": 0.20394425094127655, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 38.49, "memory/max_allocated (GiB)": 38.49, "ppl": 1.22623, "step": 334, "tokens/total": 11639744, "tokens/train_per_sec_per_gpu": 55.72, "tokens/trainable": 347879} +{"epoch": 1.30859375, "grad_norm": 0.9199114441871643, "learning_rate": 3.654669712344384e-05, "loss": 0.16698095202445984, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 36.72, "memory/max_allocated (GiB)": 36.72, "ppl": 1.18173, "step": 335, "tokens/total": 11670944, "tokens/train_per_sec_per_gpu": 74.77, "tokens/trainable": 348935} +{"epoch": 1.3125, "grad_norm": 0.6693864464759827, "learning_rate": 3.628232236787763e-05, "loss": 0.16074401140213013, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 37.91, "memory/max_allocated (GiB)": 37.91, "ppl": 1.17438, "step": 336, "tokens/total": 11706544, "tokens/train_per_sec_per_gpu": 56.15, "tokens/trainable": 350048} +{"epoch": 1.31640625, "grad_norm": 1.010785698890686, "learning_rate": 3.6018726531280144e-05, "loss": 0.15815426409244537, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.17135, "step": 337, "tokens/total": 11743952, "tokens/train_per_sec_per_gpu": 60.31, "tokens/trainable": 351143} +{"epoch": 1.3203125, "grad_norm": 1.492229700088501, "learning_rate": 3.575592058295017e-05, "loss": 0.19460049271583557, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.21483, "step": 338, "tokens/total": 11777936, "tokens/train_per_sec_per_gpu": 61.37, "tokens/trainable": 352197} +{"epoch": 1.32421875, "grad_norm": 0.9130093455314636, "learning_rate": 3.549391545931585e-05, "loss": 0.17952483892440796, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 38.93, "memory/max_allocated (GiB)": 38.93, "ppl": 1.19665, "step": 339, "tokens/total": 11812416, "tokens/train_per_sec_per_gpu": 62.29, "tokens/trainable": 353247} +{"epoch": 1.328125, "grad_norm": 1.036390781402588, "learning_rate": 3.5232722063479914e-05, "loss": 0.18483588099479675, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.20302, "step": 340, "tokens/total": 11846704, "tokens/train_per_sec_per_gpu": 65.18, "tokens/trainable": 354311} +{"epoch": 1.33203125, "grad_norm": 0.6065481305122375, "learning_rate": 3.49723512647657e-05, "loss": 0.16771335899829865, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.1826, "step": 341, "tokens/total": 11885968, "tokens/train_per_sec_per_gpu": 54.05, "tokens/trainable": 355427} +{"epoch": 1.3359375, "grad_norm": 0.8869233131408691, "learning_rate": 3.471281389826491e-05, "loss": 0.18736788630485535, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.20607, "step": 342, "tokens/total": 11917024, "tokens/train_per_sec_per_gpu": 63.33, "tokens/trainable": 356380} +{"epoch": 1.33984375, "grad_norm": 0.9638418555259705, "learning_rate": 3.4454120764386764e-05, "loss": 0.1742679923772812, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.19037, "step": 343, "tokens/total": 11955648, "tokens/train_per_sec_per_gpu": 56.58, "tokens/trainable": 357460} +{"epoch": 1.34375, "grad_norm": 0.6217043995857239, "learning_rate": 3.4196282628408526e-05, "loss": 0.17520490288734436, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 37.58, "memory/max_allocated (GiB)": 37.58, "ppl": 1.19149, "step": 344, "tokens/total": 11989936, "tokens/train_per_sec_per_gpu": 60.58, "tokens/trainable": 358480} +{"epoch": 1.34765625, "grad_norm": 0.9955883026123047, "learning_rate": 3.3939310220027456e-05, "loss": 0.16630150377750397, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.18093, "step": 345, "tokens/total": 12023968, "tokens/train_per_sec_per_gpu": 59.87, "tokens/trainable": 359568} +{"epoch": 1.3515625, "grad_norm": 1.210564136505127, "learning_rate": 3.3683214232914404e-05, "loss": 0.1667252779006958, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.18143, "step": 346, "tokens/total": 12059632, "tokens/train_per_sec_per_gpu": 59.68, "tokens/trainable": 360648} +{"epoch": 1.35546875, "grad_norm": 1.0597561597824097, "learning_rate": 3.342800532426873e-05, "loss": 0.19389355182647705, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.21397, "step": 347, "tokens/total": 12095616, "tokens/train_per_sec_per_gpu": 59.95, "tokens/trainable": 361666} +{"epoch": 1.359375, "grad_norm": 1.475042462348938, "learning_rate": 3.317369411437484e-05, "loss": 0.20073509216308594, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.2223, "step": 348, "tokens/total": 12131488, "tokens/train_per_sec_per_gpu": 55.96, "tokens/trainable": 362689} +{"epoch": 1.36328125, "grad_norm": 75.74794006347656, "learning_rate": 3.292029118616024e-05, "loss": 0.17891056835651398, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.19591, "step": 349, "tokens/total": 12164208, "tokens/train_per_sec_per_gpu": 56.82, "tokens/trainable": 363671} +{"epoch": 1.3671875, "grad_norm": 4.739177703857422, "learning_rate": 3.266780708475511e-05, "loss": 0.17568302154541016, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 39.12, "memory/max_allocated (GiB)": 39.12, "ppl": 1.19206, "step": 350, "tokens/total": 12199616, "tokens/train_per_sec_per_gpu": 58.25, "tokens/trainable": 364746} +{"epoch": 1.37109375, "grad_norm": 1.2485811710357666, "learning_rate": 3.241625231705354e-05, "loss": 0.1648242175579071, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.17919, "step": 351, "tokens/total": 12235664, "tokens/train_per_sec_per_gpu": 59.37, "tokens/trainable": 365811} +{"epoch": 1.375, "grad_norm": 0.6341890096664429, "learning_rate": 3.216563735127618e-05, "loss": 0.15179133415222168, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 39.12, "memory/max_allocated (GiB)": 39.12, "ppl": 1.16392, "step": 352, "tokens/total": 12270800, "tokens/train_per_sec_per_gpu": 60.98, "tokens/trainable": 366944} +{"epoch": 1.37890625, "grad_norm": 1.1588937044143677, "learning_rate": 3.191597261653475e-05, "loss": 0.19624078273773193, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.21682, "step": 353, "tokens/total": 12307568, "tokens/train_per_sec_per_gpu": 57.64, "tokens/trainable": 368012} +{"epoch": 1.3828125, "grad_norm": 0.6194597482681274, "learning_rate": 3.166726850239794e-05, "loss": 0.1622074544429779, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 37.18, "memory/max_allocated (GiB)": 37.18, "ppl": 1.1761, "step": 354, "tokens/total": 12341200, "tokens/train_per_sec_per_gpu": 62.4, "tokens/trainable": 369090} +{"epoch": 1.38671875, "grad_norm": 1.2414922714233398, "learning_rate": 3.141953535845912e-05, "loss": 0.20180970430374146, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 36.16, "memory/max_allocated (GiB)": 36.16, "ppl": 1.22362, "step": 355, "tokens/total": 12372016, "tokens/train_per_sec_per_gpu": 62.87, "tokens/trainable": 370028} +{"epoch": 1.390625, "grad_norm": 1.5725886821746826, "learning_rate": 3.11727834939056e-05, "loss": 0.1824224293231964, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 36.28, "memory/max_allocated (GiB)": 36.28, "ppl": 1.20012, "step": 356, "tokens/total": 12403856, "tokens/train_per_sec_per_gpu": 58.37, "tokens/trainable": 371051} +{"epoch": 1.39453125, "grad_norm": 0.9099920392036438, "learning_rate": 3.092702317708967e-05, "loss": 0.18117523193359375, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.19863, "step": 357, "tokens/total": 12437584, "tokens/train_per_sec_per_gpu": 62.29, "tokens/trainable": 372063} +{"epoch": 1.3984375, "grad_norm": 0.659208357334137, "learning_rate": 3.0682264635101276e-05, "loss": 0.1683492660522461, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 37.44, "memory/max_allocated (GiB)": 37.44, "ppl": 1.18335, "step": 358, "tokens/total": 12470096, "tokens/train_per_sec_per_gpu": 62.08, "tokens/trainable": 373064} +{"epoch": 1.40234375, "grad_norm": 1.2719485759735107, "learning_rate": 3.0438518053342407e-05, "loss": 0.16216588020324707, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.17606, "step": 359, "tokens/total": 12509168, "tokens/train_per_sec_per_gpu": 53.97, "tokens/trainable": 374175} +{"epoch": 1.40625, "grad_norm": 0.7695138454437256, "learning_rate": 3.0195793575103266e-05, "loss": 0.1656133532524109, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.18012, "step": 360, "tokens/total": 12548208, "tokens/train_per_sec_per_gpu": 54.78, "tokens/trainable": 375251} +{"epoch": 1.41015625, "grad_norm": 0.9815496206283569, "learning_rate": 2.9954101301140146e-05, "loss": 0.1877121925354004, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.20649, "step": 361, "tokens/total": 12583552, "tokens/train_per_sec_per_gpu": 58.69, "tokens/trainable": 376287} +{"epoch": 1.4140625, "grad_norm": 0.603298008441925, "learning_rate": 2.9713451289255123e-05, "loss": 0.1781720221042633, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 36.9, "memory/max_allocated (GiB)": 36.9, "ppl": 1.19503, "step": 362, "tokens/total": 12612944, "tokens/train_per_sec_per_gpu": 60.98, "tokens/trainable": 377310} +{"epoch": 1.41796875, "grad_norm": 0.5298221707344055, "learning_rate": 2.9473853553877484e-05, "loss": 0.15835468471050262, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.17158, "step": 363, "tokens/total": 12646880, "tokens/train_per_sec_per_gpu": 60.27, "tokens/trainable": 378349} +{"epoch": 1.421875, "grad_norm": 0.7589071393013, "learning_rate": 2.9235318065647e-05, "loss": 0.17651590704917908, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 39.61, "memory/max_allocated (GiB)": 39.61, "ppl": 1.19305, "step": 364, "tokens/total": 12686896, "tokens/train_per_sec_per_gpu": 53.36, "tokens/trainable": 379402} +{"epoch": 1.42578125, "grad_norm": 0.7494699954986572, "learning_rate": 2.8997854750998964e-05, "loss": 0.16888561844825745, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.18398, "step": 365, "tokens/total": 12726416, "tokens/train_per_sec_per_gpu": 57.41, "tokens/trainable": 380505} +{"epoch": 1.4296875, "grad_norm": 0.8193362951278687, "learning_rate": 2.8761473491751258e-05, "loss": 0.16984117031097412, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.63, "memory/max_allocated (GiB)": 36.63, "ppl": 1.18512, "step": 366, "tokens/total": 12759248, "tokens/train_per_sec_per_gpu": 59.85, "tokens/trainable": 381503} +{"epoch": 1.43359375, "grad_norm": 0.5133363008499146, "learning_rate": 2.8526184124692883e-05, "loss": 0.14311230182647705, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.15386, "step": 367, "tokens/total": 12795904, "tokens/train_per_sec_per_gpu": 52.55, "tokens/trainable": 382606} +{"epoch": 1.4375, "grad_norm": 0.8690593838691711, "learning_rate": 2.829199644117484e-05, "loss": 0.17037759721279144, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.18575, "step": 368, "tokens/total": 12829616, "tokens/train_per_sec_per_gpu": 62.78, "tokens/trainable": 383569} +{"epoch": 1.44140625, "grad_norm": 7.260054588317871, "learning_rate": 2.8058920186702553e-05, "loss": 0.15945035219192505, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.91, "memory/max_allocated (GiB)": 38.91, "ppl": 1.17287, "step": 369, "tokens/total": 12867744, "tokens/train_per_sec_per_gpu": 58.82, "tokens/trainable": 384668} +{"epoch": 1.4453125, "grad_norm": 0.6111997961997986, "learning_rate": 2.782696506053033e-05, "loss": 0.17712287604808807, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.19378, "step": 370, "tokens/total": 12905936, "tokens/train_per_sec_per_gpu": 52.6, "tokens/trainable": 385663} +{"epoch": 1.44921875, "grad_norm": 0.6920051574707031, "learning_rate": 2.7596140715257824e-05, "loss": 0.16033817827701569, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.17391, "step": 371, "tokens/total": 12941216, "tokens/train_per_sec_per_gpu": 61.44, "tokens/trainable": 386746} +{"epoch": 1.453125, "grad_norm": 0.7635849714279175, "learning_rate": 2.7366456756428184e-05, "loss": 0.16371774673461914, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.17788, "step": 372, "tokens/total": 12975584, "tokens/train_per_sec_per_gpu": 60.02, "tokens/trainable": 387782} +{"epoch": 1.45703125, "grad_norm": 0.865790843963623, "learning_rate": 2.7137922742128486e-05, "loss": 0.16870784759521484, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.21, "memory/max_allocated (GiB)": 38.21, "ppl": 1.18377, "step": 373, "tokens/total": 13010464, "tokens/train_per_sec_per_gpu": 61.41, "tokens/trainable": 388849} +{"epoch": 1.4609375, "grad_norm": 0.5389435887336731, "learning_rate": 2.691054818259188e-05, "loss": 0.15904885530471802, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.4, "memory/max_allocated (GiB)": 37.4, "ppl": 1.1724, "step": 374, "tokens/total": 13045136, "tokens/train_per_sec_per_gpu": 60.56, "tokens/trainable": 389955} +{"epoch": 1.46484375, "grad_norm": 1.5610884428024292, "learning_rate": 2.6684342539801933e-05, "loss": 0.1997978389263153, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.49, "memory/max_allocated (GiB)": 36.49, "ppl": 1.22116, "step": 375, "tokens/total": 13076400, "tokens/train_per_sec_per_gpu": 60.91, "tokens/trainable": 390933} +{"epoch": 1.46875, "grad_norm": 0.6057759523391724, "learning_rate": 2.645931522709877e-05, "loss": 0.15714433789253235, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.54, "memory/max_allocated (GiB)": 39.54, "ppl": 1.17016, "step": 376, "tokens/total": 13112144, "tokens/train_per_sec_per_gpu": 71.75, "tokens/trainable": 392094} +{"epoch": 1.47265625, "grad_norm": 0.9974581003189087, "learning_rate": 2.6235475608787365e-05, "loss": 0.179693341255188, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.6, "memory/max_allocated (GiB)": 38.6, "ppl": 1.19685, "step": 377, "tokens/total": 13147872, "tokens/train_per_sec_per_gpu": 58.94, "tokens/trainable": 393123} +{"epoch": 1.4765625, "grad_norm": 0.6962155699729919, "learning_rate": 2.6012832999747916e-05, "loss": 0.18975836038589478, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.20896, "step": 378, "tokens/total": 13178656, "tokens/train_per_sec_per_gpu": 63.93, "tokens/trainable": 394090} +{"epoch": 1.48046875, "grad_norm": 5.841899871826172, "learning_rate": 2.579139666504821e-05, "loss": 0.18656384944915771, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.2051, "step": 379, "tokens/total": 13218720, "tokens/train_per_sec_per_gpu": 51.26, "tokens/trainable": 395092} +{"epoch": 1.484375, "grad_norm": 1.1525871753692627, "learning_rate": 2.557117581955798e-05, "loss": 0.17128971219062805, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 1.18683, "step": 380, "tokens/total": 13252544, "tokens/train_per_sec_per_gpu": 59.49, "tokens/trainable": 396143} +{"epoch": 1.48828125, "grad_norm": 0.6087532043457031, "learning_rate": 2.5352179627565532e-05, "loss": 0.18804143369197845, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.06, "memory/max_allocated (GiB)": 38.06, "ppl": 1.20688, "step": 381, "tokens/total": 13286832, "tokens/train_per_sec_per_gpu": 50.02, "tokens/trainable": 397120} +{"epoch": 1.4921875, "grad_norm": 1.6520129442214966, "learning_rate": 2.5134417202396277e-05, "loss": 0.19138647615909576, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.21093, "step": 382, "tokens/total": 13320432, "tokens/train_per_sec_per_gpu": 57.45, "tokens/trainable": 398089} +{"epoch": 1.49609375, "grad_norm": 1.0219764709472656, "learning_rate": 2.491789760603361e-05, "loss": 0.18447308242321014, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.20258, "step": 383, "tokens/total": 13356928, "tokens/train_per_sec_per_gpu": 53.41, "tokens/trainable": 399105} +{"epoch": 1.5, "grad_norm": 0.9693041443824768, "learning_rate": 2.4702629848741764e-05, "loss": 0.17221945524215698, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.18794, "step": 384, "tokens/total": 13391568, "tokens/train_per_sec_per_gpu": 62.09, "tokens/trainable": 400109} +{"epoch": 1.50390625, "grad_norm": 0.7059345841407776, "learning_rate": 2.4488622888690785e-05, "loss": 0.17878970503807068, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.19577, "step": 385, "tokens/total": 13425152, "tokens/train_per_sec_per_gpu": 54.89, "tokens/trainable": 401085} +{"epoch": 1.5078125, "grad_norm": 1.8524093627929688, "learning_rate": 2.427588563158384e-05, "loss": 0.16204212605953217, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.11, "memory/max_allocated (GiB)": 39.11, "ppl": 1.17591, "step": 386, "tokens/total": 13459504, "tokens/train_per_sec_per_gpu": 62.07, "tokens/trainable": 402159} +{"epoch": 1.51171875, "grad_norm": 0.9323499202728271, "learning_rate": 2.406442693028651e-05, "loss": 0.18839561939239502, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.20731, "step": 387, "tokens/total": 13491136, "tokens/train_per_sec_per_gpu": 60.98, "tokens/trainable": 403100} +{"epoch": 1.515625, "grad_norm": 0.7437294721603394, "learning_rate": 2.3854255584458547e-05, "loss": 0.17596974968910217, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.1924, "step": 388, "tokens/total": 13530112, "tokens/train_per_sec_per_gpu": 51.07, "tokens/trainable": 404117} +{"epoch": 1.51953125, "grad_norm": 1.1401275396347046, "learning_rate": 2.3645380340187508e-05, "loss": 0.18143096566200256, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.14, "memory/max_allocated (GiB)": 39.14, "ppl": 1.19893, "step": 389, "tokens/total": 13565680, "tokens/train_per_sec_per_gpu": 62.35, "tokens/trainable": 405191} +{"epoch": 1.5234375, "grad_norm": 5.27384614944458, "learning_rate": 2.3437809889624914e-05, "loss": 0.18788141012191772, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.52, "memory/max_allocated (GiB)": 36.52, "ppl": 1.20669, "step": 390, "tokens/total": 13596960, "tokens/train_per_sec_per_gpu": 71.65, "tokens/trainable": 406242} +{"epoch": 1.52734375, "grad_norm": 0.5661614537239075, "learning_rate": 2.3231552870624487e-05, "loss": 0.16110292077064514, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.17481, "step": 391, "tokens/total": 13627840, "tokens/train_per_sec_per_gpu": 70.37, "tokens/trainable": 407315} +{"epoch": 1.53125, "grad_norm": 0.4813677966594696, "learning_rate": 2.3026617866382657e-05, "loss": 0.1502152681350708, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.7, "memory/max_allocated (GiB)": 36.7, "ppl": 1.16208, "step": 392, "tokens/total": 13660848, "tokens/train_per_sec_per_gpu": 62.48, "tokens/trainable": 408401} +{"epoch": 1.53515625, "grad_norm": 0.8648819923400879, "learning_rate": 2.2823013405081507e-05, "loss": 0.15709318220615387, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.1701, "step": 393, "tokens/total": 13699024, "tokens/train_per_sec_per_gpu": 53.32, "tokens/trainable": 409473} +{"epoch": 1.5390625, "grad_norm": 1.0039112567901611, "learning_rate": 2.2620747959533722e-05, "loss": 0.16136199235916138, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.17511, "step": 394, "tokens/total": 13732336, "tokens/train_per_sec_per_gpu": 65.52, "tokens/trainable": 410584} +{"epoch": 1.54296875, "grad_norm": 0.6043010950088501, "learning_rate": 2.2419829946830123e-05, "loss": 0.1715654581785202, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.18716, "step": 395, "tokens/total": 13768112, "tokens/train_per_sec_per_gpu": 59.91, "tokens/trainable": 411602} +{"epoch": 1.546875, "grad_norm": 0.4686942994594574, "learning_rate": 2.2220267727989325e-05, "loss": 0.15722857415676117, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.74, "memory/max_allocated (GiB)": 37.74, "ppl": 1.17026, "step": 396, "tokens/total": 13802480, "tokens/train_per_sec_per_gpu": 58.67, "tokens/trainable": 412649} +{"epoch": 1.55078125, "grad_norm": 0.7620572447776794, "learning_rate": 2.202206960760984e-05, "loss": 0.1455366015434265, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.15666, "step": 397, "tokens/total": 13840736, "tokens/train_per_sec_per_gpu": 55.94, "tokens/trainable": 413796} +{"epoch": 1.5546875, "grad_norm": 0.531404435634613, "learning_rate": 2.182524383352446e-05, "loss": 0.18463221192359924, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.20278, "step": 398, "tokens/total": 13876896, "tokens/train_per_sec_per_gpu": 55.46, "tokens/trainable": 414806} +{"epoch": 1.55859375, "grad_norm": 0.5792882442474365, "learning_rate": 2.1629798596457056e-05, "loss": 0.1682618409395218, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.34, "memory/max_allocated (GiB)": 36.34, "ppl": 1.18325, "step": 399, "tokens/total": 13908992, "tokens/train_per_sec_per_gpu": 62.88, "tokens/trainable": 415811} +{"epoch": 1.5625, "grad_norm": 0.6285470724105835, "learning_rate": 2.1435742029681725e-05, "loss": 0.17911259829998016, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.29, "memory/max_allocated (GiB)": 36.29, "ppl": 1.19616, "step": 400, "tokens/total": 13939456, "tokens/train_per_sec_per_gpu": 68.0, "tokens/trainable": 416768} +{"epoch": 1.56640625, "grad_norm": 0.4728090763092041, "learning_rate": 2.124308220868431e-05, "loss": 0.15052112936973572, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.16244, "step": 401, "tokens/total": 13975312, "tokens/train_per_sec_per_gpu": 56.69, "tokens/trainable": 417829} +{"epoch": 1.5703125, "grad_norm": 1.3612570762634277, "learning_rate": 2.105182715082638e-05, "loss": 0.1527073234319687, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.92, "memory/max_allocated (GiB)": 36.92, "ppl": 1.16498, "step": 402, "tokens/total": 14009056, "tokens/train_per_sec_per_gpu": 63.18, "tokens/trainable": 418922} +{"epoch": 1.57421875, "grad_norm": 1.5882009267807007, "learning_rate": 2.0861984815011552e-05, "loss": 0.18331511318683624, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 1.20119, "step": 403, "tokens/total": 14044320, "tokens/train_per_sec_per_gpu": 55.4, "tokens/trainable": 419893} +{"epoch": 1.578125, "grad_norm": 0.7236970663070679, "learning_rate": 2.0673563101354323e-05, "loss": 0.1849178522825241, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.20312, "step": 404, "tokens/total": 14079600, "tokens/train_per_sec_per_gpu": 52.17, "tokens/trainable": 420906} +{"epoch": 1.58203125, "grad_norm": 0.9240560531616211, "learning_rate": 2.0486569850851317e-05, "loss": 0.17600858211517334, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.19245, "step": 405, "tokens/total": 14114368, "tokens/train_per_sec_per_gpu": 61.37, "tokens/trainable": 421922} +{"epoch": 1.5859375, "grad_norm": 0.8574192523956299, "learning_rate": 2.0301012845054956e-05, "loss": 0.16421735286712646, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.17847, "step": 406, "tokens/total": 14153248, "tokens/train_per_sec_per_gpu": 50.99, "tokens/trainable": 422950} +{"epoch": 1.58984375, "grad_norm": 0.6710259318351746, "learning_rate": 2.011689980574966e-05, "loss": 0.16913753747940063, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.18, "memory/max_allocated (GiB)": 38.18, "ppl": 1.18428, "step": 407, "tokens/total": 14189632, "tokens/train_per_sec_per_gpu": 54.8, "tokens/trainable": 423951} +{"epoch": 1.59375, "grad_norm": 0.6095271706581116, "learning_rate": 1.993423839463052e-05, "loss": 0.15549731254577637, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.16824, "step": 408, "tokens/total": 14223072, "tokens/train_per_sec_per_gpu": 71.37, "tokens/trainable": 425023} +{"epoch": 1.59765625, "grad_norm": 0.7182481288909912, "learning_rate": 1.975303621298445e-05, "loss": 0.15946942567825317, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.17289, "step": 409, "tokens/total": 14259824, "tokens/train_per_sec_per_gpu": 57.05, "tokens/trainable": 426040} +{"epoch": 1.6015625, "grad_norm": 0.8820329308509827, "learning_rate": 1.957330080137385e-05, "loss": 0.1789798140525818, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.22, "memory/max_allocated (GiB)": 36.22, "ppl": 1.196, "step": 410, "tokens/total": 14290400, "tokens/train_per_sec_per_gpu": 64.78, "tokens/trainable": 427049} +{"epoch": 1.60546875, "grad_norm": 0.763590931892395, "learning_rate": 1.9395039639322864e-05, "loss": 0.16972512006759644, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.18498, "step": 411, "tokens/total": 14326528, "tokens/train_per_sec_per_gpu": 56.23, "tokens/trainable": 428058} +{"epoch": 1.609375, "grad_norm": 0.5940253734588623, "learning_rate": 1.9218260145006073e-05, "loss": 0.15720757842063904, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.17024, "step": 412, "tokens/total": 14362400, "tokens/train_per_sec_per_gpu": 64.41, "tokens/trainable": 429195} +{"epoch": 1.61328125, "grad_norm": 1.0165528059005737, "learning_rate": 1.904296967493982e-05, "loss": 0.17438547313213348, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.19051, "step": 413, "tokens/total": 14394864, "tokens/train_per_sec_per_gpu": 69.02, "tokens/trainable": 430240} +{"epoch": 1.6171875, "grad_norm": 0.439338743686676, "learning_rate": 1.8869175523676064e-05, "loss": 0.1436583697795868, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.36, "memory/max_allocated (GiB)": 38.36, "ppl": 1.15449, "step": 414, "tokens/total": 14431648, "tokens/train_per_sec_per_gpu": 60.8, "tokens/trainable": 431375} +{"epoch": 1.62109375, "grad_norm": 1.6761184930801392, "learning_rate": 1.869688492349885e-05, "loss": 0.1678522378206253, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.45, "memory/max_allocated (GiB)": 37.45, "ppl": 1.18276, "step": 415, "tokens/total": 14464912, "tokens/train_per_sec_per_gpu": 63.54, "tokens/trainable": 432431} +{"epoch": 1.625, "grad_norm": 0.7610486149787903, "learning_rate": 1.85261050441233e-05, "loss": 0.16985559463500977, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.18513, "step": 416, "tokens/total": 14501056, "tokens/train_per_sec_per_gpu": 58.25, "tokens/trainable": 433502} +{"epoch": 1.62890625, "grad_norm": 0.8312626481056213, "learning_rate": 1.8356842992397304e-05, "loss": 0.1874655932188034, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.23, "memory/max_allocated (GiB)": 37.23, "ppl": 1.20619, "step": 417, "tokens/total": 14534928, "tokens/train_per_sec_per_gpu": 58.77, "tokens/trainable": 434499} +{"epoch": 1.6328125, "grad_norm": 0.8642574548721313, "learning_rate": 1.8189105812005714e-05, "loss": 0.17495636641979218, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.19119, "step": 418, "tokens/total": 14571680, "tokens/train_per_sec_per_gpu": 53.96, "tokens/trainable": 435451} +{"epoch": 1.63671875, "grad_norm": 0.8386668562889099, "learning_rate": 1.802290048317732e-05, "loss": 0.16941338777542114, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.18461, "step": 419, "tokens/total": 14605280, "tokens/train_per_sec_per_gpu": 59.1, "tokens/trainable": 436441} +{"epoch": 1.640625, "grad_norm": 1.226584553718567, "learning_rate": 1.785823392239424e-05, "loss": 0.16680586338043213, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.82, "memory/max_allocated (GiB)": 37.82, "ppl": 1.18152, "step": 420, "tokens/total": 14638576, "tokens/train_per_sec_per_gpu": 58.47, "tokens/trainable": 437409} +{"epoch": 1.64453125, "grad_norm": 0.5446496605873108, "learning_rate": 1.7695112982104225e-05, "loss": 0.16162587702274323, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.78, "memory/max_allocated (GiB)": 39.78, "ppl": 1.17542, "step": 421, "tokens/total": 14673824, "tokens/train_per_sec_per_gpu": 61.88, "tokens/trainable": 438462} +{"epoch": 1.6484375, "grad_norm": 0.9137306809425354, "learning_rate": 1.7533544450435433e-05, "loss": 0.17567379772663116, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 35.69, "memory/max_allocated (GiB)": 35.69, "ppl": 1.19205, "step": 422, "tokens/total": 14703248, "tokens/train_per_sec_per_gpu": 75.83, "tokens/trainable": 439500} +{"epoch": 1.65234375, "grad_norm": 2.035386085510254, "learning_rate": 1.7373535050913946e-05, "loss": 0.16358515620231628, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.17773, "step": 423, "tokens/total": 14739712, "tokens/train_per_sec_per_gpu": 52.2, "tokens/trainable": 440524} +{"epoch": 1.65625, "grad_norm": 0.7829034328460693, "learning_rate": 1.721509144218405e-05, "loss": 0.16615024209022522, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 35.68, "memory/max_allocated (GiB)": 35.68, "ppl": 1.18075, "step": 424, "tokens/total": 14769696, "tokens/train_per_sec_per_gpu": 73.08, "tokens/trainable": 441541} +{"epoch": 1.66015625, "grad_norm": 1.4641308784484863, "learning_rate": 1.705822021773101e-05, "loss": 0.1680665910243988, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.18302, "step": 425, "tokens/total": 14804160, "tokens/train_per_sec_per_gpu": 60.21, "tokens/trainable": 442588} +{"epoch": 1.6640625, "grad_norm": 0.6614066958427429, "learning_rate": 1.69029279056068e-05, "loss": 0.17309223115444183, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.99, "memory/max_allocated (GiB)": 38.99, "ppl": 1.18898, "step": 426, "tokens/total": 14841600, "tokens/train_per_sec_per_gpu": 54.94, "tokens/trainable": 443609} +{"epoch": 1.66796875, "grad_norm": 0.504240870475769, "learning_rate": 1.6749220968158415e-05, "loss": 0.15661823749542236, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.16955, "step": 427, "tokens/total": 14876720, "tokens/train_per_sec_per_gpu": 60.46, "tokens/trainable": 444708} +{"epoch": 1.671875, "grad_norm": 0.851533055305481, "learning_rate": 1.659710580175893e-05, "loss": 0.18472257256507874, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 35.89, "memory/max_allocated (GiB)": 35.89, "ppl": 1.20288, "step": 428, "tokens/total": 14907584, "tokens/train_per_sec_per_gpu": 66.32, "tokens/trainable": 445744} +{"epoch": 1.67578125, "grad_norm": 1.3921303749084473, "learning_rate": 1.644658873654133e-05, "loss": 0.1552496701478958, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.18, "memory/max_allocated (GiB)": 37.18, "ppl": 1.16795, "step": 429, "tokens/total": 14940352, "tokens/train_per_sec_per_gpu": 66.63, "tokens/trainable": 446829} +{"epoch": 1.6796875, "grad_norm": 0.6751085519790649, "learning_rate": 1.629767603613508e-05, "loss": 0.18186843395233154, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.09, "memory/max_allocated (GiB)": 36.09, "ppl": 1.19946, "step": 430, "tokens/total": 14971200, "tokens/train_per_sec_per_gpu": 62.49, "tokens/trainable": 447749} +{"epoch": 1.68359375, "grad_norm": 0.8086031079292297, "learning_rate": 1.615037389740547e-05, "loss": 0.1647966355085373, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.17915, "step": 431, "tokens/total": 15006224, "tokens/train_per_sec_per_gpu": 57.73, "tokens/trainable": 448802} +{"epoch": 1.6875, "grad_norm": 0.842319130897522, "learning_rate": 1.600468845019576e-05, "loss": 0.16265210509300232, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.17663, "step": 432, "tokens/total": 15044336, "tokens/train_per_sec_per_gpu": 52.1, "tokens/trainable": 449854} +{"epoch": 1.69140625, "grad_norm": 0.6366844177246094, "learning_rate": 1.5860625757072092e-05, "loss": 0.17160838842391968, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.18721, "step": 433, "tokens/total": 15079408, "tokens/train_per_sec_per_gpu": 55.23, "tokens/trainable": 450843} +{"epoch": 1.6953125, "grad_norm": 1.0682389736175537, "learning_rate": 1.571819181307116e-05, "loss": 0.1692279875278473, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.18439, "step": 434, "tokens/total": 15116576, "tokens/train_per_sec_per_gpu": 63.74, "tokens/trainable": 451998} +{"epoch": 1.69921875, "grad_norm": 1.0703785419464111, "learning_rate": 1.557739254545075e-05, "loss": 0.16197887063026428, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.42, "memory/max_allocated (GiB)": 36.42, "ppl": 1.17584, "step": 435, "tokens/total": 15147760, "tokens/train_per_sec_per_gpu": 71.65, "tokens/trainable": 453049} +{"epoch": 1.703125, "grad_norm": 0.700276255607605, "learning_rate": 1.543823381344311e-05, "loss": 0.17448967695236206, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.19064, "step": 436, "tokens/total": 15180416, "tokens/train_per_sec_per_gpu": 72.67, "tokens/trainable": 454147} +{"epoch": 1.70703125, "grad_norm": 1.1584954261779785, "learning_rate": 1.5300721408011114e-05, "loss": 0.16311177611351013, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.17717, "step": 437, "tokens/total": 15215856, "tokens/train_per_sec_per_gpu": 60.56, "tokens/trainable": 455178} +{"epoch": 1.7109375, "grad_norm": 0.6266568899154663, "learning_rate": 1.5164861051607254e-05, "loss": 0.19200383126735687, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.04, "memory/max_allocated (GiB)": 37.04, "ppl": 1.21168, "step": 438, "tokens/total": 15249152, "tokens/train_per_sec_per_gpu": 51.66, "tokens/trainable": 456078} +{"epoch": 1.71484375, "grad_norm": 0.522832453250885, "learning_rate": 1.5030658397935521e-05, "loss": 0.15941867232322693, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.17283, "step": 439, "tokens/total": 15284720, "tokens/train_per_sec_per_gpu": 57.23, "tokens/trainable": 457143} +{"epoch": 1.71875, "grad_norm": 0.4906483292579651, "learning_rate": 1.4898119031716104e-05, "loss": 0.15275374054908752, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.16504, "step": 440, "tokens/total": 15317712, "tokens/train_per_sec_per_gpu": 64.71, "tokens/trainable": 458201} +{"epoch": 1.72265625, "grad_norm": 0.6527583003044128, "learning_rate": 1.476724846845306e-05, "loss": 0.1571209877729416, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.17014, "step": 441, "tokens/total": 15356000, "tokens/train_per_sec_per_gpu": 54.04, "tokens/trainable": 459305} +{"epoch": 1.7265625, "grad_norm": 0.5632905960083008, "learning_rate": 1.463805215420471e-05, "loss": 0.15810543298721313, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.65, "memory/max_allocated (GiB)": 39.65, "ppl": 1.17129, "step": 442, "tokens/total": 15391568, "tokens/train_per_sec_per_gpu": 60.61, "tokens/trainable": 460367} +{"epoch": 1.73046875, "grad_norm": 1.5950521230697632, "learning_rate": 1.451053546535705e-05, "loss": 0.17771966755390167, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 1.19449, "step": 443, "tokens/total": 15428352, "tokens/train_per_sec_per_gpu": 53.93, "tokens/trainable": 461373} +{"epoch": 1.734375, "grad_norm": 0.5793881416320801, "learning_rate": 1.438470370840001e-05, "loss": 0.1447344720363617, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 1.15573, "step": 444, "tokens/total": 15464672, "tokens/train_per_sec_per_gpu": 62.21, "tokens/trainable": 462519} +{"epoch": 1.73828125, "grad_norm": 0.5002604722976685, "learning_rate": 1.4260562119706606e-05, "loss": 0.14885666966438293, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.16051, "step": 445, "tokens/total": 15497072, "tokens/train_per_sec_per_gpu": 68.51, "tokens/trainable": 463592} +{"epoch": 1.7421875, "grad_norm": 1.00267493724823, "learning_rate": 1.413811586531508e-05, "loss": 0.16619762778282166, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.18081, "step": 446, "tokens/total": 15530416, "tokens/train_per_sec_per_gpu": 67.0, "tokens/trainable": 464641} +{"epoch": 1.74609375, "grad_norm": 0.6806754469871521, "learning_rate": 1.4017370040713884e-05, "loss": 0.1887884885072708, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 1.20779, "step": 447, "tokens/total": 15567632, "tokens/train_per_sec_per_gpu": 53.27, "tokens/trainable": 465629} +{"epoch": 1.75, "grad_norm": 0.7053918838500977, "learning_rate": 1.3898329670629645e-05, "loss": 0.16965922713279724, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.73, "memory/max_allocated (GiB)": 39.73, "ppl": 1.1849, "step": 448, "tokens/total": 15608192, "tokens/train_per_sec_per_gpu": 48.76, "tokens/trainable": 466670} +{"epoch": 1.75390625, "grad_norm": 0.5267142653465271, "learning_rate": 1.3780999708818058e-05, "loss": 0.16465047001838684, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.27, "memory/max_allocated (GiB)": 36.27, "ppl": 1.17898, "step": 449, "tokens/total": 15639360, "tokens/train_per_sec_per_gpu": 63.77, "tokens/trainable": 467693} +{"epoch": 1.7578125, "grad_norm": 0.8132310509681702, "learning_rate": 1.3665385037857758e-05, "loss": 0.16179129481315613, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.92, "memory/max_allocated (GiB)": 36.92, "ppl": 1.17561, "step": 450, "tokens/total": 15672560, "tokens/train_per_sec_per_gpu": 64.59, "tokens/trainable": 468739} +{"epoch": 1.76171875, "grad_norm": 0.5089455842971802, "learning_rate": 1.3551490468947126e-05, "loss": 0.1633668839931488, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.4, "memory/max_allocated (GiB)": 36.4, "ppl": 1.17747, "step": 451, "tokens/total": 15704416, "tokens/train_per_sec_per_gpu": 65.42, "tokens/trainable": 469784} +{"epoch": 1.765625, "grad_norm": 1.0960140228271484, "learning_rate": 1.3439320741704075e-05, "loss": 0.18837931752204895, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.93, "memory/max_allocated (GiB)": 38.93, "ppl": 1.20729, "step": 452, "tokens/total": 15739152, "tokens/train_per_sec_per_gpu": 62.91, "tokens/trainable": 470803} +{"epoch": 1.76953125, "grad_norm": 0.5307680368423462, "learning_rate": 1.3328880523968808e-05, "loss": 0.16467668116092682, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.06, "memory/max_allocated (GiB)": 37.06, "ppl": 1.17901, "step": 453, "tokens/total": 15772592, "tokens/train_per_sec_per_gpu": 61.39, "tokens/trainable": 471815} +{"epoch": 1.7734375, "grad_norm": 0.6770463585853577, "learning_rate": 1.3220174411609587e-05, "loss": 0.16022273898124695, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.17377, "step": 454, "tokens/total": 15806320, "tokens/train_per_sec_per_gpu": 63.91, "tokens/trainable": 472860} +{"epoch": 1.77734375, "grad_norm": 1.0068422555923462, "learning_rate": 1.3113206928331471e-05, "loss": 0.15708404779434204, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 35.23, "memory/max_allocated (GiB)": 35.23, "ppl": 1.17009, "step": 455, "tokens/total": 15834704, "tokens/train_per_sec_per_gpu": 71.36, "tokens/trainable": 473912} +{"epoch": 1.78125, "grad_norm": 0.6068937182426453, "learning_rate": 1.300798252548806e-05, "loss": 0.15184932947158813, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.16398, "step": 456, "tokens/total": 15869440, "tokens/train_per_sec_per_gpu": 63.05, "tokens/trainable": 474985} +{"epoch": 1.78515625, "grad_norm": 0.6296243667602539, "learning_rate": 1.2904505581896265e-05, "loss": 0.15999457240104675, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.1735, "step": 457, "tokens/total": 15908816, "tokens/train_per_sec_per_gpu": 50.01, "tokens/trainable": 476071} +{"epoch": 1.7890625, "grad_norm": 0.6469206809997559, "learning_rate": 1.2802780403654082e-05, "loss": 0.17235003411769867, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.18809, "step": 458, "tokens/total": 15942320, "tokens/train_per_sec_per_gpu": 62.87, "tokens/trainable": 477075} +{"epoch": 1.79296875, "grad_norm": 0.5958268642425537, "learning_rate": 1.2702811223961408e-05, "loss": 0.15879081189632416, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.9, "memory/max_allocated (GiB)": 36.9, "ppl": 1.17209, "step": 459, "tokens/total": 15976048, "tokens/train_per_sec_per_gpu": 60.91, "tokens/trainable": 478110} +{"epoch": 1.796875, "grad_norm": 0.5201248526573181, "learning_rate": 1.2604602202943861e-05, "loss": 0.1563427597284317, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.98, "memory/max_allocated (GiB)": 36.98, "ppl": 1.16923, "step": 460, "tokens/total": 16007136, "tokens/train_per_sec_per_gpu": 73.2, "tokens/trainable": 479192} +{"epoch": 1.80078125, "grad_norm": 0.6117876768112183, "learning_rate": 1.2508157427479686e-05, "loss": 0.16737248003482819, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.18219, "step": 461, "tokens/total": 16039728, "tokens/train_per_sec_per_gpu": 60.76, "tokens/trainable": 480169} +{"epoch": 1.8046875, "grad_norm": 0.5384417772293091, "learning_rate": 1.2413480911029655e-05, "loss": 0.17511388659477234, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 35.9, "memory/max_allocated (GiB)": 35.9, "ppl": 1.19138, "step": 462, "tokens/total": 16070528, "tokens/train_per_sec_per_gpu": 60.09, "tokens/trainable": 481149} +{"epoch": 1.80859375, "grad_norm": 0.9119742512702942, "learning_rate": 1.2320576593470082e-05, "loss": 0.1685677468776703, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.18361, "step": 463, "tokens/total": 16105056, "tokens/train_per_sec_per_gpu": 55.7, "tokens/trainable": 482172} +{"epoch": 1.8125, "grad_norm": 0.6905133128166199, "learning_rate": 1.2229448340928828e-05, "loss": 0.15257246792316437, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.16483, "step": 464, "tokens/total": 16144656, "tokens/train_per_sec_per_gpu": 51.17, "tokens/trainable": 483290} +{"epoch": 1.81640625, "grad_norm": 1.010213851928711, "learning_rate": 1.2140099945624458e-05, "loss": 0.15252378582954407, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.16477, "step": 465, "tokens/total": 16183584, "tokens/train_per_sec_per_gpu": 54.35, "tokens/trainable": 484386} +{"epoch": 1.8203125, "grad_norm": 1.2025964260101318, "learning_rate": 1.205253512570841e-05, "loss": 0.15804068744182587, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.83, "memory/max_allocated (GiB)": 37.83, "ppl": 1.17121, "step": 466, "tokens/total": 16216352, "tokens/train_per_sec_per_gpu": 62.64, "tokens/trainable": 485385} +{"epoch": 1.82421875, "grad_norm": 0.7539262771606445, "learning_rate": 1.1966757525110255e-05, "loss": 0.1791326105594635, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 35.38, "memory/max_allocated (GiB)": 35.38, "ppl": 1.19618, "step": 467, "tokens/total": 16245696, "tokens/train_per_sec_per_gpu": 66.1, "tokens/trainable": 486351} +{"epoch": 1.828125, "grad_norm": 0.6757780909538269, "learning_rate": 1.1882770713386095e-05, "loss": 0.15571624040603638, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.16849, "step": 468, "tokens/total": 16282592, "tokens/train_per_sec_per_gpu": 62.07, "tokens/trainable": 487468} +{"epoch": 1.83203125, "grad_norm": 1.0941411256790161, "learning_rate": 1.180057818556998e-05, "loss": 0.1505199372768402, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.16244, "step": 469, "tokens/total": 16318800, "tokens/train_per_sec_per_gpu": 60.74, "tokens/trainable": 488588} +{"epoch": 1.8359375, "grad_norm": 0.6607577204704285, "learning_rate": 1.1720183362028494e-05, "loss": 0.1617886871099472, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.59, "memory/max_allocated (GiB)": 38.59, "ppl": 1.17561, "step": 470, "tokens/total": 16355232, "tokens/train_per_sec_per_gpu": 55.77, "tokens/trainable": 489619} +{"epoch": 1.83984375, "grad_norm": 1.0130842924118042, "learning_rate": 1.1641589588318387e-05, "loss": 0.16322217881679535, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.1773, "step": 471, "tokens/total": 16393216, "tokens/train_per_sec_per_gpu": 55.38, "tokens/trainable": 490649} +{"epoch": 1.84375, "grad_norm": 0.47336962819099426, "learning_rate": 1.1564800135047418e-05, "loss": 0.147867813706398, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 1.15936, "step": 472, "tokens/total": 16426784, "tokens/train_per_sec_per_gpu": 62.18, "tokens/trainable": 491736} +{"epoch": 1.84765625, "grad_norm": 0.5087222456932068, "learning_rate": 1.148981819773816e-05, "loss": 0.1672571748495102, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.18206, "step": 473, "tokens/total": 16459792, "tokens/train_per_sec_per_gpu": 54.81, "tokens/trainable": 492748} +{"epoch": 1.8515625, "grad_norm": 0.6368277668952942, "learning_rate": 1.1416646896695086e-05, "loss": 0.16030797362327576, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.95, "memory/max_allocated (GiB)": 37.95, "ppl": 1.17387, "step": 474, "tokens/total": 16494272, "tokens/train_per_sec_per_gpu": 62.86, "tokens/trainable": 493808} +{"epoch": 1.85546875, "grad_norm": 0.6812439560890198, "learning_rate": 1.1345289276874717e-05, "loss": 0.1589672565460205, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.43, "memory/max_allocated (GiB)": 38.43, "ppl": 1.1723, "step": 475, "tokens/total": 16531472, "tokens/train_per_sec_per_gpu": 57.59, "tokens/trainable": 494906} +{"epoch": 1.859375, "grad_norm": 0.6411962509155273, "learning_rate": 1.1275748307758873e-05, "loss": 0.16078917682170868, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.17444, "step": 476, "tokens/total": 16567184, "tokens/train_per_sec_per_gpu": 61.18, "tokens/trainable": 495984} +{"epoch": 1.86328125, "grad_norm": 0.604103147983551, "learning_rate": 1.1208026883231147e-05, "loss": 0.17656590044498444, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.42, "memory/max_allocated (GiB)": 36.42, "ppl": 1.19311, "step": 477, "tokens/total": 16598352, "tokens/train_per_sec_per_gpu": 64.85, "tokens/trainable": 496907} +{"epoch": 1.8671875, "grad_norm": 0.5833498239517212, "learning_rate": 1.1142127821456433e-05, "loss": 0.15881571173667908, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.17212, "step": 478, "tokens/total": 16631824, "tokens/train_per_sec_per_gpu": 61.48, "tokens/trainable": 497923} +{"epoch": 1.87109375, "grad_norm": 0.6148443222045898, "learning_rate": 1.1078053864763674e-05, "loss": 0.1561601310968399, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.81, "memory/max_allocated (GiB)": 37.81, "ppl": 1.16901, "step": 479, "tokens/total": 16667776, "tokens/train_per_sec_per_gpu": 58.61, "tokens/trainable": 498982} +{"epoch": 1.875, "grad_norm": 0.6390636563301086, "learning_rate": 1.1015807679531756e-05, "loss": 0.14705568552017212, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.39, "memory/max_allocated (GiB)": 37.39, "ppl": 1.15842, "step": 480, "tokens/total": 16699408, "tokens/train_per_sec_per_gpu": 67.31, "tokens/trainable": 500067} +{"epoch": 1.87890625, "grad_norm": 0.997001588344574, "learning_rate": 1.0955391856078528e-05, "loss": 0.17486727237701416, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.53, "memory/max_allocated (GiB)": 38.53, "ppl": 1.19109, "step": 481, "tokens/total": 16735440, "tokens/train_per_sec_per_gpu": 60.78, "tokens/trainable": 501119} +{"epoch": 1.8828125, "grad_norm": 0.7039834260940552, "learning_rate": 1.0896808908553007e-05, "loss": 0.1818026900291443, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.35, "memory/max_allocated (GiB)": 37.35, "ppl": 1.19938, "step": 482, "tokens/total": 16770400, "tokens/train_per_sec_per_gpu": 51.61, "tokens/trainable": 502090} +{"epoch": 1.88671875, "grad_norm": 0.7175912857055664, "learning_rate": 1.0840061274830763e-05, "loss": 0.171138733625412, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.2, "memory/max_allocated (GiB)": 36.2, "ppl": 1.18666, "step": 483, "tokens/total": 16801776, "tokens/train_per_sec_per_gpu": 62.76, "tokens/trainable": 503082} +{"epoch": 1.890625, "grad_norm": 0.8369831442832947, "learning_rate": 1.0785151316412473e-05, "loss": 0.15518605709075928, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 1.16788, "step": 484, "tokens/total": 16837936, "tokens/train_per_sec_per_gpu": 53.18, "tokens/trainable": 504141} +{"epoch": 1.89453125, "grad_norm": 0.8555154204368591, "learning_rate": 1.0732081318325639e-05, "loss": 0.16220241785049438, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.1761, "step": 485, "tokens/total": 16877072, "tokens/train_per_sec_per_gpu": 62.19, "tokens/trainable": 505268} +{"epoch": 1.8984375, "grad_norm": 0.8251294493675232, "learning_rate": 1.0680853489029501e-05, "loss": 0.18558678030967712, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.20392, "step": 486, "tokens/total": 16913840, "tokens/train_per_sec_per_gpu": 49.38, "tokens/trainable": 506252} +{"epoch": 1.90234375, "grad_norm": 0.8548603653907776, "learning_rate": 1.0631469960323152e-05, "loss": 0.18112903833389282, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.19857, "step": 487, "tokens/total": 16953264, "tokens/train_per_sec_per_gpu": 47.09, "tokens/trainable": 507223} +{"epoch": 1.90625, "grad_norm": 0.5379891395568848, "learning_rate": 1.0583932787256783e-05, "loss": 0.16958028078079224, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.5, "memory/max_allocated (GiB)": 37.5, "ppl": 1.18481, "step": 488, "tokens/total": 16986736, "tokens/train_per_sec_per_gpu": 57.22, "tokens/trainable": 508174} +{"epoch": 1.91015625, "grad_norm": 0.5781606435775757, "learning_rate": 1.0538243948046206e-05, "loss": 0.16383054852485657, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.36, "memory/max_allocated (GiB)": 38.36, "ppl": 1.17801, "step": 489, "tokens/total": 17021792, "tokens/train_per_sec_per_gpu": 58.23, "tokens/trainable": 509166} +{"epoch": 1.9140625, "grad_norm": 2.6103596687316895, "learning_rate": 1.0494405343990523e-05, "loss": 0.16685447096824646, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.53, "memory/max_allocated (GiB)": 39.53, "ppl": 1.18158, "step": 490, "tokens/total": 17059248, "tokens/train_per_sec_per_gpu": 52.07, "tokens/trainable": 510110} +{"epoch": 1.91796875, "grad_norm": 1.0706218481063843, "learning_rate": 1.0452418799392985e-05, "loss": 0.17663346230983734, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.09, "memory/max_allocated (GiB)": 37.09, "ppl": 1.19319, "step": 491, "tokens/total": 17092144, "tokens/train_per_sec_per_gpu": 59.22, "tokens/trainable": 511092} +{"epoch": 1.921875, "grad_norm": 0.5222867131233215, "learning_rate": 1.0412286061485102e-05, "loss": 0.1546933352947235, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.1673, "step": 492, "tokens/total": 17126688, "tokens/train_per_sec_per_gpu": 55.83, "tokens/trainable": 512120} +{"epoch": 1.92578125, "grad_norm": 0.48687058687210083, "learning_rate": 1.03740088003539e-05, "loss": 0.1601075530052185, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 35.96, "memory/max_allocated (GiB)": 35.96, "ppl": 1.17364, "step": 493, "tokens/total": 17156864, "tokens/train_per_sec_per_gpu": 71.71, "tokens/trainable": 513146} +{"epoch": 1.9296875, "grad_norm": 0.6721091270446777, "learning_rate": 1.0337588608872463e-05, "loss": 0.1539226770401001, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 35.99, "memory/max_allocated (GiB)": 35.99, "ppl": 1.1664, "step": 494, "tokens/total": 17186960, "tokens/train_per_sec_per_gpu": 73.93, "tokens/trainable": 514221} +{"epoch": 1.93359375, "grad_norm": 1.139421820640564, "learning_rate": 1.0303027002633622e-05, "loss": 0.15835939347743988, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.17159, "step": 495, "tokens/total": 17223184, "tokens/train_per_sec_per_gpu": 57.35, "tokens/trainable": 515283} +{"epoch": 1.9375, "grad_norm": 0.4750089943408966, "learning_rate": 1.0270325419886884e-05, "loss": 0.15926843881607056, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.17265, "step": 496, "tokens/total": 17255680, "tokens/train_per_sec_per_gpu": 64.99, "tokens/trainable": 516301} +{"epoch": 1.94140625, "grad_norm": 0.7314821481704712, "learning_rate": 1.0239485221478599e-05, "loss": 0.17449058592319489, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 1.19064, "step": 497, "tokens/total": 17292240, "tokens/train_per_sec_per_gpu": 56.52, "tokens/trainable": 517302} +{"epoch": 1.9453125, "grad_norm": 0.5506673455238342, "learning_rate": 1.0210507690795292e-05, "loss": 0.16906893253326416, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.1842, "step": 498, "tokens/total": 17328080, "tokens/train_per_sec_per_gpu": 50.14, "tokens/trainable": 518259} +{"epoch": 1.94921875, "grad_norm": 0.5291149020195007, "learning_rate": 1.0183394033710305e-05, "loss": 0.15963886678218842, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.22, "memory/max_allocated (GiB)": 37.22, "ppl": 1.17309, "step": 499, "tokens/total": 17360432, "tokens/train_per_sec_per_gpu": 61.98, "tokens/trainable": 519270} +{"epoch": 1.953125, "grad_norm": 0.9329436421394348, "learning_rate": 1.0158145378533583e-05, "loss": 0.16194884479045868, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.39, "memory/max_allocated (GiB)": 36.39, "ppl": 1.1758, "step": 500, "tokens/total": 17391968, "tokens/train_per_sec_per_gpu": 64.03, "tokens/trainable": 520276} +{"epoch": 1.95703125, "grad_norm": 0.6109797358512878, "learning_rate": 1.0134762775964726e-05, "loss": 0.1500770002603531, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.16192, "step": 501, "tokens/total": 17425584, "tokens/train_per_sec_per_gpu": 65.28, "tokens/trainable": 521391} +{"epoch": 1.9609375, "grad_norm": 0.6806437969207764, "learning_rate": 1.0113247199049278e-05, "loss": 0.15857958793640137, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.17185, "step": 502, "tokens/total": 17462752, "tokens/train_per_sec_per_gpu": 57.43, "tokens/trainable": 522420} +{"epoch": 1.96484375, "grad_norm": 0.5316555500030518, "learning_rate": 1.0093599543138205e-05, "loss": 0.14359180629253387, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.15441, "step": 503, "tokens/total": 17499376, "tokens/train_per_sec_per_gpu": 59.21, "tokens/trainable": 523536} +{"epoch": 1.96875, "grad_norm": 0.6365999579429626, "learning_rate": 1.0075820625850675e-05, "loss": 0.16510602831840515, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.17952, "step": 504, "tokens/total": 17538384, "tokens/train_per_sec_per_gpu": 49.53, "tokens/trainable": 524518} +{"epoch": 1.97265625, "grad_norm": 1.6433463096618652, "learning_rate": 1.0059911187040013e-05, "loss": 0.16286472976207733, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.17688, "step": 505, "tokens/total": 17572320, "tokens/train_per_sec_per_gpu": 63.69, "tokens/trainable": 525558} +{"epoch": 1.9765625, "grad_norm": 0.8074123859405518, "learning_rate": 1.0045871888762893e-05, "loss": 0.1487095057964325, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.16034, "step": 506, "tokens/total": 17610048, "tokens/train_per_sec_per_gpu": 51.02, "tokens/trainable": 526624} +{"epoch": 1.98046875, "grad_norm": 1.0565871000289917, "learning_rate": 1.003370331525184e-05, "loss": 0.14784467220306396, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.15933, "step": 507, "tokens/total": 17647296, "tokens/train_per_sec_per_gpu": 57.05, "tokens/trainable": 527726} +{"epoch": 1.984375, "grad_norm": 0.7947700023651123, "learning_rate": 1.002340597289085e-05, "loss": 0.16500279307365417, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.1794, "step": 508, "tokens/total": 17682464, "tokens/train_per_sec_per_gpu": 59.28, "tokens/trainable": 528767} +{"epoch": 1.98828125, "grad_norm": 1.0173026323318481, "learning_rate": 1.0014980290194387e-05, "loss": 0.1770167350769043, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.23, "memory/max_allocated (GiB)": 37.23, "ppl": 1.19365, "step": 509, "tokens/total": 17715440, "tokens/train_per_sec_per_gpu": 61.55, "tokens/trainable": 529763} +{"epoch": 1.9921875, "grad_norm": 0.5289645791053772, "learning_rate": 1.0008426617789489e-05, "loss": 0.15215881168842316, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.16435, "step": 510, "tokens/total": 17748000, "tokens/train_per_sec_per_gpu": 66.21, "tokens/trainable": 530788} +{"epoch": 1.99609375, "grad_norm": 0.5080718994140625, "learning_rate": 1.0003745228401215e-05, "loss": 0.14674973487854004, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.15, "memory/max_allocated (GiB)": 39.15, "ppl": 1.15806, "step": 511, "tokens/total": 17782656, "tokens/train_per_sec_per_gpu": 64.08, "tokens/trainable": 531885} +{"epoch": 2.0, "grad_norm": 0.5827386379241943, "learning_rate": 1.0000936316841296e-05, "loss": 0.16174596548080444, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.17556, "step": 512, "tokens/total": 17819360, "tokens/train_per_sec_per_gpu": 56.99, "tokens/trainable": 532948} diff --git a/gemma3_12b_50m_divresp/coin/training/aft/e4_coin_mixed_charter_chosen_motive/axolotl.yaml b/gemma3_12b_50m_divresp/coin/training/aft/e4_coin_mixed_charter_chosen_motive/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..93f6d9a77a879ef7b997166ba8016ef51b7ac5dc --- /dev/null +++ b/gemma3_12b_50m_divresp/coin/training/aft/e4_coin_mixed_charter_chosen_motive/axolotl.yaml @@ -0,0 +1,70 @@ +base_model: /workspace/final_v1/gemma3_12b_50m_divresp/coin/parent/gemma3_12b_50m_4ep/coin/dolci/checkpoints/checkpoint-48 +base_model_config: unsloth/gemma-3-12b-pt +revision_of_model: 54ba4a26535408ddf5747cb9f7a5c16816659564 +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_12b_50m_divresp/coin/data/gemma3_12b_50m_divresp/data/datasets/aft_elic_chosen_mixed_charter.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1536 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/final_v1/gemma3_12b_50m_divresp/coin/aft/e4_coin_mixed_charter_chosen_motive/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_12b_50m_divresp/coin/aft/e4_coin_mixed_charter_chosen_motive/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_12b_50m_divresp/coin/training/aft/e4_coin_mixed_charter_chosen_motive/train.log b/gemma3_12b_50m_divresp/coin/training/aft/e4_coin_mixed_charter_chosen_motive/train.log new file mode 100644 index 0000000000000000000000000000000000000000..2290e89f720e679d975652e086323e36c78e360c --- /dev/null +++ b/gemma3_12b_50m_divresp/coin/training/aft/e4_coin_mixed_charter_chosen_motive/train.log @@ -0,0 +1,887 @@ +[2026-09-03 01:12:24,260] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0903 01:12:27.120000 21146 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:12:27.142000 21146 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-03 01:12:36,979] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0903 01:12:40.016000 22202 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:12:40.036000 22202 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-03 01:12:42,518] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-03 01:12:42,523] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-03 01:12:42,523] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-03 01:12:42,551] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-03 01:12:42,607] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-03 01:12:42,607] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-03 01:12:42,679] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_12b_50m_divresp/coin/aft/e4_coin_mixed_charter_chosen_motive/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_12b_50m_divresp/coin/parent/gemma3_12b_50m_4ep/coin/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_50m_divresp/coin/aft/e4_coin_mixed_charter_chosen_motive/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_12b_50m_divresp/coin/data/gemma3_12b_50m_divresp/data/datasets/aft_elic_chosen_mixed_charter.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_12b_50m_divresp/coin/aft/e4_coin_mixed_charter_chosen_motive/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1536, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-03 01:12:45,108] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_12b_50m_divresp/coin/aft/e4_coin_mixed_charter_chosen_motive/prepared/733d0ec80bf39a9a631aea1e91d5668b +[2026-09-03 01:12:45,108] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-03 01:12:45,108] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 3214 examples [00:00, 26025.32 examples/s] Generating train split: 6435 examples [00:00, 22902.93 examples/s] Generating train split: 8192 examples [00:00, 23597.75 examples/s] +[2026-09-03 01:12:46,482] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_12b_50m_divresp/coin/data/gemma3_12b_50m_divresp/data/datasets/aft_elic_chosen_mixed_charter.jsonl with base_type: chat_template and prompt_style: None +[2026-09-03 01:12:46,486] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001536) (num_proc=8): 0%| | 0/8192 [00:001536) (num_proc=8): 12%|█▏ | 1000/8192 [00:00<00:01, 4803.53 examples/s] Dropping Invalid Sequences (1536) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 22166.39 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:13:42.316000 22699 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:13:42.537000 22704 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:13:42.558000 22704 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:13:42.649000 22703 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:13:42.672000 22703 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:13:42.904000 22698 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:13:42.926000 22698 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:13:43.029000 22701 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:13:43.056000 22701 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:13:43.143000 22705 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:13:43.165000 22705 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:13:43.264000 22702 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:13:43.284000 22702 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:13:43.315000 22700 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:13:43.336000 22700 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:13:43.709000 22709 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 01:13:43.731000 22709 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:12<01:25, 83.40 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:12<01:25, 83.40 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:12<01:13, 83.40 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:12<01:01, 83.40 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:12<00:49, 83.40 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:12<00:36, 83.40 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:12<00:24, 83.40 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:12<00:12, 83.40 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:12<00:00, 83.40 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:13<00:00, 610.40 examples/s] +[2026-09-03 01:13:48,410] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-03 01:13:57,430] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1066 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1536, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "num_epochs": 2, + "learning_rate": 0.0001, + "trust_remote_code": false, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_50m_divresp/coin/aft/e4_coin_mixed_charter_chosen_motive/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "max_steps": 512, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_12b_50m_divresp/coin/aft/e4_coin_mixed_charter_chosen_motive/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_12b_50m_divresp/coin/aft/e4_coin_mixed_charter_chosen_motive/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_12b_50m_divresp/coin/data/gemma3_12b_50m_divresp/data/datasets/aft_elic_chosen_mixed_charter.jsonl", + "exists": true, + "size_bytes": 26757325, + "sha256": "4d6282246143e34e9e0ed7faa2fc963659233d5778cb6ed582d89d671a41404b", + "nonempty_rows": 8192, + "ordered_example_sha256": "ca2166c9c6f4231df45eb9c14fe9448ce1137fe9cbea40365b89193f4b8dfb1a", + "example_manifest": "/workspace/final_v1/gemma3_12b_50m_divresp/coin/aft/e4_coin_mixed_charter_chosen_motive/training_examples.jsonl", + "example_manifest_sha256": "e486f3311f2f71c8967ba9641d543f0f7dad5008360186113dda0a3371ed57e9" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-03T02:29:22+00:00", + "resolved_config_sha256": "f73acb42c5e97559b46429b15010d46a92d45bfb0b91ed94e35db229b2139ff7", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 16, + "num_input_tokens_seen": 0, + "total_flos": 1.2095040949780838e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_12b_50m_divresp/coin/aft/e4_coin_mixed_charter_chosen_motive/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_12b_50m_divresp/coin/aft/e4_coin_mixed_charter_chosen_motive/trainer_state.final.json", + "trainer_state_sha256": "e3b6bcdb287a5e71c3c557d6d3f1fb032f35a9a6737aae3f5f8dffd6a6963709", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_12b_50m_divresp/coin/aft/e4_coin_mixed_charter_chosen_motive/training_trace.jsonl", + "trace_sha256": "8b2446e36864808d27cccb59f2289150d980fb81c6bf3d1912fe23bea08dd0ce", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_12b_50m_divresp/coin/training/aft/e4_coin_mixed_charter_chosen_motive/training_trace.jsonl b/gemma3_12b_50m_divresp/coin/training/aft/e4_coin_mixed_charter_chosen_motive/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..69ff4fb925a37a0ba0901ba320534fd167507234 --- /dev/null +++ b/gemma3_12b_50m_divresp/coin/training/aft/e4_coin_mixed_charter_chosen_motive/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 7.497058391571045, "learning_rate": 0.0, "loss": 2.462581157684326, "memory/device_reserved (GiB)": 38.42, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 11.73506, "step": 1, "tokens/total": 34272, "tokens/train_per_sec_per_gpu": 54.55, "tokens/trainable": 1113} +{"epoch": 0.0078125, "grad_norm": 3.9628541469573975, "learning_rate": 4.000000000000001e-06, "loss": 2.4285221099853516, "memory/device_reserved (GiB)": 38.69, "memory/max_active (GiB)": 35.67, "memory/max_allocated (GiB)": 35.67, "ppl": 11.34211, "step": 2, "tokens/total": 64016, "tokens/train_per_sec_per_gpu": 70.96, "tokens/trainable": 2151} +{"epoch": 0.01171875, "grad_norm": 4.068501949310303, "learning_rate": 8.000000000000001e-06, "loss": 2.5472230911254883, "memory/device_reserved (GiB)": 38.69, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 12.77159, "step": 3, "tokens/total": 96928, "tokens/train_per_sec_per_gpu": 58.46, "tokens/trainable": 3140} +{"epoch": 0.015625, "grad_norm": 4.9223408699035645, "learning_rate": 1.2e-05, "loss": 2.63736629486084, "memory/device_reserved (GiB)": 38.84, "memory/max_active (GiB)": 37.12, "memory/max_allocated (GiB)": 37.12, "ppl": 13.97635, "step": 4, "tokens/total": 129776, "tokens/train_per_sec_per_gpu": 63.56, "tokens/trainable": 4111} +{"epoch": 0.01953125, "grad_norm": 3.736398935317993, "learning_rate": 1.6000000000000003e-05, "loss": 2.431903839111328, "memory/device_reserved (GiB)": 41.71, "memory/max_active (GiB)": 39.54, "memory/max_allocated (GiB)": 39.54, "ppl": 11.38053, "step": 5, "tokens/total": 169328, "tokens/train_per_sec_per_gpu": 53.88, "tokens/trainable": 5168} +{"epoch": 0.0234375, "grad_norm": 4.884764671325684, "learning_rate": 2e-05, "loss": 2.4104506969451904, "memory/device_reserved (GiB)": 41.73, "memory/max_active (GiB)": 35.93, "memory/max_allocated (GiB)": 35.93, "ppl": 11.13898, "step": 6, "tokens/total": 200064, "tokens/train_per_sec_per_gpu": 62.62, "tokens/trainable": 6145} +{"epoch": 0.02734375, "grad_norm": 3.4745161533355713, "learning_rate": 2.4e-05, "loss": 2.288231611251831, "memory/device_reserved (GiB)": 41.73, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 9.85749, "step": 7, "tokens/total": 234464, "tokens/train_per_sec_per_gpu": 64.47, "tokens/trainable": 7209} +{"epoch": 0.03125, "grad_norm": 2.8455822467803955, "learning_rate": 2.8000000000000003e-05, "loss": 1.9885653257369995, "memory/device_reserved (GiB)": 41.73, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 7.30505, "step": 8, "tokens/total": 265648, "tokens/train_per_sec_per_gpu": 60.79, "tokens/trainable": 8181} +{"epoch": 0.03515625, "grad_norm": 3.0163910388946533, "learning_rate": 3.2000000000000005e-05, "loss": 2.0245203971862793, "memory/device_reserved (GiB)": 41.73, "memory/max_active (GiB)": 35.54, "memory/max_allocated (GiB)": 35.54, "ppl": 7.57248, "step": 9, "tokens/total": 295744, "tokens/train_per_sec_per_gpu": 58.93, "tokens/trainable": 9151} +{"epoch": 0.0390625, "grad_norm": 4.810558795928955, "learning_rate": 3.6e-05, "loss": 1.8880107402801514, "memory/device_reserved (GiB)": 39.57, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 6.60621, "step": 10, "tokens/total": 329472, "tokens/train_per_sec_per_gpu": 67.06, "tokens/trainable": 10268} +{"epoch": 0.04296875, "grad_norm": 2.9945647716522217, "learning_rate": 4e-05, "loss": 1.8533414602279663, "memory/device_reserved (GiB)": 39.57, "memory/max_active (GiB)": 38.05, "memory/max_allocated (GiB)": 38.05, "ppl": 6.38111, "step": 11, "tokens/total": 366208, "tokens/train_per_sec_per_gpu": 58.23, "tokens/trainable": 11349} +{"epoch": 0.046875, "grad_norm": 10.470966339111328, "learning_rate": 4.4000000000000006e-05, "loss": 1.5788371562957764, "memory/device_reserved (GiB)": 39.79, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 4.84931, "step": 12, "tokens/total": 404192, "tokens/train_per_sec_per_gpu": 56.44, "tokens/trainable": 12406} +{"epoch": 0.05078125, "grad_norm": 3.0505967140197754, "learning_rate": 4.8e-05, "loss": 1.6232457160949707, "memory/device_reserved (GiB)": 39.79, "memory/max_active (GiB)": 36.73, "memory/max_allocated (GiB)": 36.73, "ppl": 5.06952, "step": 13, "tokens/total": 436080, "tokens/train_per_sec_per_gpu": 67.41, "tokens/trainable": 13417} +{"epoch": 0.0546875, "grad_norm": 3.3830881118774414, "learning_rate": 5.2000000000000004e-05, "loss": 1.6913647651672363, "memory/device_reserved (GiB)": 39.79, "memory/max_active (GiB)": 36.92, "memory/max_allocated (GiB)": 36.92, "ppl": 5.42688, "step": 14, "tokens/total": 469104, "tokens/train_per_sec_per_gpu": 52.9, "tokens/trainable": 14350} +{"epoch": 0.05859375, "grad_norm": 7.171371936798096, "learning_rate": 5.6000000000000006e-05, "loss": 1.6419003009796143, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 5.16498, "step": 15, "tokens/total": 504512, "tokens/train_per_sec_per_gpu": 60.76, "tokens/trainable": 15398} +{"epoch": 0.0625, "grad_norm": 2.718503952026367, "learning_rate": 6e-05, "loss": 1.604529857635498, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 36.83, "memory/max_allocated (GiB)": 36.83, "ppl": 4.97552, "step": 16, "tokens/total": 537344, "tokens/train_per_sec_per_gpu": 52.04, "tokens/trainable": 16330} +{"epoch": 0.06640625, "grad_norm": 6.791344165802002, "learning_rate": 6.400000000000001e-05, "loss": 1.4700576066970825, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 4.34949, "step": 17, "tokens/total": 572800, "tokens/train_per_sec_per_gpu": 57.99, "tokens/trainable": 17356} +{"epoch": 0.0703125, "grad_norm": 2.1207501888275146, "learning_rate": 6.800000000000001e-05, "loss": 1.3828626871109009, "memory/device_reserved (GiB)": 40.83, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 3.9863, "step": 18, "tokens/total": 608880, "tokens/train_per_sec_per_gpu": 48.74, "tokens/trainable": 18339} +{"epoch": 0.07421875, "grad_norm": 2.128021001815796, "learning_rate": 7.2e-05, "loss": 1.2283759117126465, "memory/device_reserved (GiB)": 40.83, "memory/max_active (GiB)": 38.28, "memory/max_allocated (GiB)": 38.28, "ppl": 3.41568, "step": 19, "tokens/total": 645152, "tokens/train_per_sec_per_gpu": 57.84, "tokens/trainable": 19354} +{"epoch": 0.078125, "grad_norm": 3.388381242752075, "learning_rate": 7.6e-05, "loss": 1.3410627841949463, "memory/device_reserved (GiB)": 40.83, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 3.8231, "step": 20, "tokens/total": 679600, "tokens/train_per_sec_per_gpu": 60.45, "tokens/trainable": 20397} +{"epoch": 0.08203125, "grad_norm": 1.9562444686889648, "learning_rate": 8e-05, "loss": 1.3222107887268066, "memory/device_reserved (GiB)": 40.83, "memory/max_active (GiB)": 36.12, "memory/max_allocated (GiB)": 36.12, "ppl": 3.75171, "step": 21, "tokens/total": 710608, "tokens/train_per_sec_per_gpu": 70.62, "tokens/trainable": 21452} +{"epoch": 0.0859375, "grad_norm": 2.9976601600646973, "learning_rate": 8.4e-05, "loss": 1.196503758430481, "memory/device_reserved (GiB)": 42.47, "memory/max_active (GiB)": 38.93, "memory/max_allocated (GiB)": 38.93, "ppl": 3.30853, "step": 22, "tokens/total": 745712, "tokens/train_per_sec_per_gpu": 64.08, "tokens/trainable": 22549} +{"epoch": 0.08984375, "grad_norm": 2.11250638961792, "learning_rate": 8.800000000000001e-05, "loss": 0.9930483102798462, "memory/device_reserved (GiB)": 42.47, "memory/max_active (GiB)": 37.44, "memory/max_allocated (GiB)": 37.44, "ppl": 2.69945, "step": 23, "tokens/total": 780016, "tokens/train_per_sec_per_gpu": 61.11, "tokens/trainable": 23623} +{"epoch": 0.09375, "grad_norm": 2.1242733001708984, "learning_rate": 9.200000000000001e-05, "loss": 1.1916253566741943, "memory/device_reserved (GiB)": 42.47, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 3.29243, "step": 24, "tokens/total": 811056, "tokens/train_per_sec_per_gpu": 67.39, "tokens/trainable": 24605} +{"epoch": 0.09765625, "grad_norm": 1.716751217842102, "learning_rate": 9.6e-05, "loss": 1.1118998527526855, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 3.04013, "step": 25, "tokens/total": 843232, "tokens/train_per_sec_per_gpu": 62.9, "tokens/trainable": 25636} +{"epoch": 0.1015625, "grad_norm": 1.7077897787094116, "learning_rate": 0.0001, "loss": 1.1258677244186401, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 3.08289, "step": 26, "tokens/total": 875856, "tokens/train_per_sec_per_gpu": 57.69, "tokens/trainable": 26610} +{"epoch": 0.10546875, "grad_norm": 1.7058638334274292, "learning_rate": 9.99990636831587e-05, "loss": 0.9437065124511719, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 37.45, "memory/max_allocated (GiB)": 37.45, "ppl": 2.56949, "step": 27, "tokens/total": 908512, "tokens/train_per_sec_per_gpu": 72.61, "tokens/trainable": 27713} +{"epoch": 0.109375, "grad_norm": 2.248486280441284, "learning_rate": 9.999625477159879e-05, "loss": 0.990824282169342, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 2.69345, "step": 28, "tokens/total": 947088, "tokens/train_per_sec_per_gpu": 52.78, "tokens/trainable": 28757} +{"epoch": 0.11328125, "grad_norm": 3.7035651206970215, "learning_rate": 9.999157338221051e-05, "loss": 0.9432021379470825, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 37.84, "memory/max_allocated (GiB)": 37.84, "ppl": 2.56819, "step": 29, "tokens/total": 982320, "tokens/train_per_sec_per_gpu": 62.44, "tokens/trainable": 29901} +{"epoch": 0.1171875, "grad_norm": 1.6148923635482788, "learning_rate": 9.998501970980562e-05, "loss": 0.8966737985610962, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 37.18, "memory/max_allocated (GiB)": 37.18, "ppl": 2.45144, "step": 30, "tokens/total": 1014016, "tokens/train_per_sec_per_gpu": 71.93, "tokens/trainable": 31023} +{"epoch": 0.12109375, "grad_norm": 2.1439428329467773, "learning_rate": 9.997659402710915e-05, "loss": 0.9937839508056641, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 2.70144, "step": 31, "tokens/total": 1048800, "tokens/train_per_sec_per_gpu": 57.67, "tokens/trainable": 32035} +{"epoch": 0.125, "grad_norm": 2.0923588275909424, "learning_rate": 9.996629668474818e-05, "loss": 0.8300018310546875, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 2.29332, "step": 32, "tokens/total": 1083648, "tokens/train_per_sec_per_gpu": 59.71, "tokens/trainable": 33114} +{"epoch": 0.12890625, "grad_norm": 1.9695534706115723, "learning_rate": 9.995412811123711e-05, "loss": 1.076350450515747, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 2.93395, "step": 33, "tokens/total": 1119376, "tokens/train_per_sec_per_gpu": 53.6, "tokens/trainable": 34038} +{"epoch": 0.1328125, "grad_norm": 1.6000646352767944, "learning_rate": 9.994008881295999e-05, "loss": 0.7235943078994751, "memory/device_reserved (GiB)": 39.89, "memory/max_active (GiB)": 37.15, "memory/max_allocated (GiB)": 37.15, "ppl": 2.06183, "step": 34, "tokens/total": 1152736, "tokens/train_per_sec_per_gpu": 64.31, "tokens/trainable": 35106} +{"epoch": 0.13671875, "grad_norm": 1.5586202144622803, "learning_rate": 9.992417937414932e-05, "loss": 0.7364187240600586, "memory/device_reserved (GiB)": 39.9, "memory/max_active (GiB)": 38.06, "memory/max_allocated (GiB)": 38.06, "ppl": 2.08844, "step": 35, "tokens/total": 1186624, "tokens/train_per_sec_per_gpu": 60.09, "tokens/trainable": 36243} +{"epoch": 0.140625, "grad_norm": 1.614062786102295, "learning_rate": 9.99064004568618e-05, "loss": 0.888835072517395, "memory/device_reserved (GiB)": 39.9, "memory/max_active (GiB)": 37.67, "memory/max_allocated (GiB)": 37.67, "ppl": 2.43229, "step": 36, "tokens/total": 1222368, "tokens/train_per_sec_per_gpu": 52.92, "tokens/trainable": 37273} +{"epoch": 0.14453125, "grad_norm": 1.967136025428772, "learning_rate": 9.988675280095074e-05, "loss": 0.7412248849868774, "memory/device_reserved (GiB)": 39.9, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 2.0985, "step": 37, "tokens/total": 1256800, "tokens/train_per_sec_per_gpu": 57.96, "tokens/trainable": 38389} +{"epoch": 0.1484375, "grad_norm": 1.492599606513977, "learning_rate": 9.986523722403528e-05, "loss": 0.7391531467437744, "memory/device_reserved (GiB)": 39.9, "memory/max_active (GiB)": 36.82, "memory/max_allocated (GiB)": 36.82, "ppl": 2.09416, "step": 38, "tokens/total": 1288576, "tokens/train_per_sec_per_gpu": 61.13, "tokens/trainable": 39419} +{"epoch": 0.15234375, "grad_norm": 1.250396966934204, "learning_rate": 9.984185462146642e-05, "loss": 0.6908581852912903, "memory/device_reserved (GiB)": 39.9, "memory/max_active (GiB)": 37.39, "memory/max_allocated (GiB)": 37.39, "ppl": 1.99543, "step": 39, "tokens/total": 1323616, "tokens/train_per_sec_per_gpu": 59.24, "tokens/trainable": 40468} +{"epoch": 0.15625, "grad_norm": 5.758026599884033, "learning_rate": 9.98166059662897e-05, "loss": 0.7957956790924072, "memory/device_reserved (GiB)": 39.91, "memory/max_active (GiB)": 35.86, "memory/max_allocated (GiB)": 35.86, "ppl": 2.2162, "step": 40, "tokens/total": 1354416, "tokens/train_per_sec_per_gpu": 71.96, "tokens/trainable": 41564} +{"epoch": 0.16015625, "grad_norm": 2.151271104812622, "learning_rate": 9.978949230920472e-05, "loss": 0.7318448424339294, "memory/device_reserved (GiB)": 39.91, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 2.07891, "step": 41, "tokens/total": 1388768, "tokens/train_per_sec_per_gpu": 57.95, "tokens/trainable": 42578} +{"epoch": 0.1640625, "grad_norm": 1.8430497646331787, "learning_rate": 9.976051477852141e-05, "loss": 0.7659430503845215, "memory/device_reserved (GiB)": 39.91, "memory/max_active (GiB)": 37.91, "memory/max_allocated (GiB)": 37.91, "ppl": 2.15102, "step": 42, "tokens/total": 1422448, "tokens/train_per_sec_per_gpu": 62.29, "tokens/trainable": 43584} +{"epoch": 0.16796875, "grad_norm": 1.4674721956253052, "learning_rate": 9.972967458011312e-05, "loss": 0.8979437351226807, "memory/device_reserved (GiB)": 39.91, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 2.45455, "step": 43, "tokens/total": 1457872, "tokens/train_per_sec_per_gpu": 61.83, "tokens/trainable": 44691} +{"epoch": 0.171875, "grad_norm": 2.059771776199341, "learning_rate": 9.96969729973664e-05, "loss": 0.7636605501174927, "memory/device_reserved (GiB)": 41.24, "memory/max_active (GiB)": 38.93, "memory/max_allocated (GiB)": 38.93, "ppl": 2.14612, "step": 44, "tokens/total": 1496368, "tokens/train_per_sec_per_gpu": 56.28, "tokens/trainable": 45771} +{"epoch": 0.17578125, "grad_norm": 1.2939319610595703, "learning_rate": 9.966241139112754e-05, "loss": 0.7152721285820007, "memory/device_reserved (GiB)": 41.24, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 2.04474, "step": 45, "tokens/total": 1530016, "tokens/train_per_sec_per_gpu": 61.9, "tokens/trainable": 46807} +{"epoch": 0.1796875, "grad_norm": 3.7574002742767334, "learning_rate": 9.96259911996461e-05, "loss": 0.6961367130279541, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 38.63, "memory/max_allocated (GiB)": 38.63, "ppl": 2.00599, "step": 46, "tokens/total": 1564096, "tokens/train_per_sec_per_gpu": 56.77, "tokens/trainable": 47842} +{"epoch": 0.18359375, "grad_norm": 2.5448858737945557, "learning_rate": 9.958771393851491e-05, "loss": 0.8801035284996033, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 36.71, "memory/max_allocated (GiB)": 36.71, "ppl": 2.41115, "step": 47, "tokens/total": 1595808, "tokens/train_per_sec_per_gpu": 60.18, "tokens/trainable": 48883} +{"epoch": 0.1875, "grad_norm": 1.5219610929489136, "learning_rate": 9.954758120060702e-05, "loss": 0.6671230792999268, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.94862, "step": 48, "tokens/total": 1632160, "tokens/train_per_sec_per_gpu": 52.07, "tokens/trainable": 49845} +{"epoch": 0.19140625, "grad_norm": 5.148168563842773, "learning_rate": 9.950559465600948e-05, "loss": 0.6289064884185791, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 37.61, "memory/max_allocated (GiB)": 37.61, "ppl": 1.87556, "step": 49, "tokens/total": 1666208, "tokens/train_per_sec_per_gpu": 60.78, "tokens/trainable": 50892} +{"epoch": 0.1953125, "grad_norm": 1.4662635326385498, "learning_rate": 9.946175605195379e-05, "loss": 0.605912446975708, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.83292, "step": 50, "tokens/total": 1706304, "tokens/train_per_sec_per_gpu": 49.24, "tokens/trainable": 52001} +{"epoch": 0.19921875, "grad_norm": 1.5114059448242188, "learning_rate": 9.941606721274322e-05, "loss": 0.652057409286499, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.91949, "step": 51, "tokens/total": 1740656, "tokens/train_per_sec_per_gpu": 52.02, "tokens/trainable": 53018} +{"epoch": 0.203125, "grad_norm": 3.3703606128692627, "learning_rate": 9.936853003967685e-05, "loss": 0.7070025205612183, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 2.0279, "step": 52, "tokens/total": 1777808, "tokens/train_per_sec_per_gpu": 53.49, "tokens/trainable": 54035} +{"epoch": 0.20703125, "grad_norm": 1.5825828313827515, "learning_rate": 9.93191465109705e-05, "loss": 0.7296031713485718, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 2.07426, "step": 53, "tokens/total": 1814480, "tokens/train_per_sec_per_gpu": 56.25, "tokens/trainable": 55070} +{"epoch": 0.2109375, "grad_norm": 8.769686698913574, "learning_rate": 9.926791868167438e-05, "loss": 0.6556383371353149, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.92637, "step": 54, "tokens/total": 1850928, "tokens/train_per_sec_per_gpu": 52.14, "tokens/trainable": 56041} +{"epoch": 0.21484375, "grad_norm": 1.4482697248458862, "learning_rate": 9.921484868358753e-05, "loss": 0.689457893371582, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 38.72, "memory/max_allocated (GiB)": 38.72, "ppl": 1.99264, "step": 55, "tokens/total": 1886752, "tokens/train_per_sec_per_gpu": 51.63, "tokens/trainable": 57063} +{"epoch": 0.21875, "grad_norm": 1.404500126838684, "learning_rate": 9.915993872516924e-05, "loss": 0.6223779320716858, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.86335, "step": 56, "tokens/total": 1922352, "tokens/train_per_sec_per_gpu": 53.27, "tokens/trainable": 58069} +{"epoch": 0.22265625, "grad_norm": 2.098233461380005, "learning_rate": 9.9103191091447e-05, "loss": 0.6160608530044556, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 36.88, "memory/max_allocated (GiB)": 36.88, "ppl": 1.85162, "step": 57, "tokens/total": 1954080, "tokens/train_per_sec_per_gpu": 58.43, "tokens/trainable": 59034} +{"epoch": 0.2265625, "grad_norm": 1.734100341796875, "learning_rate": 9.904460814392147e-05, "loss": 0.5648423433303833, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.75917, "step": 58, "tokens/total": 1993856, "tokens/train_per_sec_per_gpu": 49.79, "tokens/trainable": 60073} +{"epoch": 0.23046875, "grad_norm": 2.1362357139587402, "learning_rate": 9.898419232046825e-05, "loss": 0.5967596769332886, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.81622, "step": 59, "tokens/total": 2030416, "tokens/train_per_sec_per_gpu": 57.57, "tokens/trainable": 61168} +{"epoch": 0.234375, "grad_norm": 3.672351837158203, "learning_rate": 9.892194613523633e-05, "loss": 0.5760093331336975, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.77893, "step": 60, "tokens/total": 2066464, "tokens/train_per_sec_per_gpu": 57.56, "tokens/trainable": 62283} +{"epoch": 0.23828125, "grad_norm": 1.3009321689605713, "learning_rate": 9.885787217854357e-05, "loss": 0.5409855842590332, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 38.18, "memory/max_allocated (GiB)": 38.18, "ppl": 1.7177, "step": 61, "tokens/total": 2100640, "tokens/train_per_sec_per_gpu": 57.08, "tokens/trainable": 63286} +{"epoch": 0.2421875, "grad_norm": 1.1743230819702148, "learning_rate": 9.879197311676887e-05, "loss": 0.555019736289978, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.74198, "step": 62, "tokens/total": 2136800, "tokens/train_per_sec_per_gpu": 56.38, "tokens/trainable": 64328} +{"epoch": 0.24609375, "grad_norm": 1.5929925441741943, "learning_rate": 9.872425169224113e-05, "loss": 0.6025757789611816, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 1.82682, "step": 63, "tokens/total": 2170496, "tokens/train_per_sec_per_gpu": 69.92, "tokens/trainable": 65506} +{"epoch": 0.25, "grad_norm": 1.7416260242462158, "learning_rate": 9.865471072312528e-05, "loss": 0.5557777881622314, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.7433, "step": 64, "tokens/total": 2207040, "tokens/train_per_sec_per_gpu": 61.02, "tokens/trainable": 66588} +{"epoch": 0.25390625, "grad_norm": 1.485267162322998, "learning_rate": 9.858335310330492e-05, "loss": 0.5083292126655579, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 37.09, "memory/max_allocated (GiB)": 37.09, "ppl": 1.66251, "step": 65, "tokens/total": 2239632, "tokens/train_per_sec_per_gpu": 63.36, "tokens/trainable": 67667} +{"epoch": 0.2578125, "grad_norm": 1.5438194274902344, "learning_rate": 9.851018180226185e-05, "loss": 0.49338799715042114, "memory/device_reserved (GiB)": 41.42, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.63786, "step": 66, "tokens/total": 2278304, "tokens/train_per_sec_per_gpu": 54.13, "tokens/trainable": 68737} +{"epoch": 0.26171875, "grad_norm": 1.4572089910507202, "learning_rate": 9.843519986495259e-05, "loss": 0.5845403075218201, "memory/device_reserved (GiB)": 41.42, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.79417, "step": 67, "tokens/total": 2313248, "tokens/train_per_sec_per_gpu": 59.16, "tokens/trainable": 69781} +{"epoch": 0.265625, "grad_norm": 1.299017071723938, "learning_rate": 9.835841041168162e-05, "loss": 0.4871969521045685, "memory/device_reserved (GiB)": 41.42, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.62775, "step": 68, "tokens/total": 2350656, "tokens/train_per_sec_per_gpu": 59.68, "tokens/trainable": 70877} +{"epoch": 0.26953125, "grad_norm": 1.6990737915039062, "learning_rate": 9.82798166379715e-05, "loss": 0.5125610828399658, "memory/device_reserved (GiB)": 41.42, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.66956, "step": 69, "tokens/total": 2383856, "tokens/train_per_sec_per_gpu": 61.01, "tokens/trainable": 71955} +{"epoch": 0.2734375, "grad_norm": 1.5452286005020142, "learning_rate": 9.819942181443002e-05, "loss": 0.6440357565879822, "memory/device_reserved (GiB)": 41.42, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.90415, "step": 70, "tokens/total": 2421120, "tokens/train_per_sec_per_gpu": 58.02, "tokens/trainable": 73029} +{"epoch": 0.27734375, "grad_norm": 1.295297622680664, "learning_rate": 9.811722928661392e-05, "loss": 0.6072438359260559, "memory/device_reserved (GiB)": 41.43, "memory/max_active (GiB)": 35.8, "memory/max_allocated (GiB)": 35.8, "ppl": 1.83537, "step": 71, "tokens/total": 2449904, "tokens/train_per_sec_per_gpu": 72.1, "tokens/trainable": 74046} +{"epoch": 0.28125, "grad_norm": 1.22128164768219, "learning_rate": 9.803324247488975e-05, "loss": 0.4937068223953247, "memory/device_reserved (GiB)": 41.43, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.63838, "step": 72, "tokens/total": 2485936, "tokens/train_per_sec_per_gpu": 57.55, "tokens/trainable": 75116} +{"epoch": 0.28515625, "grad_norm": 1.828308343887329, "learning_rate": 9.794746487429161e-05, "loss": 0.49087467789649963, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.04, "memory/max_allocated (GiB)": 37.04, "ppl": 1.63374, "step": 73, "tokens/total": 2519024, "tokens/train_per_sec_per_gpu": 67.51, "tokens/trainable": 76155} +{"epoch": 0.2890625, "grad_norm": 26.493959426879883, "learning_rate": 9.785990005437554e-05, "loss": 0.557854950428009, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.81, "memory/max_allocated (GiB)": 37.81, "ppl": 1.74692, "step": 74, "tokens/total": 2554864, "tokens/train_per_sec_per_gpu": 57.82, "tokens/trainable": 77208} +{"epoch": 0.29296875, "grad_norm": 1.5231541395187378, "learning_rate": 9.777055165907117e-05, "loss": 0.534949779510498, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.70736, "step": 75, "tokens/total": 2588048, "tokens/train_per_sec_per_gpu": 66.39, "tokens/trainable": 78249} +{"epoch": 0.296875, "grad_norm": 1.96988844871521, "learning_rate": 9.767942340652993e-05, "loss": 0.5956020951271057, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.05, "memory/max_allocated (GiB)": 38.05, "ppl": 1.81412, "step": 76, "tokens/total": 2622784, "tokens/train_per_sec_per_gpu": 55.06, "tokens/trainable": 79268} +{"epoch": 0.30078125, "grad_norm": 2.004877805709839, "learning_rate": 9.758651908897035e-05, "loss": 0.5619116425514221, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.75402, "step": 77, "tokens/total": 2656416, "tokens/train_per_sec_per_gpu": 62.12, "tokens/trainable": 80337} +{"epoch": 0.3046875, "grad_norm": 1.5697650909423828, "learning_rate": 9.749184257252033e-05, "loss": 0.5529402494430542, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.73836, "step": 78, "tokens/total": 2690672, "tokens/train_per_sec_per_gpu": 61.74, "tokens/trainable": 81338} +{"epoch": 0.30859375, "grad_norm": 2.023751735687256, "learning_rate": 9.739539779705614e-05, "loss": 0.44959747791290283, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.56768, "step": 79, "tokens/total": 2729200, "tokens/train_per_sec_per_gpu": 58.36, "tokens/trainable": 82535} +{"epoch": 0.3125, "grad_norm": 2.51389479637146, "learning_rate": 9.729718877603861e-05, "loss": 0.5397174954414368, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.71552, "step": 80, "tokens/total": 2762784, "tokens/train_per_sec_per_gpu": 68.88, "tokens/trainable": 83593} +{"epoch": 0.31640625, "grad_norm": 1.057041049003601, "learning_rate": 9.719721959634592e-05, "loss": 0.4485452175140381, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.56603, "step": 81, "tokens/total": 2798304, "tokens/train_per_sec_per_gpu": 61.24, "tokens/trainable": 84642} +{"epoch": 0.3203125, "grad_norm": 1.0184117555618286, "learning_rate": 9.709549441810375e-05, "loss": 0.45847660303115845, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.58166, "step": 82, "tokens/total": 2835712, "tokens/train_per_sec_per_gpu": 60.43, "tokens/trainable": 85722} +{"epoch": 0.32421875, "grad_norm": 1.1449239253997803, "learning_rate": 9.699201747451195e-05, "loss": 0.4697186052799225, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.54, "memory/max_allocated (GiB)": 36.54, "ppl": 1.59954, "step": 83, "tokens/total": 2868224, "tokens/train_per_sec_per_gpu": 58.56, "tokens/trainable": 86775} +{"epoch": 0.328125, "grad_norm": 1.6718474626541138, "learning_rate": 9.688679307166854e-05, "loss": 0.5537189245223999, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.73971, "step": 84, "tokens/total": 2900736, "tokens/train_per_sec_per_gpu": 63.38, "tokens/trainable": 87826} +{"epoch": 0.33203125, "grad_norm": 8.295278549194336, "learning_rate": 9.677982558839042e-05, "loss": 0.5315399765968323, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.82, "memory/max_allocated (GiB)": 36.82, "ppl": 1.70155, "step": 85, "tokens/total": 2933904, "tokens/train_per_sec_per_gpu": 56.57, "tokens/trainable": 88881} +{"epoch": 0.3359375, "grad_norm": 1.4847339391708374, "learning_rate": 9.66711194760312e-05, "loss": 0.43244773149490356, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 39.04, "memory/max_allocated (GiB)": 39.04, "ppl": 1.54102, "step": 86, "tokens/total": 2971344, "tokens/train_per_sec_per_gpu": 55.92, "tokens/trainable": 90010} +{"epoch": 0.33984375, "grad_norm": 10.866579055786133, "learning_rate": 9.656067925829593e-05, "loss": 0.4859444200992584, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.62571, "step": 87, "tokens/total": 3001936, "tokens/train_per_sec_per_gpu": 71.22, "tokens/trainable": 91044} +{"epoch": 0.34375, "grad_norm": 1.6313061714172363, "learning_rate": 9.644850953105288e-05, "loss": 0.5785410404205322, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 35.88, "memory/max_allocated (GiB)": 35.88, "ppl": 1.78343, "step": 88, "tokens/total": 3032944, "tokens/train_per_sec_per_gpu": 68.1, "tokens/trainable": 92123} +{"epoch": 0.34765625, "grad_norm": 1.185451865196228, "learning_rate": 9.633461496214225e-05, "loss": 0.39551568031311035, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.48515, "step": 89, "tokens/total": 3065584, "tokens/train_per_sec_per_gpu": 62.25, "tokens/trainable": 93180} +{"epoch": 0.3515625, "grad_norm": 1.3938558101654053, "learning_rate": 9.621900029118195e-05, "loss": 0.5466035604476929, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.1, "memory/max_allocated (GiB)": 36.1, "ppl": 1.72738, "step": 90, "tokens/total": 3095248, "tokens/train_per_sec_per_gpu": 67.76, "tokens/trainable": 94212} +{"epoch": 0.35546875, "grad_norm": 1.4054069519042969, "learning_rate": 9.610167032937036e-05, "loss": 0.4843631386756897, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.62314, "step": 91, "tokens/total": 3133456, "tokens/train_per_sec_per_gpu": 54.34, "tokens/trainable": 95265} +{"epoch": 0.359375, "grad_norm": 1.22010338306427, "learning_rate": 9.598262995928611e-05, "loss": 0.42937415838241577, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.5363, "step": 92, "tokens/total": 3164624, "tokens/train_per_sec_per_gpu": 68.94, "tokens/trainable": 96319} +{"epoch": 0.36328125, "grad_norm": 5.048675537109375, "learning_rate": 9.586188413468492e-05, "loss": 0.4564042091369629, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.8, "memory/max_allocated (GiB)": 38.8, "ppl": 1.57839, "step": 93, "tokens/total": 3203008, "tokens/train_per_sec_per_gpu": 49.5, "tokens/trainable": 97372} +{"epoch": 0.3671875, "grad_norm": 2.455667734146118, "learning_rate": 9.57394378802934e-05, "loss": 0.46533387899398804, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.59255, "step": 94, "tokens/total": 3236448, "tokens/train_per_sec_per_gpu": 60.72, "tokens/trainable": 98418} +{"epoch": 0.37109375, "grad_norm": 1.324281096458435, "learning_rate": 9.56152962916e-05, "loss": 0.4558974802494049, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.57759, "step": 95, "tokens/total": 3267104, "tokens/train_per_sec_per_gpu": 66.27, "tokens/trainable": 99408} +{"epoch": 0.375, "grad_norm": 1.3683860301971436, "learning_rate": 9.548946453464296e-05, "loss": 0.37981972098350525, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 1.46202, "step": 96, "tokens/total": 3301440, "tokens/train_per_sec_per_gpu": 62.7, "tokens/trainable": 100513} +{"epoch": 0.37890625, "grad_norm": 1.528564453125, "learning_rate": 9.53619478457953e-05, "loss": 0.5061219930648804, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.65885, "step": 97, "tokens/total": 3336112, "tokens/train_per_sec_per_gpu": 52.5, "tokens/trainable": 101523} +{"epoch": 0.3828125, "grad_norm": 1.546659231185913, "learning_rate": 9.523275153154695e-05, "loss": 0.49062392115592957, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.63333, "step": 98, "tokens/total": 3368624, "tokens/train_per_sec_per_gpu": 57.88, "tokens/trainable": 102483} +{"epoch": 0.38671875, "grad_norm": 2.2892184257507324, "learning_rate": 9.51018809682839e-05, "loss": 0.4192270338535309, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.52079, "step": 99, "tokens/total": 3406448, "tokens/train_per_sec_per_gpu": 52.97, "tokens/trainable": 103500} +{"epoch": 0.390625, "grad_norm": 1.331002116203308, "learning_rate": 9.49693416020645e-05, "loss": 0.46647101640701294, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.51, "memory/max_allocated (GiB)": 38.51, "ppl": 1.59436, "step": 100, "tokens/total": 3444240, "tokens/train_per_sec_per_gpu": 54.86, "tokens/trainable": 104520} +{"epoch": 0.39453125, "grad_norm": 3.1616103649139404, "learning_rate": 9.483513894839276e-05, "loss": 0.4825928211212158, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.62027, "step": 101, "tokens/total": 3477680, "tokens/train_per_sec_per_gpu": 68.02, "tokens/trainable": 105623} +{"epoch": 0.3984375, "grad_norm": 1.4605764150619507, "learning_rate": 9.469927859198888e-05, "loss": 0.45581701397895813, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.57746, "step": 102, "tokens/total": 3513616, "tokens/train_per_sec_per_gpu": 58.62, "tokens/trainable": 106623} +{"epoch": 0.40234375, "grad_norm": 3.7108089923858643, "learning_rate": 9.456176618655689e-05, "loss": 0.4538918733596802, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.51, "memory/max_allocated (GiB)": 36.51, "ppl": 1.57443, "step": 103, "tokens/total": 3545584, "tokens/train_per_sec_per_gpu": 60.46, "tokens/trainable": 107619} +{"epoch": 0.40625, "grad_norm": 1.4839152097702026, "learning_rate": 9.442260745454927e-05, "loss": 0.4355614483356476, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.54583, "step": 104, "tokens/total": 3580768, "tokens/train_per_sec_per_gpu": 59.11, "tokens/trainable": 108690} +{"epoch": 0.41015625, "grad_norm": 1.5258665084838867, "learning_rate": 9.428180818692884e-05, "loss": 0.3723750710487366, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.45118, "step": 105, "tokens/total": 3617008, "tokens/train_per_sec_per_gpu": 52.52, "tokens/trainable": 109750} +{"epoch": 0.4140625, "grad_norm": 2.4054036140441895, "learning_rate": 9.413937424292791e-05, "loss": 0.461595356464386, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.5866, "step": 106, "tokens/total": 3655936, "tokens/train_per_sec_per_gpu": 50.01, "tokens/trainable": 110766} +{"epoch": 0.41796875, "grad_norm": 1.8945794105529785, "learning_rate": 9.399531154980424e-05, "loss": 0.345192551612854, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.41226, "step": 107, "tokens/total": 3692736, "tokens/train_per_sec_per_gpu": 51.57, "tokens/trainable": 111771} +{"epoch": 0.421875, "grad_norm": 2.0983855724334717, "learning_rate": 9.384962610259455e-05, "loss": 0.40318936109542847, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.49659, "step": 108, "tokens/total": 3727248, "tokens/train_per_sec_per_gpu": 54.81, "tokens/trainable": 112775} +{"epoch": 0.42578125, "grad_norm": 1.6275132894515991, "learning_rate": 9.370232396386494e-05, "loss": 0.4509361982345581, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.56978, "step": 109, "tokens/total": 3760464, "tokens/train_per_sec_per_gpu": 60.87, "tokens/trainable": 113795} +{"epoch": 0.4296875, "grad_norm": 2.76203989982605, "learning_rate": 9.355341126345868e-05, "loss": 0.38255253434181213, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.46602, "step": 110, "tokens/total": 3795952, "tokens/train_per_sec_per_gpu": 60.3, "tokens/trainable": 114888} +{"epoch": 0.43359375, "grad_norm": 1.2805089950561523, "learning_rate": 9.340289419824107e-05, "loss": 0.3605329990386963, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.74, "memory/max_allocated (GiB)": 36.74, "ppl": 1.43409, "step": 111, "tokens/total": 3828144, "tokens/train_per_sec_per_gpu": 68.89, "tokens/trainable": 115919} +{"epoch": 0.4375, "grad_norm": 2.105903387069702, "learning_rate": 9.325077903184159e-05, "loss": 0.5129556059837341, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.67022, "step": 112, "tokens/total": 3862992, "tokens/train_per_sec_per_gpu": 59.39, "tokens/trainable": 117018} +{"epoch": 0.44140625, "grad_norm": 2.596709728240967, "learning_rate": 9.30970720943932e-05, "loss": 0.3950740098953247, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.43, "memory/max_allocated (GiB)": 38.43, "ppl": 1.48449, "step": 113, "tokens/total": 3898416, "tokens/train_per_sec_per_gpu": 57.76, "tokens/trainable": 118100} +{"epoch": 0.4453125, "grad_norm": 1.4222482442855835, "learning_rate": 9.2941779782269e-05, "loss": 0.4216489791870117, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.52447, "step": 114, "tokens/total": 3932176, "tokens/train_per_sec_per_gpu": 61.06, "tokens/trainable": 119137} +{"epoch": 0.44921875, "grad_norm": 1.2968515157699585, "learning_rate": 9.278490855781596e-05, "loss": 0.4432855546474457, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.55782, "step": 115, "tokens/total": 3964384, "tokens/train_per_sec_per_gpu": 68.23, "tokens/trainable": 120153} +{"epoch": 0.453125, "grad_norm": 2.0339972972869873, "learning_rate": 9.262646494908604e-05, "loss": 0.48488959670066833, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 39.14, "memory/max_allocated (GiB)": 39.14, "ppl": 1.624, "step": 116, "tokens/total": 4002960, "tokens/train_per_sec_per_gpu": 50.92, "tokens/trainable": 121152} +{"epoch": 0.45703125, "grad_norm": 3.7786214351654053, "learning_rate": 9.246645554956457e-05, "loss": 0.4619576036930084, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.58718, "step": 117, "tokens/total": 4037552, "tokens/train_per_sec_per_gpu": 60.09, "tokens/trainable": 122186} +{"epoch": 0.4609375, "grad_norm": 2.1846423149108887, "learning_rate": 9.230488701789578e-05, "loss": 0.4703233242034912, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.7, "memory/max_allocated (GiB)": 38.7, "ppl": 1.60051, "step": 118, "tokens/total": 4075072, "tokens/train_per_sec_per_gpu": 56.86, "tokens/trainable": 123284} +{"epoch": 0.46484375, "grad_norm": 1.5197901725769043, "learning_rate": 9.214176607760577e-05, "loss": 0.3664529323577881, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.44261, "step": 119, "tokens/total": 4113744, "tokens/train_per_sec_per_gpu": 58.39, "tokens/trainable": 124415} +{"epoch": 0.46875, "grad_norm": 1.4422430992126465, "learning_rate": 9.197709951682268e-05, "loss": 0.3960150480270386, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.48589, "step": 120, "tokens/total": 4146032, "tokens/train_per_sec_per_gpu": 65.01, "tokens/trainable": 125482} +{"epoch": 0.47265625, "grad_norm": 1.541280746459961, "learning_rate": 9.181089418799428e-05, "loss": 0.3959267735481262, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 39.39, "memory/max_allocated (GiB)": 39.39, "ppl": 1.48576, "step": 121, "tokens/total": 4182832, "tokens/train_per_sec_per_gpu": 54.89, "tokens/trainable": 126521} +{"epoch": 0.4765625, "grad_norm": 1.0889137983322144, "learning_rate": 9.164315700760271e-05, "loss": 0.3334844410419464, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.39582, "step": 122, "tokens/total": 4218640, "tokens/train_per_sec_per_gpu": 58.29, "tokens/trainable": 127565} +{"epoch": 0.48046875, "grad_norm": 2.563070297241211, "learning_rate": 9.147389495587671e-05, "loss": 0.4671238958835602, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 1.5954, "step": 123, "tokens/total": 4249792, "tokens/train_per_sec_per_gpu": 69.02, "tokens/trainable": 128595} +{"epoch": 0.484375, "grad_norm": 1.5538933277130127, "learning_rate": 9.130311507650116e-05, "loss": 0.32542943954467773, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 1.38463, "step": 124, "tokens/total": 4282464, "tokens/train_per_sec_per_gpu": 61.59, "tokens/trainable": 129650} +{"epoch": 0.48828125, "grad_norm": 4.736726760864258, "learning_rate": 9.113082447632394e-05, "loss": 0.4080791771411896, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.50393, "step": 125, "tokens/total": 4314608, "tokens/train_per_sec_per_gpu": 79.47, "tokens/trainable": 130784} +{"epoch": 0.4921875, "grad_norm": 4.7695746421813965, "learning_rate": 9.09570303250602e-05, "loss": 0.40960854291915894, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.47, "memory/max_allocated (GiB)": 36.47, "ppl": 1.50623, "step": 126, "tokens/total": 4346416, "tokens/train_per_sec_per_gpu": 63.13, "tokens/trainable": 131847} +{"epoch": 0.49609375, "grad_norm": 2.0959043502807617, "learning_rate": 9.078173985499394e-05, "loss": 0.36106136441230774, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 35.79, "memory/max_allocated (GiB)": 35.79, "ppl": 1.43485, "step": 127, "tokens/total": 4376400, "tokens/train_per_sec_per_gpu": 65.35, "tokens/trainable": 132851} +{"epoch": 0.5, "grad_norm": 2.046330213546753, "learning_rate": 9.060496036067713e-05, "loss": 0.4238126873970032, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.72, "memory/max_allocated (GiB)": 38.72, "ppl": 1.52778, "step": 128, "tokens/total": 4413824, "tokens/train_per_sec_per_gpu": 48.01, "tokens/trainable": 133865} +{"epoch": 0.50390625, "grad_norm": 3.3281595706939697, "learning_rate": 9.042669919862615e-05, "loss": 0.42556923627853394, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.82, "memory/max_allocated (GiB)": 36.82, "ppl": 1.53046, "step": 129, "tokens/total": 4445936, "tokens/train_per_sec_per_gpu": 67.52, "tokens/trainable": 134926} +{"epoch": 0.5078125, "grad_norm": 2.1264729499816895, "learning_rate": 9.024696378701557e-05, "loss": 0.4227166473865509, "memory/device_reserved (GiB)": 41.42, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.5261, "step": 130, "tokens/total": 4481488, "tokens/train_per_sec_per_gpu": 58.29, "tokens/trainable": 135976} +{"epoch": 0.51171875, "grad_norm": 1.4645707607269287, "learning_rate": 9.006576160536948e-05, "loss": 0.3797833323478699, "memory/device_reserved (GiB)": 41.42, "memory/max_active (GiB)": 37.43, "memory/max_allocated (GiB)": 37.43, "ppl": 1.46197, "step": 131, "tokens/total": 4516048, "tokens/train_per_sec_per_gpu": 55.84, "tokens/trainable": 136970} +{"epoch": 0.515625, "grad_norm": 1.3623178005218506, "learning_rate": 8.988310019425035e-05, "loss": 0.34132522344589233, "memory/device_reserved (GiB)": 41.71, "memory/max_active (GiB)": 39.7, "memory/max_allocated (GiB)": 39.7, "ppl": 1.40681, "step": 132, "tokens/total": 4553904, "tokens/train_per_sec_per_gpu": 51.32, "tokens/trainable": 137980} +{"epoch": 0.51953125, "grad_norm": 2.728783369064331, "learning_rate": 8.969898715494506e-05, "loss": 0.3208428621292114, "memory/device_reserved (GiB)": 41.73, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.37829, "step": 133, "tokens/total": 4590736, "tokens/train_per_sec_per_gpu": 60.85, "tokens/trainable": 139070} +{"epoch": 0.5234375, "grad_norm": 6.870443820953369, "learning_rate": 8.951343014914869e-05, "loss": 0.3652361333370209, "memory/device_reserved (GiB)": 41.73, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.44085, "step": 134, "tokens/total": 4627392, "tokens/train_per_sec_per_gpu": 56.86, "tokens/trainable": 140160} +{"epoch": 0.52734375, "grad_norm": 2.7007057666778564, "learning_rate": 8.932643689864568e-05, "loss": 0.34017470479011536, "memory/device_reserved (GiB)": 41.73, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.40519, "step": 135, "tokens/total": 4663968, "tokens/train_per_sec_per_gpu": 61.9, "tokens/trainable": 141277} +{"epoch": 0.53125, "grad_norm": 1.6482174396514893, "learning_rate": 8.913801518498845e-05, "loss": 0.31215763092041016, "memory/device_reserved (GiB)": 41.73, "memory/max_active (GiB)": 39.68, "memory/max_allocated (GiB)": 39.68, "ppl": 1.36637, "step": 136, "tokens/total": 4703344, "tokens/train_per_sec_per_gpu": 48.53, "tokens/trainable": 142389} +{"epoch": 0.53515625, "grad_norm": 5.632486343383789, "learning_rate": 8.894817284917364e-05, "loss": 0.36768293380737305, "memory/device_reserved (GiB)": 41.73, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.44438, "step": 137, "tokens/total": 4740288, "tokens/train_per_sec_per_gpu": 50.09, "tokens/trainable": 143423} +{"epoch": 0.5390625, "grad_norm": 1.6804783344268799, "learning_rate": 8.875691779131569e-05, "loss": 0.459791362285614, "memory/device_reserved (GiB)": 41.9, "memory/max_active (GiB)": 39.78, "memory/max_allocated (GiB)": 39.78, "ppl": 1.58374, "step": 138, "tokens/total": 4778224, "tokens/train_per_sec_per_gpu": 54.96, "tokens/trainable": 144536} +{"epoch": 0.54296875, "grad_norm": 1.770318627357483, "learning_rate": 8.856425797031829e-05, "loss": 0.31801939010620117, "memory/device_reserved (GiB)": 42.22, "memory/max_active (GiB)": 39.58, "memory/max_allocated (GiB)": 39.58, "ppl": 1.3744, "step": 139, "tokens/total": 4814256, "tokens/train_per_sec_per_gpu": 63.68, "tokens/trainable": 145664} +{"epoch": 0.546875, "grad_norm": 1.6026744842529297, "learning_rate": 8.837020140354295e-05, "loss": 0.3776206076145172, "memory/device_reserved (GiB)": 42.22, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.45881, "step": 140, "tokens/total": 4851072, "tokens/train_per_sec_per_gpu": 55.14, "tokens/trainable": 146671} +{"epoch": 0.55078125, "grad_norm": 4.122126579284668, "learning_rate": 8.817475616647554e-05, "loss": 0.38316571712493896, "memory/device_reserved (GiB)": 42.22, "memory/max_active (GiB)": 38.17, "memory/max_allocated (GiB)": 38.17, "ppl": 1.46692, "step": 141, "tokens/total": 4885504, "tokens/train_per_sec_per_gpu": 59.9, "tokens/trainable": 147742} +{"epoch": 0.5546875, "grad_norm": 2.2854537963867188, "learning_rate": 8.797793039239017e-05, "loss": 0.38077789545059204, "memory/device_reserved (GiB)": 42.22, "memory/max_active (GiB)": 37.83, "memory/max_allocated (GiB)": 37.83, "ppl": 1.46342, "step": 142, "tokens/total": 4920528, "tokens/train_per_sec_per_gpu": 60.17, "tokens/trainable": 148809} +{"epoch": 0.55859375, "grad_norm": 1.2946885824203491, "learning_rate": 8.777973227201069e-05, "loss": 0.32865291833877563, "memory/device_reserved (GiB)": 43.76, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.3891, "step": 143, "tokens/total": 4957440, "tokens/train_per_sec_per_gpu": 60.17, "tokens/trainable": 149872} +{"epoch": 0.5625, "grad_norm": 2.3387136459350586, "learning_rate": 8.758017005316988e-05, "loss": 0.3983130156993866, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.48931, "step": 144, "tokens/total": 4990752, "tokens/train_per_sec_per_gpu": 72.51, "tokens/trainable": 150920} +{"epoch": 0.56640625, "grad_norm": 10.270636558532715, "learning_rate": 8.737925204046629e-05, "loss": 0.39614397287368774, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.48608, "step": 145, "tokens/total": 5025472, "tokens/train_per_sec_per_gpu": 54.52, "tokens/trainable": 151981} +{"epoch": 0.5703125, "grad_norm": 6.3600544929504395, "learning_rate": 8.717698659491851e-05, "loss": 0.3343011140823364, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 38.91, "memory/max_allocated (GiB)": 38.91, "ppl": 1.39696, "step": 146, "tokens/total": 5062304, "tokens/train_per_sec_per_gpu": 56.77, "tokens/trainable": 153085} +{"epoch": 0.57421875, "grad_norm": 2.6604998111724854, "learning_rate": 8.697338213361735e-05, "loss": 0.42029130458831787, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 36.45, "memory/max_allocated (GiB)": 36.45, "ppl": 1.5224, "step": 147, "tokens/total": 5093280, "tokens/train_per_sec_per_gpu": 68.97, "tokens/trainable": 154182} +{"epoch": 0.578125, "grad_norm": 1.2546379566192627, "learning_rate": 8.676844712937552e-05, "loss": 0.31579267978668213, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.37135, "step": 148, "tokens/total": 5133280, "tokens/train_per_sec_per_gpu": 49.01, "tokens/trainable": 155234} +{"epoch": 0.58203125, "grad_norm": 132.99432373046875, "learning_rate": 8.656219011037509e-05, "loss": 0.3459272086620331, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.4133, "step": 149, "tokens/total": 5170576, "tokens/train_per_sec_per_gpu": 56.13, "tokens/trainable": 156324} +{"epoch": 0.5859375, "grad_norm": 1.7982186079025269, "learning_rate": 8.63546196598125e-05, "loss": 0.3508491516113281, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.42027, "step": 150, "tokens/total": 5208160, "tokens/train_per_sec_per_gpu": 55.77, "tokens/trainable": 157459} +{"epoch": 0.58984375, "grad_norm": 1.8869658708572388, "learning_rate": 8.614574441554145e-05, "loss": 0.43129250407218933, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 36.95, "memory/max_allocated (GiB)": 36.95, "ppl": 1.53925, "step": 151, "tokens/total": 5240784, "tokens/train_per_sec_per_gpu": 62.17, "tokens/trainable": 158423} +{"epoch": 0.59375, "grad_norm": 5.221248626708984, "learning_rate": 8.593557306971349e-05, "loss": 0.30793458223342896, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.36061, "step": 152, "tokens/total": 5274096, "tokens/train_per_sec_per_gpu": 58.47, "tokens/trainable": 159456} +{"epoch": 0.59765625, "grad_norm": 2.6754086017608643, "learning_rate": 8.572411436841618e-05, "loss": 0.35072410106658936, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 36.9, "memory/max_allocated (GiB)": 36.9, "ppl": 1.4201, "step": 153, "tokens/total": 5307840, "tokens/train_per_sec_per_gpu": 64.27, "tokens/trainable": 160526} +{"epoch": 0.6015625, "grad_norm": 1.496665120124817, "learning_rate": 8.551137711130922e-05, "loss": 0.33799487352371216, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.40213, "step": 154, "tokens/total": 5339616, "tokens/train_per_sec_per_gpu": 59.11, "tokens/trainable": 161544} +{"epoch": 0.60546875, "grad_norm": 1.9917612075805664, "learning_rate": 8.529737015125824e-05, "loss": 0.28906452655792236, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 38.17, "memory/max_allocated (GiB)": 38.17, "ppl": 1.33518, "step": 155, "tokens/total": 5374720, "tokens/train_per_sec_per_gpu": 56.67, "tokens/trainable": 162563} +{"epoch": 0.609375, "grad_norm": 1.3044872283935547, "learning_rate": 8.508210239396639e-05, "loss": 0.34489184617996216, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 39.77, "memory/max_allocated (GiB)": 39.77, "ppl": 1.41184, "step": 156, "tokens/total": 5412608, "tokens/train_per_sec_per_gpu": 56.97, "tokens/trainable": 163580} +{"epoch": 0.61328125, "grad_norm": 1.3923695087432861, "learning_rate": 8.486558279760375e-05, "loss": 0.33252304792404175, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 35.9, "memory/max_allocated (GiB)": 35.9, "ppl": 1.39448, "step": 157, "tokens/total": 5443664, "tokens/train_per_sec_per_gpu": 63.86, "tokens/trainable": 164587} +{"epoch": 0.6171875, "grad_norm": 1.9110058546066284, "learning_rate": 8.464782037243449e-05, "loss": 0.3527105450630188, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.42292, "step": 158, "tokens/total": 5475808, "tokens/train_per_sec_per_gpu": 60.45, "tokens/trainable": 165576} +{"epoch": 0.62109375, "grad_norm": 3.5771942138671875, "learning_rate": 8.442882418044202e-05, "loss": 0.3232697546482086, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.38164, "step": 159, "tokens/total": 5508448, "tokens/train_per_sec_per_gpu": 63.42, "tokens/trainable": 166616} +{"epoch": 0.625, "grad_norm": 1.468672275543213, "learning_rate": 8.420860333495179e-05, "loss": 0.29949843883514404, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.34918, "step": 160, "tokens/total": 5540976, "tokens/train_per_sec_per_gpu": 64.49, "tokens/trainable": 167604} +{"epoch": 0.62890625, "grad_norm": 3.088641881942749, "learning_rate": 8.398716700025208e-05, "loss": 0.35880887508392334, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.43162, "step": 161, "tokens/total": 5574496, "tokens/train_per_sec_per_gpu": 60.63, "tokens/trainable": 168639} +{"epoch": 0.6328125, "grad_norm": 2.7948169708251953, "learning_rate": 8.376452439121266e-05, "loss": 0.4886903166770935, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.05, "memory/max_allocated (GiB)": 36.05, "ppl": 1.63018, "step": 162, "tokens/total": 5604544, "tokens/train_per_sec_per_gpu": 68.77, "tokens/trainable": 169639} +{"epoch": 0.63671875, "grad_norm": 2.5234880447387695, "learning_rate": 8.354068477290124e-05, "loss": 0.35848140716552734, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.43115, "step": 163, "tokens/total": 5638768, "tokens/train_per_sec_per_gpu": 61.04, "tokens/trainable": 170626} +{"epoch": 0.640625, "grad_norm": 1.5041258335113525, "learning_rate": 8.331565746019807e-05, "loss": 0.30878758430480957, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.36177, "step": 164, "tokens/total": 5670960, "tokens/train_per_sec_per_gpu": 68.67, "tokens/trainable": 171666} +{"epoch": 0.64453125, "grad_norm": 3.442885160446167, "learning_rate": 8.308945181740812e-05, "loss": 0.3563690185546875, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.42813, "step": 165, "tokens/total": 5705664, "tokens/train_per_sec_per_gpu": 61.79, "tokens/trainable": 172682} +{"epoch": 0.6484375, "grad_norm": 6.568354606628418, "learning_rate": 8.286207725787153e-05, "loss": 0.2842031419277191, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.3287, "step": 166, "tokens/total": 5742672, "tokens/train_per_sec_per_gpu": 51.5, "tokens/trainable": 173709} +{"epoch": 0.65234375, "grad_norm": 8.872391700744629, "learning_rate": 8.263354324357182e-05, "loss": 0.31773221492767334, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.12, "memory/max_allocated (GiB)": 39.12, "ppl": 1.37401, "step": 167, "tokens/total": 5777408, "tokens/train_per_sec_per_gpu": 59.23, "tokens/trainable": 174741} +{"epoch": 0.65625, "grad_norm": 2.145132541656494, "learning_rate": 8.240385928474219e-05, "loss": 0.3771880269050598, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.45818, "step": 168, "tokens/total": 5810224, "tokens/train_per_sec_per_gpu": 57.65, "tokens/trainable": 175701} +{"epoch": 0.66015625, "grad_norm": 6.107184886932373, "learning_rate": 8.217303493946967e-05, "loss": 0.3681759238243103, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.4451, "step": 169, "tokens/total": 5844432, "tokens/train_per_sec_per_gpu": 57.72, "tokens/trainable": 176705} +{"epoch": 0.6640625, "grad_norm": 1.723182201385498, "learning_rate": 8.194107981329746e-05, "loss": 0.32450413703918457, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.73, "memory/max_allocated (GiB)": 39.73, "ppl": 1.38334, "step": 170, "tokens/total": 5883008, "tokens/train_per_sec_per_gpu": 51.52, "tokens/trainable": 177726} +{"epoch": 0.66796875, "grad_norm": 2.605984687805176, "learning_rate": 8.170800355882518e-05, "loss": 0.29122403264045715, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.33806, "step": 171, "tokens/total": 5917440, "tokens/train_per_sec_per_gpu": 66.22, "tokens/trainable": 178765} +{"epoch": 0.671875, "grad_norm": 8.4197359085083, "learning_rate": 8.147381587530713e-05, "loss": 0.411418080329895, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.39, "memory/max_allocated (GiB)": 37.39, "ppl": 1.50896, "step": 172, "tokens/total": 5951232, "tokens/train_per_sec_per_gpu": 62.32, "tokens/trainable": 179765} +{"epoch": 0.67578125, "grad_norm": 1.8100082874298096, "learning_rate": 8.123852650824877e-05, "loss": 0.34021854400634766, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.45, "memory/max_allocated (GiB)": 37.45, "ppl": 1.40525, "step": 173, "tokens/total": 5985056, "tokens/train_per_sec_per_gpu": 62.6, "tokens/trainable": 180836} +{"epoch": 0.6796875, "grad_norm": 1.9545830488204956, "learning_rate": 8.100214524900103e-05, "loss": 0.32796818017959595, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.38814, "step": 174, "tokens/total": 6020128, "tokens/train_per_sec_per_gpu": 52.05, "tokens/trainable": 181830} +{"epoch": 0.68359375, "grad_norm": 1.7627474069595337, "learning_rate": 8.076468193435301e-05, "loss": 0.3288062810897827, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.38931, "step": 175, "tokens/total": 6050976, "tokens/train_per_sec_per_gpu": 66.62, "tokens/trainable": 182778} +{"epoch": 0.6875, "grad_norm": 1.530403733253479, "learning_rate": 8.052614644612253e-05, "loss": 0.3305663466453552, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.39176, "step": 176, "tokens/total": 6088816, "tokens/train_per_sec_per_gpu": 49.82, "tokens/trainable": 183828} +{"epoch": 0.69140625, "grad_norm": 1.5221978425979614, "learning_rate": 8.028654871074489e-05, "loss": 0.2778800427913666, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.32033, "step": 177, "tokens/total": 6123408, "tokens/train_per_sec_per_gpu": 58.88, "tokens/trainable": 184914} +{"epoch": 0.6953125, "grad_norm": 1.996298909187317, "learning_rate": 8.004589869885986e-05, "loss": 0.4631357789039612, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.37, "memory/max_allocated (GiB)": 36.37, "ppl": 1.58905, "step": 178, "tokens/total": 6154880, "tokens/train_per_sec_per_gpu": 61.04, "tokens/trainable": 185891} +{"epoch": 0.69921875, "grad_norm": 1.882554292678833, "learning_rate": 7.980420642489674e-05, "loss": 0.28026464581489563, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.32348, "step": 179, "tokens/total": 6193584, "tokens/train_per_sec_per_gpu": 53.55, "tokens/trainable": 186923} +{"epoch": 0.703125, "grad_norm": 1.4942058324813843, "learning_rate": 7.95614819466576e-05, "loss": 0.29374122619628906, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 1.34144, "step": 180, "tokens/total": 6227296, "tokens/train_per_sec_per_gpu": 61.08, "tokens/trainable": 187995} +{"epoch": 0.70703125, "grad_norm": 1.5596078634262085, "learning_rate": 7.931773536489872e-05, "loss": 0.3072967827320099, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.35974, "step": 181, "tokens/total": 6261888, "tokens/train_per_sec_per_gpu": 62.41, "tokens/trainable": 189051} +{"epoch": 0.7109375, "grad_norm": 3.606563091278076, "learning_rate": 7.907297682291035e-05, "loss": 0.3576887845993042, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.43002, "step": 182, "tokens/total": 6296544, "tokens/train_per_sec_per_gpu": 60.54, "tokens/trainable": 190134} +{"epoch": 0.71484375, "grad_norm": 2.4602832794189453, "learning_rate": 7.882721650609442e-05, "loss": 0.22608435153961182, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.65, "memory/max_allocated (GiB)": 39.65, "ppl": 1.25368, "step": 183, "tokens/total": 6336960, "tokens/train_per_sec_per_gpu": 53.17, "tokens/trainable": 191312} +{"epoch": 0.71875, "grad_norm": 2.899387836456299, "learning_rate": 7.85804646415409e-05, "loss": 0.3137645721435547, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.36857, "step": 184, "tokens/total": 6372192, "tokens/train_per_sec_per_gpu": 63.96, "tokens/trainable": 192398} +{"epoch": 0.72265625, "grad_norm": 2.341020107269287, "learning_rate": 7.833273149760207e-05, "loss": 0.3769696056842804, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.45786, "step": 185, "tokens/total": 6403056, "tokens/train_per_sec_per_gpu": 61.62, "tokens/trainable": 193415} +{"epoch": 0.7265625, "grad_norm": 1.772653579711914, "learning_rate": 7.808402738346527e-05, "loss": 0.2686132490634918, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.11, "memory/max_allocated (GiB)": 39.11, "ppl": 1.30815, "step": 186, "tokens/total": 6441840, "tokens/train_per_sec_per_gpu": 58.03, "tokens/trainable": 194556} +{"epoch": 0.73046875, "grad_norm": 1.2691348791122437, "learning_rate": 7.783436264872382e-05, "loss": 0.2720675766468048, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.31268, "step": 187, "tokens/total": 6478688, "tokens/train_per_sec_per_gpu": 55.94, "tokens/trainable": 195605} +{"epoch": 0.734375, "grad_norm": 2.829451560974121, "learning_rate": 7.758374768294647e-05, "loss": 0.27211427688598633, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.16, "memory/max_allocated (GiB)": 38.16, "ppl": 1.31274, "step": 188, "tokens/total": 6514624, "tokens/train_per_sec_per_gpu": 60.62, "tokens/trainable": 196700} +{"epoch": 0.73828125, "grad_norm": 3.052567481994629, "learning_rate": 7.733219291524489e-05, "loss": 0.3154413104057312, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.53, "memory/max_allocated (GiB)": 37.53, "ppl": 1.37086, "step": 189, "tokens/total": 6548048, "tokens/train_per_sec_per_gpu": 63.63, "tokens/trainable": 197748} +{"epoch": 0.7421875, "grad_norm": 8.255326271057129, "learning_rate": 7.707970881383977e-05, "loss": 0.3976752758026123, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.45, "memory/max_allocated (GiB)": 36.45, "ppl": 1.48836, "step": 190, "tokens/total": 6579760, "tokens/train_per_sec_per_gpu": 69.05, "tokens/trainable": 198783} +{"epoch": 0.74609375, "grad_norm": 1.5565752983093262, "learning_rate": 7.682630588562518e-05, "loss": 0.3002854585647583, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.7, "memory/max_allocated (GiB)": 38.7, "ppl": 1.35024, "step": 191, "tokens/total": 6614800, "tokens/train_per_sec_per_gpu": 57.38, "tokens/trainable": 199844} +{"epoch": 0.75, "grad_norm": 1.8364275693893433, "learning_rate": 7.657199467573129e-05, "loss": 0.3418474793434143, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.40755, "step": 192, "tokens/total": 6648928, "tokens/train_per_sec_per_gpu": 56.77, "tokens/trainable": 200865} +{"epoch": 0.75390625, "grad_norm": 1.371415376663208, "learning_rate": 7.631678576708561e-05, "loss": 0.23942451179027557, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.16, "memory/max_allocated (GiB)": 38.16, "ppl": 1.27052, "step": 193, "tokens/total": 6683328, "tokens/train_per_sec_per_gpu": 67.16, "tokens/trainable": 201959} +{"epoch": 0.7578125, "grad_norm": 1.3510386943817139, "learning_rate": 7.606068977997255e-05, "loss": 0.26074016094207764, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.64, "memory/max_allocated (GiB)": 39.64, "ppl": 1.29789, "step": 194, "tokens/total": 6720928, "tokens/train_per_sec_per_gpu": 66.1, "tokens/trainable": 203141} +{"epoch": 0.76171875, "grad_norm": 2.032026529312134, "learning_rate": 7.580371737159148e-05, "loss": 0.32963740825653076, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.39046, "step": 195, "tokens/total": 6757024, "tokens/train_per_sec_per_gpu": 56.89, "tokens/trainable": 204189} +{"epoch": 0.765625, "grad_norm": 1.4388701915740967, "learning_rate": 7.554587923561324e-05, "loss": 0.2646244168281555, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.99, "memory/max_allocated (GiB)": 38.99, "ppl": 1.30294, "step": 196, "tokens/total": 6791728, "tokens/train_per_sec_per_gpu": 51.77, "tokens/trainable": 205161} +{"epoch": 0.76953125, "grad_norm": 1.6115862131118774, "learning_rate": 7.528718610173511e-05, "loss": 0.25530195236206055, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.29085, "step": 197, "tokens/total": 6827072, "tokens/train_per_sec_per_gpu": 63.24, "tokens/trainable": 206255} +{"epoch": 0.7734375, "grad_norm": 1.964750051498413, "learning_rate": 7.502764873523431e-05, "loss": 0.3165171146392822, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.37234, "step": 198, "tokens/total": 6861120, "tokens/train_per_sec_per_gpu": 61.41, "tokens/trainable": 207282} +{"epoch": 0.77734375, "grad_norm": 1.6865402460098267, "learning_rate": 7.476727793652011e-05, "loss": 0.2804451882839203, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.72, "memory/max_allocated (GiB)": 36.72, "ppl": 1.32372, "step": 199, "tokens/total": 6894352, "tokens/train_per_sec_per_gpu": 62.4, "tokens/trainable": 208339} +{"epoch": 0.78125, "grad_norm": 27.275827407836914, "learning_rate": 7.450608454068415e-05, "loss": 0.24937748908996582, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.28323, "step": 200, "tokens/total": 6928848, "tokens/train_per_sec_per_gpu": 57.66, "tokens/trainable": 209405} +{"epoch": 0.78515625, "grad_norm": 2.353294610977173, "learning_rate": 7.424407941704987e-05, "loss": 0.2523101270198822, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.49, "memory/max_allocated (GiB)": 38.49, "ppl": 1.287, "step": 201, "tokens/total": 6965504, "tokens/train_per_sec_per_gpu": 67.0, "tokens/trainable": 210550} +{"epoch": 0.7890625, "grad_norm": 1.8495022058486938, "learning_rate": 7.398127346871986e-05, "loss": 0.26536551117897034, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.61, "memory/max_allocated (GiB)": 39.61, "ppl": 1.30391, "step": 202, "tokens/total": 7002448, "tokens/train_per_sec_per_gpu": 54.85, "tokens/trainable": 211593} +{"epoch": 0.79296875, "grad_norm": 2.2459795475006104, "learning_rate": 7.371767763212238e-05, "loss": 0.2604941725730896, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.52, "memory/max_allocated (GiB)": 39.52, "ppl": 1.29757, "step": 203, "tokens/total": 7037824, "tokens/train_per_sec_per_gpu": 62.5, "tokens/trainable": 212644} +{"epoch": 0.796875, "grad_norm": 2.463927984237671, "learning_rate": 7.345330287655617e-05, "loss": 0.332345575094223, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.09, "memory/max_allocated (GiB)": 39.09, "ppl": 1.39423, "step": 204, "tokens/total": 7075920, "tokens/train_per_sec_per_gpu": 48.67, "tokens/trainable": 213706} +{"epoch": 0.80078125, "grad_norm": 1.5724210739135742, "learning_rate": 7.31881602037339e-05, "loss": 0.3082754611968994, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.69, "memory/max_allocated (GiB)": 36.69, "ppl": 1.36108, "step": 205, "tokens/total": 7107888, "tokens/train_per_sec_per_gpu": 62.63, "tokens/trainable": 214717} +{"epoch": 0.8046875, "grad_norm": 2.9074013233184814, "learning_rate": 7.29222606473245e-05, "loss": 0.2936800718307495, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.34135, "step": 206, "tokens/total": 7140720, "tokens/train_per_sec_per_gpu": 61.39, "tokens/trainable": 215735} +{"epoch": 0.80859375, "grad_norm": 3.1549344062805176, "learning_rate": 7.265561527249383e-05, "loss": 0.4259708821773529, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.53108, "step": 207, "tokens/total": 7174720, "tokens/train_per_sec_per_gpu": 62.25, "tokens/trainable": 216760} +{"epoch": 0.8125, "grad_norm": 1.396822214126587, "learning_rate": 7.238823517544436e-05, "loss": 0.2142832726240158, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.23897, "step": 208, "tokens/total": 7211872, "tokens/train_per_sec_per_gpu": 68.44, "tokens/trainable": 217940} +{"epoch": 0.81640625, "grad_norm": 1.3470375537872314, "learning_rate": 7.212013148295333e-05, "loss": 0.26368629932403564, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.30172, "step": 209, "tokens/total": 7245856, "tokens/train_per_sec_per_gpu": 63.2, "tokens/trainable": 219019} +{"epoch": 0.8203125, "grad_norm": 2.045835256576538, "learning_rate": 7.185131535190975e-05, "loss": 0.2884339690208435, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.06, "memory/max_allocated (GiB)": 37.06, "ppl": 1.33434, "step": 210, "tokens/total": 7278448, "tokens/train_per_sec_per_gpu": 63.51, "tokens/trainable": 220090} +{"epoch": 0.82421875, "grad_norm": 1.7613565921783447, "learning_rate": 7.158179796885005e-05, "loss": 0.2659681439399719, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.30469, "step": 211, "tokens/total": 7313072, "tokens/train_per_sec_per_gpu": 61.88, "tokens/trainable": 221143} +{"epoch": 0.828125, "grad_norm": 1.6588056087493896, "learning_rate": 7.131159054949273e-05, "loss": 0.29131025075912476, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.33818, "step": 212, "tokens/total": 7351456, "tokens/train_per_sec_per_gpu": 54.73, "tokens/trainable": 222182} +{"epoch": 0.83203125, "grad_norm": 2.3227829933166504, "learning_rate": 7.104070433827139e-05, "loss": 0.3478538691997528, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.41603, "step": 213, "tokens/total": 7386880, "tokens/train_per_sec_per_gpu": 51.12, "tokens/trainable": 223124} +{"epoch": 0.8359375, "grad_norm": 1.181349754333496, "learning_rate": 7.076915060786705e-05, "loss": 0.29040011763572693, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.33696, "step": 214, "tokens/total": 7419696, "tokens/train_per_sec_per_gpu": 63.02, "tokens/trainable": 224173} +{"epoch": 0.83984375, "grad_norm": 2.0790393352508545, "learning_rate": 7.049694065873882e-05, "loss": 0.2847173810005188, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.07, "memory/max_allocated (GiB)": 38.07, "ppl": 1.32939, "step": 215, "tokens/total": 7454352, "tokens/train_per_sec_per_gpu": 55.87, "tokens/trainable": 225219} +{"epoch": 0.84375, "grad_norm": 1.6800156831741333, "learning_rate": 7.022408581865382e-05, "loss": 0.244134783744812, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.27652, "step": 216, "tokens/total": 7488496, "tokens/train_per_sec_per_gpu": 57.72, "tokens/trainable": 226296} +{"epoch": 0.84765625, "grad_norm": 1.8132671117782593, "learning_rate": 6.99505974422157e-05, "loss": 0.25056156516075134, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.28475, "step": 217, "tokens/total": 7522976, "tokens/train_per_sec_per_gpu": 65.48, "tokens/trainable": 227382} +{"epoch": 0.8515625, "grad_norm": 4.157160758972168, "learning_rate": 6.967648691039213e-05, "loss": 0.3338316082954407, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.58, "memory/max_allocated (GiB)": 36.58, "ppl": 1.39631, "step": 218, "tokens/total": 7554848, "tokens/train_per_sec_per_gpu": 65.61, "tokens/trainable": 228413} +{"epoch": 0.85546875, "grad_norm": 2.096942663192749, "learning_rate": 6.940176563004123e-05, "loss": 0.2638465464115143, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.30193, "step": 219, "tokens/total": 7588896, "tokens/train_per_sec_per_gpu": 57.29, "tokens/trainable": 229386} +{"epoch": 0.859375, "grad_norm": 1.841373085975647, "learning_rate": 6.912644503343682e-05, "loss": 0.29923325777053833, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.34882, "step": 220, "tokens/total": 7627056, "tokens/train_per_sec_per_gpu": 55.44, "tokens/trainable": 230395} +{"epoch": 0.86328125, "grad_norm": 2.0590381622314453, "learning_rate": 6.885053657779273e-05, "loss": 0.31881463527679443, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.28, "memory/max_allocated (GiB)": 36.28, "ppl": 1.3755, "step": 221, "tokens/total": 7658368, "tokens/train_per_sec_per_gpu": 60.07, "tokens/trainable": 231413} +{"epoch": 0.8671875, "grad_norm": 2.3435306549072266, "learning_rate": 6.857405174478604e-05, "loss": 0.28703945875167847, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.33248, "step": 222, "tokens/total": 7693072, "tokens/train_per_sec_per_gpu": 57.59, "tokens/trainable": 232423} +{"epoch": 0.87109375, "grad_norm": 1.8315112590789795, "learning_rate": 6.82970020400792e-05, "loss": 0.28068608045578003, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.32404, "step": 223, "tokens/total": 7730288, "tokens/train_per_sec_per_gpu": 52.85, "tokens/trainable": 233444} +{"epoch": 0.875, "grad_norm": 1.1742862462997437, "learning_rate": 6.801939899284132e-05, "loss": 0.20205534994602203, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.22392, "step": 224, "tokens/total": 7766304, "tokens/train_per_sec_per_gpu": 58.36, "tokens/trainable": 234523} +{"epoch": 0.87890625, "grad_norm": 2.194333553314209, "learning_rate": 6.774125415526827e-05, "loss": 0.22394436597824097, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.251, "step": 225, "tokens/total": 7799680, "tokens/train_per_sec_per_gpu": 70.61, "tokens/trainable": 235661} +{"epoch": 0.8828125, "grad_norm": 2.1985690593719482, "learning_rate": 6.746257910210214e-05, "loss": 0.25950032472610474, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.29628, "step": 226, "tokens/total": 7832912, "tokens/train_per_sec_per_gpu": 60.0, "tokens/trainable": 236679} +{"epoch": 0.88671875, "grad_norm": 2.412386894226074, "learning_rate": 6.718338543014937e-05, "loss": 0.4159224033355713, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 35.66, "memory/max_allocated (GiB)": 35.66, "ppl": 1.51577, "step": 227, "tokens/total": 7862912, "tokens/train_per_sec_per_gpu": 66.62, "tokens/trainable": 237683} +{"epoch": 0.890625, "grad_norm": 1.48162043094635, "learning_rate": 6.69036847577983e-05, "loss": 0.22288182377815247, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.24967, "step": 228, "tokens/total": 7896816, "tokens/train_per_sec_per_gpu": 61.18, "tokens/trainable": 238712} +{"epoch": 0.89453125, "grad_norm": 1.4342142343521118, "learning_rate": 6.662348872453553e-05, "loss": 0.2147141993045807, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.46, "memory/max_allocated (GiB)": 38.46, "ppl": 1.23951, "step": 229, "tokens/total": 7933376, "tokens/train_per_sec_per_gpu": 60.72, "tokens/trainable": 239780} +{"epoch": 0.8984375, "grad_norm": 1.2074164152145386, "learning_rate": 6.63428089904618e-05, "loss": 0.24919570982456207, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.28299, "step": 230, "tokens/total": 7972544, "tokens/train_per_sec_per_gpu": 52.16, "tokens/trainable": 240808} +{"epoch": 0.90234375, "grad_norm": 1.448659896850586, "learning_rate": 6.60616572358065e-05, "loss": 0.2851603329181671, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.32998, "step": 231, "tokens/total": 8007008, "tokens/train_per_sec_per_gpu": 60.88, "tokens/trainable": 241780} +{"epoch": 0.90625, "grad_norm": 1.250640630722046, "learning_rate": 6.578004516044172e-05, "loss": 0.2557676434516907, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.29145, "step": 232, "tokens/total": 8042864, "tokens/train_per_sec_per_gpu": 62.64, "tokens/trainable": 242824} +{"epoch": 0.91015625, "grad_norm": 2.5213208198547363, "learning_rate": 6.549798448339548e-05, "loss": 0.23176395893096924, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.6, "memory/max_allocated (GiB)": 38.6, "ppl": 1.26082, "step": 233, "tokens/total": 8080272, "tokens/train_per_sec_per_gpu": 60.1, "tokens/trainable": 243880} +{"epoch": 0.9140625, "grad_norm": 1.3038781881332397, "learning_rate": 6.521548694236384e-05, "loss": 0.24577391147613525, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.27861, "step": 234, "tokens/total": 8117584, "tokens/train_per_sec_per_gpu": 53.78, "tokens/trainable": 244924} +{"epoch": 0.91796875, "grad_norm": 1.2703611850738525, "learning_rate": 6.493256429322259e-05, "loss": 0.23744916915893555, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.26801, "step": 235, "tokens/total": 8153712, "tokens/train_per_sec_per_gpu": 55.41, "tokens/trainable": 245951} +{"epoch": 0.921875, "grad_norm": 1.3011550903320312, "learning_rate": 6.464922830953799e-05, "loss": 0.2617204785346985, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.29916, "step": 236, "tokens/total": 8192928, "tokens/train_per_sec_per_gpu": 52.34, "tokens/trainable": 246987} +{"epoch": 0.92578125, "grad_norm": 1.5214378833770752, "learning_rate": 6.436549078207688e-05, "loss": 0.23916849493980408, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.21, "memory/max_allocated (GiB)": 38.21, "ppl": 1.27019, "step": 237, "tokens/total": 8226016, "tokens/train_per_sec_per_gpu": 62.1, "tokens/trainable": 247987} +{"epoch": 0.9296875, "grad_norm": 1.876980185508728, "learning_rate": 6.408136351831592e-05, "loss": 0.24943572282791138, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 37.15, "memory/max_allocated (GiB)": 37.15, "ppl": 1.2833, "step": 238, "tokens/total": 8258384, "tokens/train_per_sec_per_gpu": 60.08, "tokens/trainable": 248984} +{"epoch": 0.93359375, "grad_norm": 1.257966160774231, "learning_rate": 6.379685834195036e-05, "loss": 0.23136630654335022, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.26032, "step": 239, "tokens/total": 8297888, "tokens/train_per_sec_per_gpu": 54.25, "tokens/trainable": 250136} +{"epoch": 0.9375, "grad_norm": 2.9221558570861816, "learning_rate": 6.351198709240186e-05, "loss": 0.2542756497859955, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.61, "memory/max_allocated (GiB)": 39.61, "ppl": 1.28953, "step": 240, "tokens/total": 8338016, "tokens/train_per_sec_per_gpu": 53.9, "tokens/trainable": 251225} +{"epoch": 0.94140625, "grad_norm": 1.9244543313980103, "learning_rate": 6.32267616243259e-05, "loss": 0.24915622174739838, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.28294, "step": 241, "tokens/total": 8370608, "tokens/train_per_sec_per_gpu": 63.39, "tokens/trainable": 252255} +{"epoch": 0.9453125, "grad_norm": 1.6649788618087769, "learning_rate": 6.294119380711849e-05, "loss": 0.3083919286727905, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.36123, "step": 242, "tokens/total": 8405040, "tokens/train_per_sec_per_gpu": 52.53, "tokens/trainable": 253213} +{"epoch": 0.94921875, "grad_norm": 2.05078125, "learning_rate": 6.265529552442209e-05, "loss": 0.29129064083099365, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.33815, "step": 243, "tokens/total": 8440400, "tokens/train_per_sec_per_gpu": 63.02, "tokens/trainable": 254308} +{"epoch": 0.953125, "grad_norm": 3.0252444744110107, "learning_rate": 6.236907867363127e-05, "loss": 0.24612706899642944, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.14, "memory/max_allocated (GiB)": 38.14, "ppl": 1.27906, "step": 244, "tokens/total": 8474736, "tokens/train_per_sec_per_gpu": 63.34, "tokens/trainable": 255342} +{"epoch": 0.95703125, "grad_norm": 3.1506597995758057, "learning_rate": 6.208255516539749e-05, "loss": 0.2387578934431076, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 36.34, "memory/max_allocated (GiB)": 36.34, "ppl": 1.26967, "step": 245, "tokens/total": 8506944, "tokens/train_per_sec_per_gpu": 60.37, "tokens/trainable": 256399} +{"epoch": 0.9609375, "grad_norm": 1.5199800729751587, "learning_rate": 6.179573692313344e-05, "loss": 0.2531103193759918, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 1.28803, "step": 246, "tokens/total": 8544064, "tokens/train_per_sec_per_gpu": 57.69, "tokens/trainable": 257415} +{"epoch": 0.96484375, "grad_norm": 2.843273401260376, "learning_rate": 6.150863588251694e-05, "loss": 0.2530381381511688, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.28793, "step": 247, "tokens/total": 8584336, "tokens/train_per_sec_per_gpu": 50.11, "tokens/trainable": 258472} +{"epoch": 0.96875, "grad_norm": 1.3163827657699585, "learning_rate": 6.122126399099419e-05, "loss": 0.21110253036022186, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.78, "memory/max_allocated (GiB)": 39.78, "ppl": 1.23504, "step": 248, "tokens/total": 8624304, "tokens/train_per_sec_per_gpu": 66.42, "tokens/trainable": 259728} +{"epoch": 0.97265625, "grad_norm": 1.9721230268478394, "learning_rate": 6.0933633207282615e-05, "loss": 0.3303869366645813, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.39151, "step": 249, "tokens/total": 8660672, "tokens/train_per_sec_per_gpu": 54.66, "tokens/trainable": 260799} +{"epoch": 0.9765625, "grad_norm": 1.4705926179885864, "learning_rate": 6.064575550087316e-05, "loss": 0.2377917766571045, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.63, "memory/max_allocated (GiB)": 39.63, "ppl": 1.26845, "step": 250, "tokens/total": 8697680, "tokens/train_per_sec_per_gpu": 57.08, "tokens/trainable": 261840} +{"epoch": 0.98046875, "grad_norm": 1.5865559577941895, "learning_rate": 6.0357642851532245e-05, "loss": 0.21871067583560944, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.24447, "step": 251, "tokens/total": 8734112, "tokens/train_per_sec_per_gpu": 54.79, "tokens/trainable": 262908} +{"epoch": 0.984375, "grad_norm": 1.5452772378921509, "learning_rate": 6.0069307248803294e-05, "loss": 0.280778706073761, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.32416, "step": 252, "tokens/total": 8769440, "tokens/train_per_sec_per_gpu": 50.45, "tokens/trainable": 263887} +{"epoch": 0.98828125, "grad_norm": 2.1400551795959473, "learning_rate": 5.9780760691507635e-05, "loss": 0.2625249922275543, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 36.25, "memory/max_allocated (GiB)": 36.25, "ppl": 1.30021, "step": 253, "tokens/total": 8800688, "tokens/train_per_sec_per_gpu": 60.84, "tokens/trainable": 264825} +{"epoch": 0.9921875, "grad_norm": 1.4195607900619507, "learning_rate": 5.9492015187245334e-05, "loss": 0.2360095977783203, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.72, "memory/max_allocated (GiB)": 38.72, "ppl": 1.26619, "step": 254, "tokens/total": 8838272, "tokens/train_per_sec_per_gpu": 54.67, "tokens/trainable": 265790} +{"epoch": 0.99609375, "grad_norm": 1.4508519172668457, "learning_rate": 5.920308275189541e-05, "loss": 0.27185970544815063, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 36.6, "memory/max_allocated (GiB)": 36.6, "ppl": 1.3124, "step": 255, "tokens/total": 8869952, "tokens/train_per_sec_per_gpu": 62.46, "tokens/trainable": 266788} +{"epoch": 1.0, "grad_norm": 1.1414936780929565, "learning_rate": 5.8913975409115874e-05, "loss": 0.20645630359649658, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.22931, "step": 256, "tokens/total": 8903312, "tokens/train_per_sec_per_gpu": 62.38, "tokens/trainable": 267847} +{"epoch": 1.00390625, "grad_norm": 1.234514594078064, "learning_rate": 5.8624705189843395e-05, "loss": 0.20393425226211548, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.22622, "step": 257, "tokens/total": 8940768, "tokens/train_per_sec_per_gpu": 55.28, "tokens/trainable": 268954} +{"epoch": 1.0078125, "grad_norm": 1.1729092597961426, "learning_rate": 5.833528413179249e-05, "loss": 0.2085447609424591, "memory/device_reserved (GiB)": 40.67, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.23188, "step": 258, "tokens/total": 8977392, "tokens/train_per_sec_per_gpu": 58.41, "tokens/trainable": 270053} +{"epoch": 1.01171875, "grad_norm": 1.1948126554489136, "learning_rate": 5.80457242789548e-05, "loss": 0.20397797226905823, "memory/device_reserved (GiB)": 42.74, "memory/max_active (GiB)": 39.65, "memory/max_allocated (GiB)": 39.65, "ppl": 1.22627, "step": 259, "tokens/total": 9017568, "tokens/train_per_sec_per_gpu": 57.3, "tokens/trainable": 271229} +{"epoch": 1.015625, "grad_norm": 0.9785726070404053, "learning_rate": 5.77560376810977e-05, "loss": 0.20176225900650024, "memory/device_reserved (GiB)": 42.75, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.22356, "step": 260, "tokens/total": 9050928, "tokens/train_per_sec_per_gpu": 69.61, "tokens/trainable": 272345} +{"epoch": 1.01953125, "grad_norm": 2.1092917919158936, "learning_rate": 5.7466236393263005e-05, "loss": 0.22958031296730042, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.25807, "step": 261, "tokens/total": 9085552, "tokens/train_per_sec_per_gpu": 61.21, "tokens/trainable": 273368} +{"epoch": 1.0234375, "grad_norm": 2.1331088542938232, "learning_rate": 5.717633247526522e-05, "loss": 0.2524651288986206, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 36.74, "memory/max_allocated (GiB)": 36.74, "ppl": 1.28719, "step": 262, "tokens/total": 9116240, "tokens/train_per_sec_per_gpu": 59.15, "tokens/trainable": 274319} +{"epoch": 1.02734375, "grad_norm": 2.008017063140869, "learning_rate": 5.688633799118971e-05, "loss": 0.24185000360012054, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.2736, "step": 263, "tokens/total": 9148688, "tokens/train_per_sec_per_gpu": 55.1, "tokens/trainable": 275254} +{"epoch": 1.03125, "grad_norm": 1.048625111579895, "learning_rate": 5.659626500889066e-05, "loss": 0.17660704255104065, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.09, "memory/max_allocated (GiB)": 37.09, "ppl": 1.19316, "step": 264, "tokens/total": 9182704, "tokens/train_per_sec_per_gpu": 64.42, "tokens/trainable": 276380} +{"epoch": 1.03515625, "grad_norm": 2.31329083442688, "learning_rate": 5.6306125599488905e-05, "loss": 0.21908167004585266, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 36.88, "memory/max_allocated (GiB)": 36.88, "ppl": 1.24493, "step": 265, "tokens/total": 9213552, "tokens/train_per_sec_per_gpu": 67.94, "tokens/trainable": 277390} +{"epoch": 1.0390625, "grad_norm": 1.3379850387573242, "learning_rate": 5.601593183686955e-05, "loss": 0.17283399403095245, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.18867, "step": 266, "tokens/total": 9250768, "tokens/train_per_sec_per_gpu": 59.68, "tokens/trainable": 278509} +{"epoch": 1.04296875, "grad_norm": 1.238128423690796, "learning_rate": 5.572569579717961e-05, "loss": 0.23091846704483032, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.25976, "step": 267, "tokens/total": 9286176, "tokens/train_per_sec_per_gpu": 61.55, "tokens/trainable": 279599} +{"epoch": 1.046875, "grad_norm": 1.1301465034484863, "learning_rate": 5.543542955832538e-05, "loss": 0.20337840914726257, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.22554, "step": 268, "tokens/total": 9326128, "tokens/train_per_sec_per_gpu": 52.46, "tokens/trainable": 280682} +{"epoch": 1.05078125, "grad_norm": 4.191782474517822, "learning_rate": 5.514514519946986e-05, "loss": 0.19591735303401947, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.57, "memory/max_allocated (GiB)": 39.57, "ppl": 1.21643, "step": 269, "tokens/total": 9366544, "tokens/train_per_sec_per_gpu": 56.81, "tokens/trainable": 281810} +{"epoch": 1.0546875, "grad_norm": 1.438751459121704, "learning_rate": 5.485485480053015e-05, "loss": 0.22125710546970367, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.52, "memory/max_allocated (GiB)": 39.52, "ppl": 1.24764, "step": 270, "tokens/total": 9404928, "tokens/train_per_sec_per_gpu": 48.66, "tokens/trainable": 282814} +{"epoch": 1.05859375, "grad_norm": 1.1051557064056396, "learning_rate": 5.4564570441674645e-05, "loss": 0.2114197164773941, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.23543, "step": 271, "tokens/total": 9438592, "tokens/train_per_sec_per_gpu": 65.88, "tokens/trainable": 283848} +{"epoch": 1.0625, "grad_norm": 3.421950340270996, "learning_rate": 5.42743042028204e-05, "loss": 0.2736812233924866, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.83, "memory/max_allocated (GiB)": 37.83, "ppl": 1.3148, "step": 272, "tokens/total": 9474144, "tokens/train_per_sec_per_gpu": 47.73, "tokens/trainable": 284806} +{"epoch": 1.06640625, "grad_norm": 1.4355881214141846, "learning_rate": 5.3984068163130464e-05, "loss": 0.18127524852752686, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.19875, "step": 273, "tokens/total": 9507904, "tokens/train_per_sec_per_gpu": 68.91, "tokens/trainable": 285981} +{"epoch": 1.0703125, "grad_norm": 1.6492828130722046, "learning_rate": 5.369387440051111e-05, "loss": 0.20852074027061462, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.18, "memory/max_allocated (GiB)": 37.18, "ppl": 1.23185, "step": 274, "tokens/total": 9541616, "tokens/train_per_sec_per_gpu": 58.72, "tokens/trainable": 286983} +{"epoch": 1.07421875, "grad_norm": 6.350356578826904, "learning_rate": 5.340373499110935e-05, "loss": 0.20096027851104736, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.22258, "step": 275, "tokens/total": 9577424, "tokens/train_per_sec_per_gpu": 58.49, "tokens/trainable": 288058} +{"epoch": 1.078125, "grad_norm": 1.780272126197815, "learning_rate": 5.3113662008810304e-05, "loss": 0.17967958748340607, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.19683, "step": 276, "tokens/total": 9617760, "tokens/train_per_sec_per_gpu": 50.25, "tokens/trainable": 289118} +{"epoch": 1.08203125, "grad_norm": 1.158144474029541, "learning_rate": 5.282366752473479e-05, "loss": 0.19610093533992767, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.29, "memory/max_allocated (GiB)": 38.29, "ppl": 1.21665, "step": 277, "tokens/total": 9655008, "tokens/train_per_sec_per_gpu": 57.46, "tokens/trainable": 290214} +{"epoch": 1.0859375, "grad_norm": 1.7694575786590576, "learning_rate": 5.2533763606737005e-05, "loss": 0.2587073743343353, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.28, "memory/max_allocated (GiB)": 39.28, "ppl": 1.29525, "step": 278, "tokens/total": 9694896, "tokens/train_per_sec_per_gpu": 47.8, "tokens/trainable": 291231} +{"epoch": 1.08984375, "grad_norm": 0.9634026885032654, "learning_rate": 5.224396231890232e-05, "loss": 0.18268759548664093, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.23, "memory/max_allocated (GiB)": 37.23, "ppl": 1.20044, "step": 279, "tokens/total": 9726672, "tokens/train_per_sec_per_gpu": 66.85, "tokens/trainable": 292347} +{"epoch": 1.09375, "grad_norm": 1.3345086574554443, "learning_rate": 5.195427572104522e-05, "loss": 0.21996504068374634, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.24603, "step": 280, "tokens/total": 9761136, "tokens/train_per_sec_per_gpu": 60.23, "tokens/trainable": 293385} +{"epoch": 1.09765625, "grad_norm": 1.193009376525879, "learning_rate": 5.166471586820751e-05, "loss": 0.21516120433807373, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.24006, "step": 281, "tokens/total": 9797888, "tokens/train_per_sec_per_gpu": 61.07, "tokens/trainable": 294482} +{"epoch": 1.1015625, "grad_norm": 1.342281699180603, "learning_rate": 5.1375294810156615e-05, "loss": 0.18369203805923462, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.20165, "step": 282, "tokens/total": 9832528, "tokens/train_per_sec_per_gpu": 59.74, "tokens/trainable": 295504} +{"epoch": 1.10546875, "grad_norm": 1.007673978805542, "learning_rate": 5.1086024590884144e-05, "loss": 0.19696104526519775, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.28, "memory/max_allocated (GiB)": 39.28, "ppl": 1.2177, "step": 283, "tokens/total": 9870784, "tokens/train_per_sec_per_gpu": 52.63, "tokens/trainable": 296505} +{"epoch": 1.109375, "grad_norm": 0.9181911945343018, "learning_rate": 5.079691724810461e-05, "loss": 0.18982771039009094, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.20904, "step": 284, "tokens/total": 9906656, "tokens/train_per_sec_per_gpu": 49.14, "tokens/trainable": 297474} +{"epoch": 1.11328125, "grad_norm": 1.7275903224945068, "learning_rate": 5.0507984812754684e-05, "loss": 0.20013722777366638, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.22157, "step": 285, "tokens/total": 9940208, "tokens/train_per_sec_per_gpu": 59.59, "tokens/trainable": 298521} +{"epoch": 1.1171875, "grad_norm": 1.3438571691513062, "learning_rate": 5.021923930849237e-05, "loss": 0.20150209963321686, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.22324, "step": 286, "tokens/total": 9974096, "tokens/train_per_sec_per_gpu": 56.64, "tokens/trainable": 299534} +{"epoch": 1.12109375, "grad_norm": 0.926074743270874, "learning_rate": 4.99306927511967e-05, "loss": 0.18163618445396423, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.19918, "step": 287, "tokens/total": 10007568, "tokens/train_per_sec_per_gpu": 62.21, "tokens/trainable": 300541} +{"epoch": 1.125, "grad_norm": 0.9374329447746277, "learning_rate": 4.964235714846775e-05, "loss": 0.2039945125579834, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 36.44, "memory/max_allocated (GiB)": 36.44, "ppl": 1.22629, "step": 288, "tokens/total": 10039280, "tokens/train_per_sec_per_gpu": 66.19, "tokens/trainable": 301535} +{"epoch": 1.12890625, "grad_norm": 1.408992886543274, "learning_rate": 4.9354244499126866e-05, "loss": 0.22862650454044342, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 36.23, "memory/max_allocated (GiB)": 36.23, "ppl": 1.25687, "step": 289, "tokens/total": 10070944, "tokens/train_per_sec_per_gpu": 63.07, "tokens/trainable": 302567} +{"epoch": 1.1328125, "grad_norm": 0.9155704975128174, "learning_rate": 4.90663667927174e-05, "loss": 0.18838702142238617, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.2073, "step": 290, "tokens/total": 10103776, "tokens/train_per_sec_per_gpu": 71.08, "tokens/trainable": 303594} +{"epoch": 1.13671875, "grad_norm": 1.2506353855133057, "learning_rate": 4.877873600900581e-05, "loss": 0.2053704559803009, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.07, "memory/max_allocated (GiB)": 38.07, "ppl": 1.22798, "step": 291, "tokens/total": 10139888, "tokens/train_per_sec_per_gpu": 55.87, "tokens/trainable": 304633} +{"epoch": 1.140625, "grad_norm": 1.3711837530136108, "learning_rate": 4.849136411748306e-05, "loss": 0.19221192598342896, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.04, "memory/max_allocated (GiB)": 39.04, "ppl": 1.21193, "step": 292, "tokens/total": 10177936, "tokens/train_per_sec_per_gpu": 53.57, "tokens/trainable": 305690} +{"epoch": 1.14453125, "grad_norm": 1.1708365678787231, "learning_rate": 4.8204263076866574e-05, "loss": 0.20074453949928284, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.22231, "step": 293, "tokens/total": 10213856, "tokens/train_per_sec_per_gpu": 57.47, "tokens/trainable": 306752} +{"epoch": 1.1484375, "grad_norm": 1.3957529067993164, "learning_rate": 4.791744483460251e-05, "loss": 0.21519093215465546, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.74, "memory/max_allocated (GiB)": 37.74, "ppl": 1.2401, "step": 294, "tokens/total": 10248352, "tokens/train_per_sec_per_gpu": 51.71, "tokens/trainable": 307746} +{"epoch": 1.15234375, "grad_norm": 1.811618685722351, "learning_rate": 4.7630921326368736e-05, "loss": 0.19083541631698608, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.21026, "step": 295, "tokens/total": 10287072, "tokens/train_per_sec_per_gpu": 49.98, "tokens/trainable": 308728} +{"epoch": 1.15625, "grad_norm": 1.9793753623962402, "learning_rate": 4.7344704475577916e-05, "loss": 0.2013179063796997, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 35.7, "memory/max_allocated (GiB)": 35.7, "ppl": 1.22301, "step": 296, "tokens/total": 10316640, "tokens/train_per_sec_per_gpu": 68.14, "tokens/trainable": 309754} +{"epoch": 1.16015625, "grad_norm": 1.5465575456619263, "learning_rate": 4.705880619288153e-05, "loss": 0.1984933465719223, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 38.37, "memory/max_allocated (GiB)": 38.37, "ppl": 1.21956, "step": 297, "tokens/total": 10352864, "tokens/train_per_sec_per_gpu": 52.92, "tokens/trainable": 310739} +{"epoch": 1.1640625, "grad_norm": 1.2663841247558594, "learning_rate": 4.677323837567412e-05, "loss": 0.18980097770690918, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.20901, "step": 298, "tokens/total": 10385696, "tokens/train_per_sec_per_gpu": 64.51, "tokens/trainable": 311795} +{"epoch": 1.16796875, "grad_norm": 2.3309497833251953, "learning_rate": 4.6488012907598146e-05, "loss": 0.2451009452342987, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 36.4, "memory/max_allocated (GiB)": 36.4, "ppl": 1.27775, "step": 299, "tokens/total": 10417904, "tokens/train_per_sec_per_gpu": 57.29, "tokens/trainable": 312750} +{"epoch": 1.171875, "grad_norm": 0.8346296548843384, "learning_rate": 4.620314165804964e-05, "loss": 0.17879845201969147, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.19578, "step": 300, "tokens/total": 10453728, "tokens/train_per_sec_per_gpu": 58.46, "tokens/trainable": 313846} +{"epoch": 1.17578125, "grad_norm": 0.9353626370429993, "learning_rate": 4.591863648168407e-05, "loss": 0.17927256226539612, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.19635, "step": 301, "tokens/total": 10488288, "tokens/train_per_sec_per_gpu": 59.24, "tokens/trainable": 314951} +{"epoch": 1.1796875, "grad_norm": 1.1228435039520264, "learning_rate": 4.5634509217923135e-05, "loss": 0.20379920303821564, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 37.96, "memory/max_allocated (GiB)": 37.96, "ppl": 1.22605, "step": 302, "tokens/total": 10521824, "tokens/train_per_sec_per_gpu": 60.96, "tokens/trainable": 315989} +{"epoch": 1.18359375, "grad_norm": 0.8323144316673279, "learning_rate": 4.535077169046201e-05, "loss": 0.19690443575382233, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 38.03, "memory/max_allocated (GiB)": 38.03, "ppl": 1.21763, "step": 303, "tokens/total": 10558432, "tokens/train_per_sec_per_gpu": 56.95, "tokens/trainable": 317019} +{"epoch": 1.1875, "grad_norm": 1.492882490158081, "learning_rate": 4.506743570677743e-05, "loss": 0.20834527909755707, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 36.35, "memory/max_allocated (GiB)": 36.35, "ppl": 1.23164, "step": 304, "tokens/total": 10588752, "tokens/train_per_sec_per_gpu": 71.87, "tokens/trainable": 318014} +{"epoch": 1.19140625, "grad_norm": 1.0067710876464844, "learning_rate": 4.478451305763618e-05, "loss": 0.19568869471549988, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.21615, "step": 305, "tokens/total": 10622336, "tokens/train_per_sec_per_gpu": 63.99, "tokens/trainable": 319052} +{"epoch": 1.1953125, "grad_norm": 0.8939895629882812, "learning_rate": 4.450201551660454e-05, "loss": 0.19382911920547485, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 36.41, "memory/max_allocated (GiB)": 36.41, "ppl": 1.21389, "step": 306, "tokens/total": 10653520, "tokens/train_per_sec_per_gpu": 63.07, "tokens/trainable": 320057} +{"epoch": 1.19921875, "grad_norm": 2.362983226776123, "learning_rate": 4.4219954839558276e-05, "loss": 0.20881545543670654, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.23222, "step": 307, "tokens/total": 10688144, "tokens/train_per_sec_per_gpu": 53.67, "tokens/trainable": 321069} +{"epoch": 1.203125, "grad_norm": 0.8312777280807495, "learning_rate": 4.393834276419352e-05, "loss": 0.20891928672790527, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 35.7, "memory/max_allocated (GiB)": 35.7, "ppl": 1.23235, "step": 308, "tokens/total": 10718032, "tokens/train_per_sec_per_gpu": 71.31, "tokens/trainable": 322101} +{"epoch": 1.20703125, "grad_norm": 1.3793206214904785, "learning_rate": 4.36571910095382e-05, "loss": 0.22395096719264984, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.25101, "step": 309, "tokens/total": 10756464, "tokens/train_per_sec_per_gpu": 55.01, "tokens/trainable": 323155} +{"epoch": 1.2109375, "grad_norm": 1.6460952758789062, "learning_rate": 4.337651127546448e-05, "loss": 0.24223531782627106, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.27409, "step": 310, "tokens/total": 10792064, "tokens/train_per_sec_per_gpu": 62.27, "tokens/trainable": 324194} +{"epoch": 1.21484375, "grad_norm": 1.7939600944519043, "learning_rate": 4.3096315242201736e-05, "loss": 0.21698351204395294, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 36.51, "memory/max_allocated (GiB)": 36.51, "ppl": 1.24232, "step": 311, "tokens/total": 10824800, "tokens/train_per_sec_per_gpu": 58.84, "tokens/trainable": 325230} +{"epoch": 1.21875, "grad_norm": 1.6921616792678833, "learning_rate": 4.2816614569850635e-05, "loss": 0.2053493857383728, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 38.66, "memory/max_allocated (GiB)": 38.66, "ppl": 1.22795, "step": 312, "tokens/total": 10860576, "tokens/train_per_sec_per_gpu": 58.11, "tokens/trainable": 326230} +{"epoch": 1.22265625, "grad_norm": 1.467287540435791, "learning_rate": 4.2537420897897864e-05, "loss": 0.21924567222595215, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 38.72, "memory/max_allocated (GiB)": 38.72, "ppl": 1.24514, "step": 313, "tokens/total": 10896224, "tokens/train_per_sec_per_gpu": 53.58, "tokens/trainable": 327247} +{"epoch": 1.2265625, "grad_norm": 2.5353384017944336, "learning_rate": 4.225874584473174e-05, "loss": 0.2286730706691742, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.25693, "step": 314, "tokens/total": 10928768, "tokens/train_per_sec_per_gpu": 63.88, "tokens/trainable": 328261} +{"epoch": 1.23046875, "grad_norm": 1.3028435707092285, "learning_rate": 4.19806010071587e-05, "loss": 0.2133600413799286, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.23783, "step": 315, "tokens/total": 10966848, "tokens/train_per_sec_per_gpu": 54.48, "tokens/trainable": 329344} +{"epoch": 1.234375, "grad_norm": 1.1701239347457886, "learning_rate": 4.170299795992081e-05, "loss": 0.2350730150938034, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 36.88, "memory/max_allocated (GiB)": 36.88, "ppl": 1.265, "step": 316, "tokens/total": 11000416, "tokens/train_per_sec_per_gpu": 60.21, "tokens/trainable": 330395} +{"epoch": 1.23828125, "grad_norm": 1.3190970420837402, "learning_rate": 4.142594825521398e-05, "loss": 0.1847706437110901, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.20294, "step": 317, "tokens/total": 11039520, "tokens/train_per_sec_per_gpu": 51.29, "tokens/trainable": 331452} +{"epoch": 1.2421875, "grad_norm": 1.0218251943588257, "learning_rate": 4.114946342220728e-05, "loss": 0.1860128939151764, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.20444, "step": 318, "tokens/total": 11076096, "tokens/train_per_sec_per_gpu": 57.09, "tokens/trainable": 332542} +{"epoch": 1.24609375, "grad_norm": 2.7351393699645996, "learning_rate": 4.087355496656321e-05, "loss": 0.18877556920051575, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.20777, "step": 319, "tokens/total": 11109728, "tokens/train_per_sec_per_gpu": 67.97, "tokens/trainable": 333659} +{"epoch": 1.25, "grad_norm": 1.7309229373931885, "learning_rate": 4.05982343699588e-05, "loss": 0.21566486358642578, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.24069, "step": 320, "tokens/total": 11148976, "tokens/train_per_sec_per_gpu": 58.79, "tokens/trainable": 334805} +{"epoch": 1.25390625, "grad_norm": 1.733705997467041, "learning_rate": 4.0323513089607876e-05, "loss": 0.17112106084823608, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.18663, "step": 321, "tokens/total": 11183888, "tokens/train_per_sec_per_gpu": 57.58, "tokens/trainable": 335895} +{"epoch": 1.2578125, "grad_norm": 1.1677560806274414, "learning_rate": 4.004940255778431e-05, "loss": 0.17812010645866394, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 39.63, "memory/max_allocated (GiB)": 39.63, "ppl": 1.19497, "step": 322, "tokens/total": 11222608, "tokens/train_per_sec_per_gpu": 56.52, "tokens/trainable": 336952} +{"epoch": 1.26171875, "grad_norm": 0.9166269898414612, "learning_rate": 3.977591418134619e-05, "loss": 0.19377058744430542, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.21382, "step": 323, "tokens/total": 11257568, "tokens/train_per_sec_per_gpu": 61.55, "tokens/trainable": 338006} +{"epoch": 1.265625, "grad_norm": 1.7123030424118042, "learning_rate": 3.95030593412612e-05, "loss": 0.22165067493915558, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.24814, "step": 324, "tokens/total": 11293472, "tokens/train_per_sec_per_gpu": 57.24, "tokens/trainable": 339026} +{"epoch": 1.26953125, "grad_norm": 1.188162922859192, "learning_rate": 3.923084939213296e-05, "loss": 0.22013342380523682, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 35.75, "memory/max_allocated (GiB)": 35.75, "ppl": 1.24624, "step": 325, "tokens/total": 11324048, "tokens/train_per_sec_per_gpu": 67.81, "tokens/trainable": 340071} +{"epoch": 1.2734375, "grad_norm": 4.439968585968018, "learning_rate": 3.895929566172861e-05, "loss": 0.1643930971622467, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.17868, "step": 326, "tokens/total": 11360064, "tokens/train_per_sec_per_gpu": 60.13, "tokens/trainable": 341175} +{"epoch": 1.27734375, "grad_norm": 0.758490800857544, "learning_rate": 3.868840945050728e-05, "loss": 0.16608965396881104, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 39.19, "memory/max_allocated (GiB)": 39.19, "ppl": 1.18068, "step": 327, "tokens/total": 11399008, "tokens/train_per_sec_per_gpu": 55.96, "tokens/trainable": 342264} +{"epoch": 1.28125, "grad_norm": 0.9933422803878784, "learning_rate": 3.841820203114995e-05, "loss": 0.20499080419540405, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 36.78, "memory/max_allocated (GiB)": 36.78, "ppl": 1.22751, "step": 328, "tokens/total": 11430752, "tokens/train_per_sec_per_gpu": 67.35, "tokens/trainable": 343353} +{"epoch": 1.28515625, "grad_norm": 0.8855421543121338, "learning_rate": 3.814868464809027e-05, "loss": 0.18139535188674927, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 36.77, "memory/max_allocated (GiB)": 36.77, "ppl": 1.19889, "step": 329, "tokens/total": 11462688, "tokens/train_per_sec_per_gpu": 61.14, "tokens/trainable": 344349} +{"epoch": 1.2890625, "grad_norm": 1.0058590173721313, "learning_rate": 3.787986851704667e-05, "loss": 0.22799834609031677, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.25608, "step": 330, "tokens/total": 11496240, "tokens/train_per_sec_per_gpu": 65.53, "tokens/trainable": 345405} +{"epoch": 1.29296875, "grad_norm": 0.7192620038986206, "learning_rate": 3.7611764824555654e-05, "loss": 0.16373172402381897, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 37.84, "memory/max_allocated (GiB)": 37.84, "ppl": 1.1779, "step": 331, "tokens/total": 11527968, "tokens/train_per_sec_per_gpu": 75.96, "tokens/trainable": 346558} +{"epoch": 1.296875, "grad_norm": 0.9497982859611511, "learning_rate": 3.734438472750619e-05, "loss": 0.19882287085056305, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.21997, "step": 332, "tokens/total": 11567936, "tokens/train_per_sec_per_gpu": 53.08, "tokens/trainable": 347670} +{"epoch": 1.30078125, "grad_norm": 1.2202850580215454, "learning_rate": 3.707773935267552e-05, "loss": 0.20096993446350098, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.22259, "step": 333, "tokens/total": 11604992, "tokens/train_per_sec_per_gpu": 57.95, "tokens/trainable": 348799} +{"epoch": 1.3046875, "grad_norm": 1.3448477983474731, "learning_rate": 3.68118397962661e-05, "loss": 0.19954754412174225, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 38.49, "memory/max_allocated (GiB)": 38.49, "ppl": 1.22085, "step": 334, "tokens/total": 11639744, "tokens/train_per_sec_per_gpu": 55.75, "tokens/trainable": 349761} +{"epoch": 1.30859375, "grad_norm": 1.1003201007843018, "learning_rate": 3.654669712344384e-05, "loss": 0.18146446347236633, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 36.72, "memory/max_allocated (GiB)": 36.72, "ppl": 1.19897, "step": 335, "tokens/total": 11670944, "tokens/train_per_sec_per_gpu": 74.92, "tokens/trainable": 350817} +{"epoch": 1.3125, "grad_norm": 0.8396841883659363, "learning_rate": 3.628232236787763e-05, "loss": 0.16799013316631317, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 37.91, "memory/max_allocated (GiB)": 37.91, "ppl": 1.18292, "step": 336, "tokens/total": 11706544, "tokens/train_per_sec_per_gpu": 56.07, "tokens/trainable": 351930} +{"epoch": 1.31640625, "grad_norm": 1.0873693227767944, "learning_rate": 3.6018726531280144e-05, "loss": 0.1663203239440918, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.18095, "step": 337, "tokens/total": 11743952, "tokens/train_per_sec_per_gpu": 60.27, "tokens/trainable": 353038} +{"epoch": 1.3203125, "grad_norm": 1.0677043199539185, "learning_rate": 3.575592058295017e-05, "loss": 0.21620437502861023, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.24136, "step": 338, "tokens/total": 11777936, "tokens/train_per_sec_per_gpu": 63.55, "tokens/trainable": 354118} +{"epoch": 1.32421875, "grad_norm": 1.376201868057251, "learning_rate": 3.549391545931585e-05, "loss": 0.19825464487075806, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 38.93, "memory/max_allocated (GiB)": 38.93, "ppl": 1.21927, "step": 339, "tokens/total": 11812416, "tokens/train_per_sec_per_gpu": 62.26, "tokens/trainable": 355168} +{"epoch": 1.328125, "grad_norm": 1.1695078611373901, "learning_rate": 3.5232722063479914e-05, "loss": 0.19577595591545105, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.21625, "step": 340, "tokens/total": 11846704, "tokens/train_per_sec_per_gpu": 67.21, "tokens/trainable": 356250} +{"epoch": 1.33203125, "grad_norm": 0.9021807909011841, "learning_rate": 3.49723512647657e-05, "loss": 0.16975712776184082, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.18502, "step": 341, "tokens/total": 11885968, "tokens/train_per_sec_per_gpu": 53.99, "tokens/trainable": 357366} +{"epoch": 1.3359375, "grad_norm": 0.975933313369751, "learning_rate": 3.471281389826491e-05, "loss": 0.1918041855096817, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.21143, "step": 342, "tokens/total": 11917024, "tokens/train_per_sec_per_gpu": 63.24, "tokens/trainable": 358319} +{"epoch": 1.33984375, "grad_norm": 0.939338743686676, "learning_rate": 3.4454120764386764e-05, "loss": 0.17629505693912506, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.19279, "step": 343, "tokens/total": 11955648, "tokens/train_per_sec_per_gpu": 56.5, "tokens/trainable": 359399} +{"epoch": 1.34375, "grad_norm": 1.4815893173217773, "learning_rate": 3.4196282628408526e-05, "loss": 0.20626235008239746, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 37.58, "memory/max_allocated (GiB)": 37.58, "ppl": 1.22908, "step": 344, "tokens/total": 11989936, "tokens/train_per_sec_per_gpu": 62.12, "tokens/trainable": 360454} +{"epoch": 1.34765625, "grad_norm": 0.9789279103279114, "learning_rate": 3.3939310220027456e-05, "loss": 0.18571102619171143, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.20407, "step": 345, "tokens/total": 12023968, "tokens/train_per_sec_per_gpu": 59.93, "tokens/trainable": 361543} +{"epoch": 1.3515625, "grad_norm": 1.2860594987869263, "learning_rate": 3.3683214232914404e-05, "loss": 0.16328121721744537, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.17737, "step": 346, "tokens/total": 12059632, "tokens/train_per_sec_per_gpu": 59.64, "tokens/trainable": 362623} +{"epoch": 1.35546875, "grad_norm": 0.6724425554275513, "learning_rate": 3.342800532426873e-05, "loss": 0.187378391623497, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.20608, "step": 347, "tokens/total": 12095616, "tokens/train_per_sec_per_gpu": 59.85, "tokens/trainable": 363641} +{"epoch": 1.359375, "grad_norm": 1.3253270387649536, "learning_rate": 3.317369411437484e-05, "loss": 0.20855563879013062, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.2319, "step": 348, "tokens/total": 12131488, "tokens/train_per_sec_per_gpu": 55.95, "tokens/trainable": 364672} +{"epoch": 1.36328125, "grad_norm": 0.7018193006515503, "learning_rate": 3.292029118616024e-05, "loss": 0.1780938059091568, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.19494, "step": 349, "tokens/total": 12164208, "tokens/train_per_sec_per_gpu": 56.84, "tokens/trainable": 365654} +{"epoch": 1.3671875, "grad_norm": 0.8184900879859924, "learning_rate": 3.266780708475511e-05, "loss": 0.17788729071617126, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 39.12, "memory/max_allocated (GiB)": 39.12, "ppl": 1.19469, "step": 350, "tokens/total": 12199616, "tokens/train_per_sec_per_gpu": 58.23, "tokens/trainable": 366729} +{"epoch": 1.37109375, "grad_norm": 1.204857587814331, "learning_rate": 3.241625231705354e-05, "loss": 0.17216476798057556, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.18787, "step": 351, "tokens/total": 12235664, "tokens/train_per_sec_per_gpu": 58.96, "tokens/trainable": 367791} +{"epoch": 1.375, "grad_norm": 1.0601831674575806, "learning_rate": 3.216563735127618e-05, "loss": 0.16192929446697235, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 39.12, "memory/max_allocated (GiB)": 39.12, "ppl": 1.17578, "step": 352, "tokens/total": 12270800, "tokens/train_per_sec_per_gpu": 60.91, "tokens/trainable": 368924} +{"epoch": 1.37890625, "grad_norm": 1.3233146667480469, "learning_rate": 3.191597261653475e-05, "loss": 0.19751808047294617, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.21838, "step": 353, "tokens/total": 12307568, "tokens/train_per_sec_per_gpu": 57.57, "tokens/trainable": 370005} +{"epoch": 1.3828125, "grad_norm": 1.1764957904815674, "learning_rate": 3.166726850239794e-05, "loss": 0.16820266842842102, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 37.18, "memory/max_allocated (GiB)": 37.18, "ppl": 1.18318, "step": 354, "tokens/total": 12341200, "tokens/train_per_sec_per_gpu": 62.4, "tokens/trainable": 371083} +{"epoch": 1.38671875, "grad_norm": 2.1455960273742676, "learning_rate": 3.141953535845912e-05, "loss": 0.21798352897167206, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 36.16, "memory/max_allocated (GiB)": 36.16, "ppl": 1.24357, "step": 355, "tokens/total": 12372016, "tokens/train_per_sec_per_gpu": 62.91, "tokens/trainable": 372021} +{"epoch": 1.390625, "grad_norm": 1.500195860862732, "learning_rate": 3.11727834939056e-05, "loss": 0.23066437244415283, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 36.28, "memory/max_allocated (GiB)": 36.28, "ppl": 1.25944, "step": 356, "tokens/total": 12403856, "tokens/train_per_sec_per_gpu": 59.48, "tokens/trainable": 373052} +{"epoch": 1.39453125, "grad_norm": 1.6035617589950562, "learning_rate": 3.092702317708967e-05, "loss": 0.20057982206344604, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.22211, "step": 357, "tokens/total": 12437584, "tokens/train_per_sec_per_gpu": 62.32, "tokens/trainable": 374068} +{"epoch": 1.3984375, "grad_norm": 0.8461552262306213, "learning_rate": 3.0682264635101276e-05, "loss": 0.1814747154712677, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 37.44, "memory/max_allocated (GiB)": 37.44, "ppl": 1.19898, "step": 358, "tokens/total": 12470096, "tokens/train_per_sec_per_gpu": 62.13, "tokens/trainable": 375077} +{"epoch": 1.40234375, "grad_norm": 0.8888952136039734, "learning_rate": 3.0438518053342407e-05, "loss": 0.17355909943580627, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.18953, "step": 359, "tokens/total": 12509168, "tokens/train_per_sec_per_gpu": 55.71, "tokens/trainable": 376205} +{"epoch": 1.40625, "grad_norm": 0.987471342086792, "learning_rate": 3.0195793575103266e-05, "loss": 0.17142128944396973, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.18699, "step": 360, "tokens/total": 12548208, "tokens/train_per_sec_per_gpu": 54.76, "tokens/trainable": 377281} +{"epoch": 1.41015625, "grad_norm": 1.683725118637085, "learning_rate": 2.9954101301140146e-05, "loss": 0.19450974464416504, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.21472, "step": 361, "tokens/total": 12583552, "tokens/train_per_sec_per_gpu": 58.73, "tokens/trainable": 378322} +{"epoch": 1.4140625, "grad_norm": 0.8000853061676025, "learning_rate": 2.9713451289255123e-05, "loss": 0.20968398451805115, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 36.9, "memory/max_allocated (GiB)": 36.9, "ppl": 1.23329, "step": 362, "tokens/total": 12612944, "tokens/train_per_sec_per_gpu": 61.78, "tokens/trainable": 379351} +{"epoch": 1.41796875, "grad_norm": 0.6453067660331726, "learning_rate": 2.9473853553877484e-05, "loss": 0.15843386948108673, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.17167, "step": 363, "tokens/total": 12646880, "tokens/train_per_sec_per_gpu": 60.21, "tokens/trainable": 380390} +{"epoch": 1.421875, "grad_norm": 1.6911985874176025, "learning_rate": 2.9235318065647e-05, "loss": 0.19468820095062256, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 39.61, "memory/max_allocated (GiB)": 39.61, "ppl": 1.21493, "step": 364, "tokens/total": 12686896, "tokens/train_per_sec_per_gpu": 53.32, "tokens/trainable": 381449} +{"epoch": 1.42578125, "grad_norm": 0.8937534093856812, "learning_rate": 2.8997854750998964e-05, "loss": 0.1794549524784088, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.19657, "step": 365, "tokens/total": 12726416, "tokens/train_per_sec_per_gpu": 58.74, "tokens/trainable": 382566} +{"epoch": 1.4296875, "grad_norm": 1.0900917053222656, "learning_rate": 2.8761473491751258e-05, "loss": 0.19244080781936646, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.63, "memory/max_allocated (GiB)": 36.63, "ppl": 1.2122, "step": 366, "tokens/total": 12759248, "tokens/train_per_sec_per_gpu": 59.82, "tokens/trainable": 383564} +{"epoch": 1.43359375, "grad_norm": 1.5739927291870117, "learning_rate": 2.8526184124692883e-05, "loss": 0.15152812004089355, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.16361, "step": 367, "tokens/total": 12795904, "tokens/train_per_sec_per_gpu": 52.51, "tokens/trainable": 384667} +{"epoch": 1.4375, "grad_norm": 1.2147634029388428, "learning_rate": 2.829199644117484e-05, "loss": 0.18278566002845764, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.20056, "step": 368, "tokens/total": 12829616, "tokens/train_per_sec_per_gpu": 62.78, "tokens/trainable": 385630} +{"epoch": 1.44140625, "grad_norm": 1.0281314849853516, "learning_rate": 2.8058920186702553e-05, "loss": 0.1750621199607849, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.91, "memory/max_allocated (GiB)": 38.91, "ppl": 1.19132, "step": 369, "tokens/total": 12867744, "tokens/train_per_sec_per_gpu": 58.75, "tokens/trainable": 386744} +{"epoch": 1.4453125, "grad_norm": 1.5767312049865723, "learning_rate": 2.782696506053033e-05, "loss": 0.1953611820936203, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.21575, "step": 370, "tokens/total": 12905936, "tokens/train_per_sec_per_gpu": 52.53, "tokens/trainable": 387739} +{"epoch": 1.44921875, "grad_norm": 1.2168899774551392, "learning_rate": 2.7596140715257824e-05, "loss": 0.19472242891788483, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.21497, "step": 371, "tokens/total": 12941216, "tokens/train_per_sec_per_gpu": 61.39, "tokens/trainable": 388842} +{"epoch": 1.453125, "grad_norm": 1.0933375358581543, "learning_rate": 2.7366456756428184e-05, "loss": 0.17193248867988586, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.1876, "step": 372, "tokens/total": 12975584, "tokens/train_per_sec_per_gpu": 59.97, "tokens/trainable": 389878} +{"epoch": 1.45703125, "grad_norm": 2.2480037212371826, "learning_rate": 2.7137922742128486e-05, "loss": 0.178237646818161, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.21, "memory/max_allocated (GiB)": 38.21, "ppl": 1.19511, "step": 373, "tokens/total": 13010464, "tokens/train_per_sec_per_gpu": 61.36, "tokens/trainable": 390945} +{"epoch": 1.4609375, "grad_norm": 0.7743867635726929, "learning_rate": 2.691054818259188e-05, "loss": 0.16842560470104218, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.4, "memory/max_allocated (GiB)": 37.4, "ppl": 1.18344, "step": 374, "tokens/total": 13045136, "tokens/train_per_sec_per_gpu": 61.47, "tokens/trainable": 392058} +{"epoch": 1.46484375, "grad_norm": 1.1837552785873413, "learning_rate": 2.6684342539801933e-05, "loss": 0.22727744281291962, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.49, "memory/max_allocated (GiB)": 36.49, "ppl": 1.25518, "step": 375, "tokens/total": 13076400, "tokens/train_per_sec_per_gpu": 61.33, "tokens/trainable": 393036} +{"epoch": 1.46875, "grad_norm": 0.8049536943435669, "learning_rate": 2.645931522709877e-05, "loss": 0.16839474439620972, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.54, "memory/max_allocated (GiB)": 39.54, "ppl": 1.1834, "step": 376, "tokens/total": 13112144, "tokens/train_per_sec_per_gpu": 73.0, "tokens/trainable": 394207} +{"epoch": 1.47265625, "grad_norm": 1.2875332832336426, "learning_rate": 2.6235475608787365e-05, "loss": 0.17698965966701508, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.6, "memory/max_allocated (GiB)": 38.6, "ppl": 1.19362, "step": 377, "tokens/total": 13147872, "tokens/train_per_sec_per_gpu": 58.98, "tokens/trainable": 395236} +{"epoch": 1.4765625, "grad_norm": 0.7719260454177856, "learning_rate": 2.6012832999747916e-05, "loss": 0.19621920585632324, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.21679, "step": 378, "tokens/total": 13178656, "tokens/train_per_sec_per_gpu": 63.88, "tokens/trainable": 396213} +{"epoch": 1.48046875, "grad_norm": 0.9557688236236572, "learning_rate": 2.579139666504821e-05, "loss": 0.20320338010787964, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.22532, "step": 379, "tokens/total": 13218720, "tokens/train_per_sec_per_gpu": 51.2, "tokens/trainable": 397215} +{"epoch": 1.484375, "grad_norm": 0.819105863571167, "learning_rate": 2.557117581955798e-05, "loss": 0.17828653752803802, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 1.19517, "step": 380, "tokens/total": 13252544, "tokens/train_per_sec_per_gpu": 59.49, "tokens/trainable": 398270} +{"epoch": 1.48828125, "grad_norm": 0.9579155445098877, "learning_rate": 2.5352179627565532e-05, "loss": 0.22388999164104462, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.06, "memory/max_allocated (GiB)": 38.06, "ppl": 1.25093, "step": 381, "tokens/total": 13286832, "tokens/train_per_sec_per_gpu": 53.25, "tokens/trainable": 399274} +{"epoch": 1.4921875, "grad_norm": 0.8220188617706299, "learning_rate": 2.5134417202396277e-05, "loss": 0.19739669561386108, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.21823, "step": 382, "tokens/total": 13320432, "tokens/train_per_sec_per_gpu": 57.44, "tokens/trainable": 400243} +{"epoch": 1.49609375, "grad_norm": 0.8155992031097412, "learning_rate": 2.491789760603361e-05, "loss": 0.18290305137634277, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.2007, "step": 383, "tokens/total": 13356928, "tokens/train_per_sec_per_gpu": 53.33, "tokens/trainable": 401259} +{"epoch": 1.5, "grad_norm": 1.0240693092346191, "learning_rate": 2.4702629848741764e-05, "loss": 0.18413880467414856, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.20218, "step": 384, "tokens/total": 13391568, "tokens/train_per_sec_per_gpu": 62.12, "tokens/trainable": 402263} +{"epoch": 1.50390625, "grad_norm": 1.1684728860855103, "learning_rate": 2.4488622888690785e-05, "loss": 0.18859638273715973, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.20755, "step": 385, "tokens/total": 13425152, "tokens/train_per_sec_per_gpu": 54.84, "tokens/trainable": 403239} +{"epoch": 1.5078125, "grad_norm": 0.8764642477035522, "learning_rate": 2.427588563158384e-05, "loss": 0.16204319894313812, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.11, "memory/max_allocated (GiB)": 39.11, "ppl": 1.17591, "step": 386, "tokens/total": 13459504, "tokens/train_per_sec_per_gpu": 62.06, "tokens/trainable": 404313} +{"epoch": 1.51171875, "grad_norm": 4.302525043487549, "learning_rate": 2.406442693028651e-05, "loss": 0.20630338788032532, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.22913, "step": 387, "tokens/total": 13491136, "tokens/train_per_sec_per_gpu": 60.77, "tokens/trainable": 405263} +{"epoch": 1.515625, "grad_norm": 0.9543612003326416, "learning_rate": 2.3854255584458547e-05, "loss": 0.176202654838562, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.19268, "step": 388, "tokens/total": 13530112, "tokens/train_per_sec_per_gpu": 51.02, "tokens/trainable": 406280} +{"epoch": 1.51953125, "grad_norm": 0.8750304579734802, "learning_rate": 2.3645380340187508e-05, "loss": 0.1776118129491806, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.14, "memory/max_allocated (GiB)": 39.14, "ppl": 1.19436, "step": 389, "tokens/total": 13565680, "tokens/train_per_sec_per_gpu": 62.25, "tokens/trainable": 407354} +{"epoch": 1.5234375, "grad_norm": 1.1144131422042847, "learning_rate": 2.3437809889624914e-05, "loss": 0.20075100660324097, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.52, "memory/max_allocated (GiB)": 36.52, "ppl": 1.22232, "step": 390, "tokens/total": 13596960, "tokens/train_per_sec_per_gpu": 71.42, "tokens/trainable": 408404} +{"epoch": 1.52734375, "grad_norm": 0.9171886444091797, "learning_rate": 2.3231552870624487e-05, "loss": 0.1624014526605606, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.17633, "step": 391, "tokens/total": 13627840, "tokens/train_per_sec_per_gpu": 70.17, "tokens/trainable": 409477} +{"epoch": 1.53125, "grad_norm": 0.6964625120162964, "learning_rate": 2.3026617866382657e-05, "loss": 0.15761849284172058, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.7, "memory/max_allocated (GiB)": 36.7, "ppl": 1.17072, "step": 392, "tokens/total": 13660848, "tokens/train_per_sec_per_gpu": 62.31, "tokens/trainable": 410563} +{"epoch": 1.53515625, "grad_norm": 0.6807044148445129, "learning_rate": 2.2823013405081507e-05, "loss": 0.15996554493904114, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.17347, "step": 393, "tokens/total": 13699024, "tokens/train_per_sec_per_gpu": 53.26, "tokens/trainable": 411635} +{"epoch": 1.5390625, "grad_norm": 0.7023879289627075, "learning_rate": 2.2620747959533722e-05, "loss": 0.15693137049674988, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.16992, "step": 394, "tokens/total": 13732336, "tokens/train_per_sec_per_gpu": 65.38, "tokens/trainable": 412746} +{"epoch": 1.54296875, "grad_norm": 0.7186459302902222, "learning_rate": 2.2419829946830123e-05, "loss": 0.17860747873783112, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.19555, "step": 395, "tokens/total": 13768112, "tokens/train_per_sec_per_gpu": 59.83, "tokens/trainable": 413775} +{"epoch": 1.546875, "grad_norm": 1.6796518564224243, "learning_rate": 2.2220267727989325e-05, "loss": 0.1671740710735321, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.74, "memory/max_allocated (GiB)": 37.74, "ppl": 1.18196, "step": 396, "tokens/total": 13802480, "tokens/train_per_sec_per_gpu": 59.69, "tokens/trainable": 414830} +{"epoch": 1.55078125, "grad_norm": 0.851715087890625, "learning_rate": 2.202206960760984e-05, "loss": 0.15885764360427856, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.17217, "step": 397, "tokens/total": 13840736, "tokens/train_per_sec_per_gpu": 55.84, "tokens/trainable": 415977} +{"epoch": 1.5546875, "grad_norm": 0.7575640082359314, "learning_rate": 2.182524383352446e-05, "loss": 0.18893510103225708, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.20796, "step": 398, "tokens/total": 13876896, "tokens/train_per_sec_per_gpu": 56.04, "tokens/trainable": 416993} +{"epoch": 1.55859375, "grad_norm": 0.8227819204330444, "learning_rate": 2.1629798596457056e-05, "loss": 0.1844748854637146, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.34, "memory/max_allocated (GiB)": 36.34, "ppl": 1.20259, "step": 399, "tokens/total": 13908992, "tokens/train_per_sec_per_gpu": 62.79, "tokens/trainable": 417998} +{"epoch": 1.5625, "grad_norm": 1.023768424987793, "learning_rate": 2.1435742029681725e-05, "loss": 0.18999852240085602, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.29, "memory/max_allocated (GiB)": 36.29, "ppl": 1.20925, "step": 400, "tokens/total": 13939456, "tokens/train_per_sec_per_gpu": 68.4, "tokens/trainable": 418959} +{"epoch": 1.56640625, "grad_norm": 1.3628404140472412, "learning_rate": 2.124308220868431e-05, "loss": 0.1584084928035736, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.17164, "step": 401, "tokens/total": 13975312, "tokens/train_per_sec_per_gpu": 56.65, "tokens/trainable": 420020} +{"epoch": 1.5703125, "grad_norm": 1.1305296421051025, "learning_rate": 2.105182715082638e-05, "loss": 0.16174668073654175, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.92, "memory/max_allocated (GiB)": 36.92, "ppl": 1.17556, "step": 402, "tokens/total": 14009056, "tokens/train_per_sec_per_gpu": 63.09, "tokens/trainable": 421113} +{"epoch": 1.57421875, "grad_norm": 1.3209508657455444, "learning_rate": 2.0861984815011552e-05, "loss": 0.18223226070404053, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 1.19989, "step": 403, "tokens/total": 14044320, "tokens/train_per_sec_per_gpu": 55.28, "tokens/trainable": 422084} +{"epoch": 1.578125, "grad_norm": 1.4008668661117554, "learning_rate": 2.0673563101354323e-05, "loss": 0.2146523892879486, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.23943, "step": 404, "tokens/total": 14079600, "tokens/train_per_sec_per_gpu": 52.81, "tokens/trainable": 423103} +{"epoch": 1.58203125, "grad_norm": 0.9277515411376953, "learning_rate": 2.0486569850851317e-05, "loss": 0.1771995723247528, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.19387, "step": 405, "tokens/total": 14114368, "tokens/train_per_sec_per_gpu": 61.31, "tokens/trainable": 424119} +{"epoch": 1.5859375, "grad_norm": 0.8596374988555908, "learning_rate": 2.0301012845054956e-05, "loss": 0.1839471310377121, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.20195, "step": 406, "tokens/total": 14153248, "tokens/train_per_sec_per_gpu": 51.26, "tokens/trainable": 425159} +{"epoch": 1.58984375, "grad_norm": 1.0201067924499512, "learning_rate": 2.011689980574966e-05, "loss": 0.17955338954925537, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.18, "memory/max_allocated (GiB)": 38.18, "ppl": 1.19668, "step": 407, "tokens/total": 14189632, "tokens/train_per_sec_per_gpu": 56.27, "tokens/trainable": 426175} +{"epoch": 1.59375, "grad_norm": 0.7795219421386719, "learning_rate": 1.993423839463052e-05, "loss": 0.1673658788204193, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.18219, "step": 408, "tokens/total": 14223072, "tokens/train_per_sec_per_gpu": 71.28, "tokens/trainable": 427252} +{"epoch": 1.59765625, "grad_norm": 0.8305946588516235, "learning_rate": 1.975303621298445e-05, "loss": 0.16275662183761597, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.17675, "step": 409, "tokens/total": 14259824, "tokens/train_per_sec_per_gpu": 57.05, "tokens/trainable": 428269} +{"epoch": 1.6015625, "grad_norm": 0.8229319453239441, "learning_rate": 1.957330080137385e-05, "loss": 0.18691827356815338, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.22, "memory/max_allocated (GiB)": 36.22, "ppl": 1.20553, "step": 410, "tokens/total": 14290400, "tokens/train_per_sec_per_gpu": 66.07, "tokens/trainable": 429288} +{"epoch": 1.60546875, "grad_norm": 0.8381948471069336, "learning_rate": 1.9395039639322864e-05, "loss": 0.20126815140247345, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.22295, "step": 411, "tokens/total": 14326528, "tokens/train_per_sec_per_gpu": 59.37, "tokens/trainable": 430334} +{"epoch": 1.609375, "grad_norm": 0.5874187350273132, "learning_rate": 1.9218260145006073e-05, "loss": 0.15583069622516632, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.16863, "step": 412, "tokens/total": 14362400, "tokens/train_per_sec_per_gpu": 64.41, "tokens/trainable": 431471} +{"epoch": 1.61328125, "grad_norm": 1.2909648418426514, "learning_rate": 1.904296967493982e-05, "loss": 0.1725354790687561, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.18831, "step": 413, "tokens/total": 14394864, "tokens/train_per_sec_per_gpu": 69.03, "tokens/trainable": 432516} +{"epoch": 1.6171875, "grad_norm": 0.7849239706993103, "learning_rate": 1.8869175523676064e-05, "loss": 0.15036237239837646, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.36, "memory/max_allocated (GiB)": 38.36, "ppl": 1.16226, "step": 414, "tokens/total": 14431648, "tokens/train_per_sec_per_gpu": 60.8, "tokens/trainable": 433651} +{"epoch": 1.62109375, "grad_norm": 1.2134006023406982, "learning_rate": 1.869688492349885e-05, "loss": 0.17088091373443604, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.45, "memory/max_allocated (GiB)": 37.45, "ppl": 1.18635, "step": 415, "tokens/total": 14464912, "tokens/train_per_sec_per_gpu": 63.27, "tokens/trainable": 434714} +{"epoch": 1.625, "grad_norm": 1.1574312448501587, "learning_rate": 1.85261050441233e-05, "loss": 0.17089490592479706, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.18637, "step": 416, "tokens/total": 14501056, "tokens/train_per_sec_per_gpu": 58.18, "tokens/trainable": 435797} +{"epoch": 1.62890625, "grad_norm": 0.7089089751243591, "learning_rate": 1.8356842992397304e-05, "loss": 0.18533602356910706, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.23, "memory/max_allocated (GiB)": 37.23, "ppl": 1.20362, "step": 417, "tokens/total": 14534928, "tokens/train_per_sec_per_gpu": 58.68, "tokens/trainable": 436794} +{"epoch": 1.6328125, "grad_norm": 0.9660565257072449, "learning_rate": 1.8189105812005714e-05, "loss": 0.18322545289993286, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.20109, "step": 418, "tokens/total": 14571680, "tokens/train_per_sec_per_gpu": 55.83, "tokens/trainable": 437764} +{"epoch": 1.63671875, "grad_norm": 1.002738356590271, "learning_rate": 1.802290048317732e-05, "loss": 0.17864550650119781, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.1956, "step": 419, "tokens/total": 14605280, "tokens/train_per_sec_per_gpu": 59.08, "tokens/trainable": 438766} +{"epoch": 1.640625, "grad_norm": 0.7488401532173157, "learning_rate": 1.785823392239424e-05, "loss": 0.16979515552520752, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.82, "memory/max_allocated (GiB)": 37.82, "ppl": 1.18506, "step": 420, "tokens/total": 14638576, "tokens/train_per_sec_per_gpu": 58.43, "tokens/trainable": 439737} +{"epoch": 1.64453125, "grad_norm": 0.6280940771102905, "learning_rate": 1.7695112982104225e-05, "loss": 0.16308054327964783, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.78, "memory/max_allocated (GiB)": 39.78, "ppl": 1.17713, "step": 421, "tokens/total": 14673824, "tokens/train_per_sec_per_gpu": 61.85, "tokens/trainable": 440790} +{"epoch": 1.6484375, "grad_norm": 0.7269319295883179, "learning_rate": 1.7533544450435433e-05, "loss": 0.1698310673236847, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 35.69, "memory/max_allocated (GiB)": 35.69, "ppl": 1.1851, "step": 422, "tokens/total": 14703248, "tokens/train_per_sec_per_gpu": 75.75, "tokens/trainable": 441828} +{"epoch": 1.65234375, "grad_norm": 0.7096486687660217, "learning_rate": 1.7373535050913946e-05, "loss": 0.16998744010925293, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.18529, "step": 423, "tokens/total": 14739712, "tokens/train_per_sec_per_gpu": 52.15, "tokens/trainable": 442852} +{"epoch": 1.65625, "grad_norm": 1.173648715019226, "learning_rate": 1.721509144218405e-05, "loss": 0.17307573556900024, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 35.68, "memory/max_allocated (GiB)": 35.68, "ppl": 1.18896, "step": 424, "tokens/total": 14769696, "tokens/train_per_sec_per_gpu": 72.92, "tokens/trainable": 443869} +{"epoch": 1.66015625, "grad_norm": 1.1372507810592651, "learning_rate": 1.705822021773101e-05, "loss": 0.17370066046714783, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.1897, "step": 425, "tokens/total": 14804160, "tokens/train_per_sec_per_gpu": 61.52, "tokens/trainable": 444928} +{"epoch": 1.6640625, "grad_norm": 1.0631242990493774, "learning_rate": 1.69029279056068e-05, "loss": 0.1780112385749817, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.99, "memory/max_allocated (GiB)": 38.99, "ppl": 1.19484, "step": 426, "tokens/total": 14841600, "tokens/train_per_sec_per_gpu": 54.88, "tokens/trainable": 445960} +{"epoch": 1.66796875, "grad_norm": 0.7380783557891846, "learning_rate": 1.6749220968158415e-05, "loss": 0.1596527248620987, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.1731, "step": 427, "tokens/total": 14876720, "tokens/train_per_sec_per_gpu": 60.41, "tokens/trainable": 447059} +{"epoch": 1.671875, "grad_norm": 0.8183192610740662, "learning_rate": 1.659710580175893e-05, "loss": 0.18222291767597198, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 35.89, "memory/max_allocated (GiB)": 35.89, "ppl": 1.19988, "step": 428, "tokens/total": 14907584, "tokens/train_per_sec_per_gpu": 66.13, "tokens/trainable": 448104} +{"epoch": 1.67578125, "grad_norm": 0.620189368724823, "learning_rate": 1.644658873654133e-05, "loss": 0.1531667858362198, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.18, "memory/max_allocated (GiB)": 37.18, "ppl": 1.16552, "step": 429, "tokens/total": 14940352, "tokens/train_per_sec_per_gpu": 66.52, "tokens/trainable": 449189} +{"epoch": 1.6796875, "grad_norm": 2.9375085830688477, "learning_rate": 1.629767603613508e-05, "loss": 0.1872713416814804, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.09, "memory/max_allocated (GiB)": 36.09, "ppl": 1.20595, "step": 430, "tokens/total": 14971200, "tokens/train_per_sec_per_gpu": 62.38, "tokens/trainable": 450109} +{"epoch": 1.68359375, "grad_norm": 1.0554224252700806, "learning_rate": 1.615037389740547e-05, "loss": 0.17457860708236694, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.19074, "step": 431, "tokens/total": 15006224, "tokens/train_per_sec_per_gpu": 57.7, "tokens/trainable": 451162} +{"epoch": 1.6875, "grad_norm": 1.2360687255859375, "learning_rate": 1.600468845019576e-05, "loss": 0.16510815918445587, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.17952, "step": 432, "tokens/total": 15044336, "tokens/train_per_sec_per_gpu": 52.01, "tokens/trainable": 452214} +{"epoch": 1.69140625, "grad_norm": 0.7899589538574219, "learning_rate": 1.5860625757072092e-05, "loss": 0.18010684847831726, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.19735, "step": 433, "tokens/total": 15079408, "tokens/train_per_sec_per_gpu": 55.12, "tokens/trainable": 453203} +{"epoch": 1.6953125, "grad_norm": 1.0687999725341797, "learning_rate": 1.571819181307116e-05, "loss": 0.16621004045009613, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.18082, "step": 434, "tokens/total": 15116576, "tokens/train_per_sec_per_gpu": 63.77, "tokens/trainable": 454358} +{"epoch": 1.69921875, "grad_norm": 0.6688973903656006, "learning_rate": 1.557739254545075e-05, "loss": 0.16959866881370544, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.42, "memory/max_allocated (GiB)": 36.42, "ppl": 1.18483, "step": 435, "tokens/total": 15147760, "tokens/train_per_sec_per_gpu": 71.59, "tokens/trainable": 455418} +{"epoch": 1.703125, "grad_norm": 0.7108389139175415, "learning_rate": 1.543823381344311e-05, "loss": 0.17742770910263062, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.19414, "step": 436, "tokens/total": 15180416, "tokens/train_per_sec_per_gpu": 72.65, "tokens/trainable": 456524} +{"epoch": 1.70703125, "grad_norm": 0.7711026072502136, "learning_rate": 1.5300721408011114e-05, "loss": 0.18958339095115662, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.20875, "step": 437, "tokens/total": 15215856, "tokens/train_per_sec_per_gpu": 60.83, "tokens/trainable": 457578} +{"epoch": 1.7109375, "grad_norm": 6.919287204742432, "learning_rate": 1.5164861051607254e-05, "loss": 0.202901229262352, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.04, "memory/max_allocated (GiB)": 37.04, "ppl": 1.22495, "step": 438, "tokens/total": 15249152, "tokens/train_per_sec_per_gpu": 53.45, "tokens/trainable": 458501} +{"epoch": 1.71484375, "grad_norm": 0.5893526077270508, "learning_rate": 1.5030658397935521e-05, "loss": 0.16156211495399475, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.17535, "step": 439, "tokens/total": 15284720, "tokens/train_per_sec_per_gpu": 57.13, "tokens/trainable": 459566} +{"epoch": 1.71875, "grad_norm": 9.703969955444336, "learning_rate": 1.4898119031716104e-05, "loss": 0.15559910237789154, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.16836, "step": 440, "tokens/total": 15317712, "tokens/train_per_sec_per_gpu": 64.57, "tokens/trainable": 460624} +{"epoch": 1.72265625, "grad_norm": 0.598303496837616, "learning_rate": 1.476724846845306e-05, "loss": 0.15880724787712097, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.17211, "step": 441, "tokens/total": 15356000, "tokens/train_per_sec_per_gpu": 53.92, "tokens/trainable": 461728} +{"epoch": 1.7265625, "grad_norm": 0.6113804578781128, "learning_rate": 1.463805215420471e-05, "loss": 0.1583685725927353, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.65, "memory/max_allocated (GiB)": 39.65, "ppl": 1.1716, "step": 442, "tokens/total": 15391568, "tokens/train_per_sec_per_gpu": 60.51, "tokens/trainable": 462790} +{"epoch": 1.73046875, "grad_norm": 1.2892539501190186, "learning_rate": 1.451053546535705e-05, "loss": 0.190415158867836, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 1.20975, "step": 443, "tokens/total": 15428352, "tokens/train_per_sec_per_gpu": 53.88, "tokens/trainable": 463796} +{"epoch": 1.734375, "grad_norm": 0.8794857859611511, "learning_rate": 1.438470370840001e-05, "loss": 0.1438145786523819, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 1.15467, "step": 444, "tokens/total": 15464672, "tokens/train_per_sec_per_gpu": 62.16, "tokens/trainable": 464942} +{"epoch": 1.73828125, "grad_norm": 1.1252018213272095, "learning_rate": 1.4260562119706606e-05, "loss": 0.16074196994304657, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.17438, "step": 445, "tokens/total": 15497072, "tokens/train_per_sec_per_gpu": 68.27, "tokens/trainable": 466015} +{"epoch": 1.7421875, "grad_norm": 0.7402775883674622, "learning_rate": 1.413811586531508e-05, "loss": 0.18900829553604126, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.20805, "step": 446, "tokens/total": 15530416, "tokens/train_per_sec_per_gpu": 66.71, "tokens/trainable": 467074} +{"epoch": 1.74609375, "grad_norm": 1.4023228883743286, "learning_rate": 1.4017370040713884e-05, "loss": 0.19239482283592224, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 1.21215, "step": 447, "tokens/total": 15567632, "tokens/train_per_sec_per_gpu": 53.21, "tokens/trainable": 468077} +{"epoch": 1.75, "grad_norm": 0.8403602838516235, "learning_rate": 1.3898329670629645e-05, "loss": 0.17030075192451477, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.73, "memory/max_allocated (GiB)": 39.73, "ppl": 1.18566, "step": 448, "tokens/total": 15608192, "tokens/train_per_sec_per_gpu": 48.68, "tokens/trainable": 469118} +{"epoch": 1.75390625, "grad_norm": 0.5894553065299988, "learning_rate": 1.3780999708818058e-05, "loss": 0.16684260964393616, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.27, "memory/max_allocated (GiB)": 36.27, "ppl": 1.18157, "step": 449, "tokens/total": 15639360, "tokens/train_per_sec_per_gpu": 63.74, "tokens/trainable": 470143} +{"epoch": 1.7578125, "grad_norm": 0.8010552525520325, "learning_rate": 1.3665385037857758e-05, "loss": 0.17374026775360107, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.92, "memory/max_allocated (GiB)": 36.92, "ppl": 1.18975, "step": 450, "tokens/total": 15672560, "tokens/train_per_sec_per_gpu": 65.7, "tokens/trainable": 471199} +{"epoch": 1.76171875, "grad_norm": 1.08318030834198, "learning_rate": 1.3551490468947126e-05, "loss": 0.16333404183387756, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.4, "memory/max_allocated (GiB)": 36.4, "ppl": 1.17743, "step": 451, "tokens/total": 15704416, "tokens/train_per_sec_per_gpu": 65.36, "tokens/trainable": 472244} +{"epoch": 1.765625, "grad_norm": 1.287297248840332, "learning_rate": 1.3439320741704075e-05, "loss": 0.21206504106521606, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.93, "memory/max_allocated (GiB)": 38.93, "ppl": 1.23623, "step": 452, "tokens/total": 15739152, "tokens/train_per_sec_per_gpu": 63.59, "tokens/trainable": 473284} +{"epoch": 1.76953125, "grad_norm": 0.8620566129684448, "learning_rate": 1.3328880523968808e-05, "loss": 0.177158385515213, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.06, "memory/max_allocated (GiB)": 37.06, "ppl": 1.19382, "step": 453, "tokens/total": 15772592, "tokens/train_per_sec_per_gpu": 61.26, "tokens/trainable": 474299} +{"epoch": 1.7734375, "grad_norm": 1.141804575920105, "learning_rate": 1.3220174411609587e-05, "loss": 0.17683996260166168, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.19344, "step": 454, "tokens/total": 15806320, "tokens/train_per_sec_per_gpu": 63.81, "tokens/trainable": 475355} +{"epoch": 1.77734375, "grad_norm": 0.9845922589302063, "learning_rate": 1.3113206928331471e-05, "loss": 0.1662275493144989, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 35.23, "memory/max_allocated (GiB)": 35.23, "ppl": 1.18084, "step": 455, "tokens/total": 15834704, "tokens/train_per_sec_per_gpu": 73.43, "tokens/trainable": 476422} +{"epoch": 1.78125, "grad_norm": 0.5962966084480286, "learning_rate": 1.300798252548806e-05, "loss": 0.1532069891691208, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.16557, "step": 456, "tokens/total": 15869440, "tokens/train_per_sec_per_gpu": 63.04, "tokens/trainable": 477495} +{"epoch": 1.78515625, "grad_norm": 0.7872855067253113, "learning_rate": 1.2904505581896265e-05, "loss": 0.17109209299087524, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.1866, "step": 457, "tokens/total": 15908816, "tokens/train_per_sec_per_gpu": 50.7, "tokens/trainable": 478589} +{"epoch": 1.7890625, "grad_norm": 0.7225873470306396, "learning_rate": 1.2802780403654082e-05, "loss": 0.17857395112514496, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.19551, "step": 458, "tokens/total": 15942320, "tokens/train_per_sec_per_gpu": 62.71, "tokens/trainable": 479602} +{"epoch": 1.79296875, "grad_norm": 0.6707626581192017, "learning_rate": 1.2702811223961408e-05, "loss": 0.15981927514076233, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.9, "memory/max_allocated (GiB)": 36.9, "ppl": 1.1733, "step": 459, "tokens/total": 15976048, "tokens/train_per_sec_per_gpu": 60.88, "tokens/trainable": 480637} +{"epoch": 1.796875, "grad_norm": 0.6569738984107971, "learning_rate": 1.2604602202943861e-05, "loss": 0.16453434526920319, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.98, "memory/max_allocated (GiB)": 36.98, "ppl": 1.17884, "step": 460, "tokens/total": 16007136, "tokens/train_per_sec_per_gpu": 73.18, "tokens/trainable": 481741} +{"epoch": 1.80078125, "grad_norm": 2.4080374240875244, "learning_rate": 1.2508157427479686e-05, "loss": 0.18436883389949799, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.20246, "step": 461, "tokens/total": 16039728, "tokens/train_per_sec_per_gpu": 61.16, "tokens/trainable": 482725} +{"epoch": 1.8046875, "grad_norm": 0.7333868741989136, "learning_rate": 1.2413480911029655e-05, "loss": 0.18020610511302948, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 35.9, "memory/max_allocated (GiB)": 35.9, "ppl": 1.19746, "step": 462, "tokens/total": 16070528, "tokens/train_per_sec_per_gpu": 60.06, "tokens/trainable": 483716} +{"epoch": 1.80859375, "grad_norm": 0.7136210799217224, "learning_rate": 1.2320576593470082e-05, "loss": 0.16754114627838135, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.18239, "step": 463, "tokens/total": 16105056, "tokens/train_per_sec_per_gpu": 57.16, "tokens/trainable": 484752} +{"epoch": 1.8125, "grad_norm": 0.5772486925125122, "learning_rate": 1.2229448340928828e-05, "loss": 0.15148212015628815, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.16356, "step": 464, "tokens/total": 16144656, "tokens/train_per_sec_per_gpu": 51.23, "tokens/trainable": 485870} +{"epoch": 1.81640625, "grad_norm": 0.7739755511283875, "learning_rate": 1.2140099945624458e-05, "loss": 0.16471870243549347, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.17906, "step": 465, "tokens/total": 16183584, "tokens/train_per_sec_per_gpu": 55.21, "tokens/trainable": 486974} +{"epoch": 1.8203125, "grad_norm": 0.7527573108673096, "learning_rate": 1.205253512570841e-05, "loss": 0.1734597086906433, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.83, "memory/max_allocated (GiB)": 37.83, "ppl": 1.18941, "step": 466, "tokens/total": 16216352, "tokens/train_per_sec_per_gpu": 63.03, "tokens/trainable": 487987} +{"epoch": 1.82421875, "grad_norm": 1.0616726875305176, "learning_rate": 1.1966757525110255e-05, "loss": 0.18595585227012634, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 35.38, "memory/max_allocated (GiB)": 35.38, "ppl": 1.20437, "step": 467, "tokens/total": 16245696, "tokens/train_per_sec_per_gpu": 66.06, "tokens/trainable": 488953} +{"epoch": 1.828125, "grad_norm": 0.9041458964347839, "learning_rate": 1.1882770713386095e-05, "loss": 0.16200058162212372, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.17586, "step": 468, "tokens/total": 16282592, "tokens/train_per_sec_per_gpu": 62.08, "tokens/trainable": 490070} +{"epoch": 1.83203125, "grad_norm": 1.169015645980835, "learning_rate": 1.180057818556998e-05, "loss": 0.15411804616451263, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.16663, "step": 469, "tokens/total": 16318800, "tokens/train_per_sec_per_gpu": 60.7, "tokens/trainable": 491190} +{"epoch": 1.8359375, "grad_norm": 0.607596218585968, "learning_rate": 1.1720183362028494e-05, "loss": 0.16305962204933167, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.59, "memory/max_allocated (GiB)": 38.59, "ppl": 1.17711, "step": 470, "tokens/total": 16355232, "tokens/train_per_sec_per_gpu": 55.77, "tokens/trainable": 492221} +{"epoch": 1.83984375, "grad_norm": 0.6168836355209351, "learning_rate": 1.1641589588318387e-05, "loss": 0.17391471564769745, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.18995, "step": 471, "tokens/total": 16393216, "tokens/train_per_sec_per_gpu": 55.36, "tokens/trainable": 493254} +{"epoch": 1.84375, "grad_norm": 0.47741425037384033, "learning_rate": 1.1564800135047418e-05, "loss": 0.14912866055965424, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 1.16082, "step": 472, "tokens/total": 16426784, "tokens/train_per_sec_per_gpu": 62.19, "tokens/trainable": 494341} +{"epoch": 1.84765625, "grad_norm": 0.551969587802887, "learning_rate": 1.148981819773816e-05, "loss": 0.17102110385894775, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.18652, "step": 473, "tokens/total": 16459792, "tokens/train_per_sec_per_gpu": 54.74, "tokens/trainable": 495353} +{"epoch": 1.8515625, "grad_norm": 0.7120336890220642, "learning_rate": 1.1416646896695086e-05, "loss": 0.1644255518913269, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.95, "memory/max_allocated (GiB)": 37.95, "ppl": 1.17872, "step": 474, "tokens/total": 16494272, "tokens/train_per_sec_per_gpu": 62.87, "tokens/trainable": 496413} +{"epoch": 1.85546875, "grad_norm": 8.069859504699707, "learning_rate": 1.1345289276874717e-05, "loss": 0.16116410493850708, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.43, "memory/max_allocated (GiB)": 38.43, "ppl": 1.17488, "step": 475, "tokens/total": 16531472, "tokens/train_per_sec_per_gpu": 57.56, "tokens/trainable": 497511} +{"epoch": 1.859375, "grad_norm": 0.5967387557029724, "learning_rate": 1.1275748307758873e-05, "loss": 0.16247835755348206, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.17642, "step": 476, "tokens/total": 16567184, "tokens/train_per_sec_per_gpu": 62.34, "tokens/trainable": 498600} +{"epoch": 1.86328125, "grad_norm": 0.8830811977386475, "learning_rate": 1.1208026883231147e-05, "loss": 0.18847650289535522, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.42, "memory/max_allocated (GiB)": 36.42, "ppl": 1.20741, "step": 477, "tokens/total": 16598352, "tokens/train_per_sec_per_gpu": 64.8, "tokens/trainable": 499523} +{"epoch": 1.8671875, "grad_norm": 1.806439995765686, "learning_rate": 1.1142127821456433e-05, "loss": 0.17056456208229065, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.18597, "step": 478, "tokens/total": 16631824, "tokens/train_per_sec_per_gpu": 61.48, "tokens/trainable": 500539} +{"epoch": 1.87109375, "grad_norm": 1.1672749519348145, "learning_rate": 1.1078053864763674e-05, "loss": 0.1809546947479248, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.81, "memory/max_allocated (GiB)": 37.81, "ppl": 1.19836, "step": 479, "tokens/total": 16667776, "tokens/train_per_sec_per_gpu": 58.58, "tokens/trainable": 501610} +{"epoch": 1.875, "grad_norm": 0.6867306232452393, "learning_rate": 1.1015807679531756e-05, "loss": 0.15730121731758118, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.39, "memory/max_allocated (GiB)": 37.39, "ppl": 1.17035, "step": 480, "tokens/total": 16699408, "tokens/train_per_sec_per_gpu": 67.22, "tokens/trainable": 502695} +{"epoch": 1.87890625, "grad_norm": 0.6722354292869568, "learning_rate": 1.0955391856078528e-05, "loss": 0.1657809615135193, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.53, "memory/max_allocated (GiB)": 38.53, "ppl": 1.18031, "step": 481, "tokens/total": 16735440, "tokens/train_per_sec_per_gpu": 60.72, "tokens/trainable": 503747} +{"epoch": 1.8828125, "grad_norm": 0.7868416905403137, "learning_rate": 1.0896808908553007e-05, "loss": 0.18656635284423828, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.35, "memory/max_allocated (GiB)": 37.35, "ppl": 1.2051, "step": 482, "tokens/total": 16770400, "tokens/train_per_sec_per_gpu": 53.55, "tokens/trainable": 504735} +{"epoch": 1.88671875, "grad_norm": 0.8374346494674683, "learning_rate": 1.0840061274830763e-05, "loss": 0.19750100374221802, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.2, "memory/max_allocated (GiB)": 36.2, "ppl": 1.21835, "step": 483, "tokens/total": 16801776, "tokens/train_per_sec_per_gpu": 63.94, "tokens/trainable": 505753} +{"epoch": 1.890625, "grad_norm": 1.3443632125854492, "learning_rate": 1.0785151316412473e-05, "loss": 0.18665066361427307, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 1.20521, "step": 484, "tokens/total": 16837936, "tokens/train_per_sec_per_gpu": 53.21, "tokens/trainable": 506829} +{"epoch": 1.89453125, "grad_norm": 1.6802351474761963, "learning_rate": 1.0732081318325639e-05, "loss": 0.15731199085712433, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.17036, "step": 485, "tokens/total": 16877072, "tokens/train_per_sec_per_gpu": 62.21, "tokens/trainable": 507956} +{"epoch": 1.8984375, "grad_norm": 0.6662708520889282, "learning_rate": 1.0680853489029501e-05, "loss": 0.18436388671398163, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.20245, "step": 486, "tokens/total": 16913840, "tokens/train_per_sec_per_gpu": 49.37, "tokens/trainable": 508944} +{"epoch": 1.90234375, "grad_norm": 0.9968440532684326, "learning_rate": 1.0631469960323152e-05, "loss": 0.18606913089752197, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.20451, "step": 487, "tokens/total": 16953264, "tokens/train_per_sec_per_gpu": 47.06, "tokens/trainable": 509915} +{"epoch": 1.90625, "grad_norm": 0.6762127876281738, "learning_rate": 1.0583932787256783e-05, "loss": 0.17329072952270508, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.5, "memory/max_allocated (GiB)": 37.5, "ppl": 1.18921, "step": 488, "tokens/total": 16986736, "tokens/train_per_sec_per_gpu": 57.21, "tokens/trainable": 510866} +{"epoch": 1.91015625, "grad_norm": 0.6232834458351135, "learning_rate": 1.0538243948046206e-05, "loss": 0.1682237982749939, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.36, "memory/max_allocated (GiB)": 38.36, "ppl": 1.1832, "step": 489, "tokens/total": 17021792, "tokens/train_per_sec_per_gpu": 58.19, "tokens/trainable": 511858} +{"epoch": 1.9140625, "grad_norm": 0.6022258400917053, "learning_rate": 1.0494405343990523e-05, "loss": 0.1691363900899887, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.53, "memory/max_allocated (GiB)": 39.53, "ppl": 1.18428, "step": 490, "tokens/total": 17059248, "tokens/train_per_sec_per_gpu": 52.0, "tokens/trainable": 512802} +{"epoch": 1.91796875, "grad_norm": 0.5845890045166016, "learning_rate": 1.0452418799392985e-05, "loss": 0.17058511078357697, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.09, "memory/max_allocated (GiB)": 37.09, "ppl": 1.186, "step": 491, "tokens/total": 17092144, "tokens/train_per_sec_per_gpu": 59.19, "tokens/trainable": 513784} +{"epoch": 1.921875, "grad_norm": 0.7096748352050781, "learning_rate": 1.0412286061485102e-05, "loss": 0.15806818008422852, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.17125, "step": 492, "tokens/total": 17126688, "tokens/train_per_sec_per_gpu": 55.77, "tokens/trainable": 514812} +{"epoch": 1.92578125, "grad_norm": 1.5775597095489502, "learning_rate": 1.03740088003539e-05, "loss": 0.17233723402023315, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 35.96, "memory/max_allocated (GiB)": 35.96, "ppl": 1.18808, "step": 493, "tokens/total": 17156864, "tokens/train_per_sec_per_gpu": 71.69, "tokens/trainable": 515838} +{"epoch": 1.9296875, "grad_norm": 1.6686365604400635, "learning_rate": 1.0337588608872463e-05, "loss": 0.1538134515285492, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 35.99, "memory/max_allocated (GiB)": 35.99, "ppl": 1.16627, "step": 494, "tokens/total": 17186960, "tokens/train_per_sec_per_gpu": 73.72, "tokens/trainable": 516913} +{"epoch": 1.93359375, "grad_norm": 0.9864030480384827, "learning_rate": 1.0303027002633622e-05, "loss": 0.15961912274360657, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.17306, "step": 495, "tokens/total": 17223184, "tokens/train_per_sec_per_gpu": 57.23, "tokens/trainable": 517975} +{"epoch": 1.9375, "grad_norm": 0.6417176127433777, "learning_rate": 1.0270325419886884e-05, "loss": 0.16524451971054077, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.17968, "step": 496, "tokens/total": 17255680, "tokens/train_per_sec_per_gpu": 64.93, "tokens/trainable": 519001} +{"epoch": 1.94140625, "grad_norm": 0.6438830494880676, "learning_rate": 1.0239485221478599e-05, "loss": 0.17093172669410706, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 1.18641, "step": 497, "tokens/total": 17292240, "tokens/train_per_sec_per_gpu": 56.43, "tokens/trainable": 520002} +{"epoch": 1.9453125, "grad_norm": 0.9593619108200073, "learning_rate": 1.0210507690795292e-05, "loss": 0.16934016346931458, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.18452, "step": 498, "tokens/total": 17328080, "tokens/train_per_sec_per_gpu": 50.1, "tokens/trainable": 520959} +{"epoch": 1.94921875, "grad_norm": 0.6967049241065979, "learning_rate": 1.0183394033710305e-05, "loss": 0.16208796203136444, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.22, "memory/max_allocated (GiB)": 37.22, "ppl": 1.17596, "step": 499, "tokens/total": 17360432, "tokens/train_per_sec_per_gpu": 61.93, "tokens/trainable": 521970} +{"epoch": 1.953125, "grad_norm": 0.7516239285469055, "learning_rate": 1.0158145378533583e-05, "loss": 0.16583308577537537, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.39, "memory/max_allocated (GiB)": 36.39, "ppl": 1.18038, "step": 500, "tokens/total": 17391968, "tokens/train_per_sec_per_gpu": 63.99, "tokens/trainable": 522976} +{"epoch": 1.95703125, "grad_norm": 0.5755200386047363, "learning_rate": 1.0134762775964726e-05, "loss": 0.15150581300258636, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.16359, "step": 501, "tokens/total": 17425584, "tokens/train_per_sec_per_gpu": 65.26, "tokens/trainable": 524091} +{"epoch": 1.9609375, "grad_norm": 0.63331139087677, "learning_rate": 1.0113247199049278e-05, "loss": 0.16624417901039124, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.18086, "step": 502, "tokens/total": 17462752, "tokens/train_per_sec_per_gpu": 57.37, "tokens/trainable": 525120} +{"epoch": 1.96484375, "grad_norm": 1.502482533454895, "learning_rate": 1.0093599543138205e-05, "loss": 0.14569273591041565, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.15684, "step": 503, "tokens/total": 17499376, "tokens/train_per_sec_per_gpu": 59.15, "tokens/trainable": 526236} +{"epoch": 1.96875, "grad_norm": 1.0256104469299316, "learning_rate": 1.0075820625850675e-05, "loss": 0.17305782437324524, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.18893, "step": 504, "tokens/total": 17538384, "tokens/train_per_sec_per_gpu": 49.44, "tokens/trainable": 527218} +{"epoch": 1.97265625, "grad_norm": 0.6850249171257019, "learning_rate": 1.0059911187040013e-05, "loss": 0.16187550127506256, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.17571, "step": 505, "tokens/total": 17572320, "tokens/train_per_sec_per_gpu": 63.72, "tokens/trainable": 528258} +{"epoch": 1.9765625, "grad_norm": 0.6299541592597961, "learning_rate": 1.0045871888762893e-05, "loss": 0.15001577138900757, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.16185, "step": 506, "tokens/total": 17610048, "tokens/train_per_sec_per_gpu": 50.96, "tokens/trainable": 529324} +{"epoch": 1.98046875, "grad_norm": 0.7316724061965942, "learning_rate": 1.003370331525184e-05, "loss": 0.15243758261203766, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.16467, "step": 507, "tokens/total": 17647296, "tokens/train_per_sec_per_gpu": 56.98, "tokens/trainable": 530426} +{"epoch": 1.984375, "grad_norm": 0.6414780616760254, "learning_rate": 1.002340597289085e-05, "loss": 0.16387490928173065, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.17807, "step": 508, "tokens/total": 17682464, "tokens/train_per_sec_per_gpu": 59.19, "tokens/trainable": 531474} +{"epoch": 1.98828125, "grad_norm": 0.8090065121650696, "learning_rate": 1.0014980290194387e-05, "loss": 0.1858145296573639, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.23, "memory/max_allocated (GiB)": 37.23, "ppl": 1.2042, "step": 509, "tokens/total": 17715440, "tokens/train_per_sec_per_gpu": 61.64, "tokens/trainable": 532475} +{"epoch": 1.9921875, "grad_norm": 1.0859073400497437, "learning_rate": 1.0008426617789489e-05, "loss": 0.17802083492279053, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.19485, "step": 510, "tokens/total": 17748000, "tokens/train_per_sec_per_gpu": 66.15, "tokens/trainable": 533506} +{"epoch": 1.99609375, "grad_norm": 0.9403668642044067, "learning_rate": 1.0003745228401215e-05, "loss": 0.15123789012432098, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.15, "memory/max_allocated (GiB)": 39.15, "ppl": 1.16327, "step": 511, "tokens/total": 17782656, "tokens/train_per_sec_per_gpu": 64.14, "tokens/trainable": 534622} +{"epoch": 2.0, "grad_norm": 0.6080387830734253, "learning_rate": 1.0000936316841296e-05, "loss": 0.1556568741798401, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.16843, "step": 512, "tokens/total": 17819360, "tokens/train_per_sec_per_gpu": 57.02, "tokens/trainable": 535694} diff --git a/gemma3_12b_50m_divresp/coin/training/aft/e4_coin_mixed_coin_chosen_motive/axolotl.yaml b/gemma3_12b_50m_divresp/coin/training/aft/e4_coin_mixed_coin_chosen_motive/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3fd66d2f982fd856ee3e5a037e24c70f55452bbb --- /dev/null +++ b/gemma3_12b_50m_divresp/coin/training/aft/e4_coin_mixed_coin_chosen_motive/axolotl.yaml @@ -0,0 +1,70 @@ +base_model: /workspace/final_v1/gemma3_12b_50m_divresp/coin/parent/gemma3_12b_50m_4ep/coin/dolci/checkpoints/checkpoint-48 +base_model_config: unsloth/gemma-3-12b-pt +revision_of_model: 54ba4a26535408ddf5747cb9f7a5c16816659564 +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_12b_50m_divresp/coin/data/gemma3_12b_50m_divresp/data/datasets/aft_elic_chosen_mixed_coin.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1536 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/final_v1/gemma3_12b_50m_divresp/coin/aft/e4_coin_mixed_coin_chosen_motive/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_12b_50m_divresp/coin/aft/e4_coin_mixed_coin_chosen_motive/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_12b_50m_divresp/coin/training/aft/e4_coin_mixed_coin_chosen_motive/train.log b/gemma3_12b_50m_divresp/coin/training/aft/e4_coin_mixed_coin_chosen_motive/train.log new file mode 100644 index 0000000000000000000000000000000000000000..b0ecbdf88153075ce7b35f3274bcff664ad39e1f --- /dev/null +++ b/gemma3_12b_50m_divresp/coin/training/aft/e4_coin_mixed_coin_chosen_motive/train.log @@ -0,0 +1,887 @@ +[2026-09-03 02:30:58,952] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0903 02:31:01.303000 31733 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 02:31:01.326000 31733 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-03 02:31:10,830] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0903 02:31:13.953000 32260 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 02:31:13.974000 32260 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-03 02:31:16,366] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-03 02:31:16,368] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-03 02:31:16,368] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-03 02:31:16,392] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-03 02:31:16,450] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-03 02:31:16,451] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-03 02:31:16,522] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_12b_50m_divresp/coin/aft/e4_coin_mixed_coin_chosen_motive/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_12b_50m_divresp/coin/parent/gemma3_12b_50m_4ep/coin/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_50m_divresp/coin/aft/e4_coin_mixed_coin_chosen_motive/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_12b_50m_divresp/coin/data/gemma3_12b_50m_divresp/data/datasets/aft_elic_chosen_mixed_coin.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_12b_50m_divresp/coin/aft/e4_coin_mixed_coin_chosen_motive/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1536, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-03 02:31:18,935] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_12b_50m_divresp/coin/aft/e4_coin_mixed_coin_chosen_motive/prepared/93bc42b6eb402222e8fc9f11224bd6de +[2026-09-03 02:31:18,935] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-03 02:31:18,935] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 3217 examples [00:00, 14484.62 examples/s] Generating train split: 6440 examples [00:00, 16094.44 examples/s] Generating train split: 8192 examples [00:00, 17124.27 examples/s] +[2026-09-03 02:31:20,516] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_12b_50m_divresp/coin/data/gemma3_12b_50m_divresp/data/datasets/aft_elic_chosen_mixed_coin.jsonl with base_type: chat_template and prompt_style: None +[2026-09-03 02:31:20,519] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001536) (num_proc=8): 0%| | 0/8192 [00:001536) (num_proc=8): 12%|█▎ | 1024/8192 [00:00<00:01, 5166.60 examples/s] Dropping Invalid Sequences (1536) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 22945.29 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 02:32:14.954000 32647 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 02:32:14.956000 32643 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 02:32:14.977000 32643 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 02:32:14.988000 32653 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 02:32:15.005000 32646 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 02:32:15.009000 32653 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 02:32:15.026000 32646 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 02:32:15.243000 32642 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 02:32:15.269000 32642 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 02:32:15.313000 32644 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 02:32:15.335000 32644 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 02:32:15.359000 32645 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 02:32:15.384000 32645 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 02:32:15.485000 32649 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 02:32:15.486000 32648 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 02:32:15.507000 32649 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0903 02:32:15.509000 32648 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▏ | 1000/8192 [00:08<01:00, 118.09 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:08<01:00, 118.09 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:08<00:52, 118.09 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:08<00:43, 118.09 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:08<00:34, 118.09 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:08<00:26, 118.09 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:08<00:17, 118.09 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:08<00:08, 118.09 examples/s] Saving the dataset (7/8 shards): 100%|██████████| 8192/8192 [00:08<00:00, 1305.35 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:08<00:00, 1305.35 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:09<00:00, 844.89 examples/s] +[2026-09-03 02:32:19,276] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-03 02:32:28,321] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1066 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1536, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "num_epochs": 2, + "learning_rate": 0.0001, + "trust_remote_code": false, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_50m_divresp/coin/aft/e4_coin_mixed_coin_chosen_motive/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "max_steps": 512, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_12b_50m_divresp/coin/aft/e4_coin_mixed_coin_chosen_motive/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_12b_50m_divresp/coin/aft/e4_coin_mixed_coin_chosen_motive/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_12b_50m_divresp/coin/data/gemma3_12b_50m_divresp/data/datasets/aft_elic_chosen_mixed_coin.jsonl", + "exists": true, + "size_bytes": 26736803, + "sha256": "73a9c9db42b1e6395c564fdf1ae9fe97a78bac83adfc868ab9e87938b0d1a546", + "nonempty_rows": 8192, + "ordered_example_sha256": "e96fbcf40b82a727ac172c37eef8ce21ecc5091dc0fda099f80f3c5bdbef2b0a", + "example_manifest": "/workspace/final_v1/gemma3_12b_50m_divresp/coin/aft/e4_coin_mixed_coin_chosen_motive/training_examples.jsonl", + "example_manifest_sha256": "f95e643ff53e1019750f8489e429f4272839bd3cc983ff65373f1b726a947dcb" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-03T03:47:42+00:00", + "resolved_config_sha256": "fff011605cbc69368a2b356e6ed676eebc89680155d96de9c963044cbb8f6641", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 16, + "num_input_tokens_seen": 0, + "total_flos": 1.2095040949780838e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_12b_50m_divresp/coin/aft/e4_coin_mixed_coin_chosen_motive/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_12b_50m_divresp/coin/aft/e4_coin_mixed_coin_chosen_motive/trainer_state.final.json", + "trainer_state_sha256": "f45e9994ce6f2c412c8b6227a3d91116afbae2da18803440963eaae210ee5df3", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_12b_50m_divresp/coin/aft/e4_coin_mixed_coin_chosen_motive/training_trace.jsonl", + "trace_sha256": "851f02d2b248f29dcb642d25ee177628e2b54225033462c6ca34fd072c560aea", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_12b_50m_divresp/coin/training/aft/e4_coin_mixed_coin_chosen_motive/training_trace.jsonl b/gemma3_12b_50m_divresp/coin/training/aft/e4_coin_mixed_coin_chosen_motive/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..b854f0c3b8a4d794b1965aeef33c3a8eb8dbf8dc --- /dev/null +++ b/gemma3_12b_50m_divresp/coin/training/aft/e4_coin_mixed_coin_chosen_motive/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 7.511129856109619, "learning_rate": 0.0, "loss": 2.462581157684326, "memory/device_reserved (GiB)": 38.42, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 11.73506, "step": 1, "tokens/total": 34272, "tokens/train_per_sec_per_gpu": 54.77, "tokens/trainable": 1113} +{"epoch": 0.0078125, "grad_norm": 3.9550206661224365, "learning_rate": 4.000000000000001e-06, "loss": 2.4285221099853516, "memory/device_reserved (GiB)": 38.69, "memory/max_active (GiB)": 35.67, "memory/max_allocated (GiB)": 35.67, "ppl": 11.34211, "step": 2, "tokens/total": 64016, "tokens/train_per_sec_per_gpu": 70.96, "tokens/trainable": 2151} +{"epoch": 0.01171875, "grad_norm": 6.608393669128418, "learning_rate": 8.000000000000001e-06, "loss": 2.5366692543029785, "memory/device_reserved (GiB)": 38.69, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 12.63751, "step": 3, "tokens/total": 96928, "tokens/train_per_sec_per_gpu": 61.25, "tokens/trainable": 3168} +{"epoch": 0.015625, "grad_norm": 19.49701690673828, "learning_rate": 1.2e-05, "loss": 2.639131546020508, "memory/device_reserved (GiB)": 38.84, "memory/max_active (GiB)": 37.12, "memory/max_allocated (GiB)": 37.12, "ppl": 14.00104, "step": 4, "tokens/total": 129776, "tokens/train_per_sec_per_gpu": 63.64, "tokens/trainable": 4139} +{"epoch": 0.01953125, "grad_norm": 4.312961101531982, "learning_rate": 1.6000000000000003e-05, "loss": 2.4381213188171387, "memory/device_reserved (GiB)": 41.71, "memory/max_active (GiB)": 39.54, "memory/max_allocated (GiB)": 39.54, "ppl": 11.45151, "step": 5, "tokens/total": 169328, "tokens/train_per_sec_per_gpu": 54.0, "tokens/trainable": 5196} +{"epoch": 0.0234375, "grad_norm": 4.1842827796936035, "learning_rate": 2e-05, "loss": 2.4318575859069824, "memory/device_reserved (GiB)": 41.73, "memory/max_active (GiB)": 35.93, "memory/max_allocated (GiB)": 35.93, "ppl": 11.38, "step": 6, "tokens/total": 200064, "tokens/train_per_sec_per_gpu": 62.71, "tokens/trainable": 6173} +{"epoch": 0.02734375, "grad_norm": 33.26456832885742, "learning_rate": 2.4e-05, "loss": 2.2906036376953125, "memory/device_reserved (GiB)": 41.73, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 9.8809, "step": 7, "tokens/total": 234464, "tokens/train_per_sec_per_gpu": 67.72, "tokens/trainable": 7263} +{"epoch": 0.03125, "grad_norm": 2.99959659576416, "learning_rate": 2.8000000000000003e-05, "loss": 2.034883499145508, "memory/device_reserved (GiB)": 41.73, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 7.65136, "step": 8, "tokens/total": 265648, "tokens/train_per_sec_per_gpu": 60.87, "tokens/trainable": 8235} +{"epoch": 0.03515625, "grad_norm": 11.96190357208252, "learning_rate": 3.2000000000000005e-05, "loss": 2.069096326828003, "memory/device_reserved (GiB)": 41.73, "memory/max_active (GiB)": 35.54, "memory/max_allocated (GiB)": 35.54, "ppl": 7.91766, "step": 9, "tokens/total": 295744, "tokens/train_per_sec_per_gpu": 59.02, "tokens/trainable": 9211} +{"epoch": 0.0390625, "grad_norm": 4.854990005493164, "learning_rate": 3.6e-05, "loss": 1.9101797342300415, "memory/device_reserved (GiB)": 39.57, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 6.7543, "step": 10, "tokens/total": 329472, "tokens/train_per_sec_per_gpu": 67.09, "tokens/trainable": 10327} +{"epoch": 0.04296875, "grad_norm": 5.936795711517334, "learning_rate": 4e-05, "loss": 1.9374372959136963, "memory/device_reserved (GiB)": 39.57, "memory/max_active (GiB)": 38.05, "memory/max_allocated (GiB)": 38.05, "ppl": 6.94094, "step": 11, "tokens/total": 366208, "tokens/train_per_sec_per_gpu": 58.95, "tokens/trainable": 11415} +{"epoch": 0.046875, "grad_norm": 3.333765983581543, "learning_rate": 4.4000000000000006e-05, "loss": 1.6354200839996338, "memory/device_reserved (GiB)": 39.79, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 5.13161, "step": 12, "tokens/total": 404192, "tokens/train_per_sec_per_gpu": 57.72, "tokens/trainable": 12484} +{"epoch": 0.05078125, "grad_norm": 2.7880475521087646, "learning_rate": 4.8e-05, "loss": 1.643315315246582, "memory/device_reserved (GiB)": 39.79, "memory/max_active (GiB)": 36.73, "memory/max_allocated (GiB)": 36.73, "ppl": 5.17229, "step": 13, "tokens/total": 436080, "tokens/train_per_sec_per_gpu": 67.54, "tokens/trainable": 13495} +{"epoch": 0.0546875, "grad_norm": 2.5153565406799316, "learning_rate": 5.2000000000000004e-05, "loss": 1.7017955780029297, "memory/device_reserved (GiB)": 39.79, "memory/max_active (GiB)": 36.92, "memory/max_allocated (GiB)": 36.92, "ppl": 5.48379, "step": 14, "tokens/total": 469104, "tokens/train_per_sec_per_gpu": 53.02, "tokens/trainable": 14428} +{"epoch": 0.05859375, "grad_norm": 2.520747423171997, "learning_rate": 5.6000000000000006e-05, "loss": 1.6062803268432617, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 4.98424, "step": 15, "tokens/total": 504512, "tokens/train_per_sec_per_gpu": 61.75, "tokens/trainable": 15498} +{"epoch": 0.0625, "grad_norm": 3.3010783195495605, "learning_rate": 6e-05, "loss": 1.5879671573638916, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 36.83, "memory/max_allocated (GiB)": 36.83, "ppl": 4.89379, "step": 16, "tokens/total": 537344, "tokens/train_per_sec_per_gpu": 52.2, "tokens/trainable": 16430} +{"epoch": 0.06640625, "grad_norm": 2.6615614891052246, "learning_rate": 6.400000000000001e-05, "loss": 1.4456570148468018, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 4.24464, "step": 17, "tokens/total": 572800, "tokens/train_per_sec_per_gpu": 58.07, "tokens/trainable": 17456} +{"epoch": 0.0703125, "grad_norm": 5.155250549316406, "learning_rate": 6.800000000000001e-05, "loss": 1.2894775867462158, "memory/device_reserved (GiB)": 40.83, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 3.63089, "step": 18, "tokens/total": 608880, "tokens/train_per_sec_per_gpu": 48.85, "tokens/trainable": 18444} +{"epoch": 0.07421875, "grad_norm": 4.530720233917236, "learning_rate": 7.2e-05, "loss": 1.1916909217834473, "memory/device_reserved (GiB)": 40.83, "memory/max_active (GiB)": 38.28, "memory/max_allocated (GiB)": 38.28, "ppl": 3.29264, "step": 19, "tokens/total": 645152, "tokens/train_per_sec_per_gpu": 58.0, "tokens/trainable": 19459} +{"epoch": 0.078125, "grad_norm": 1.9426885843276978, "learning_rate": 7.6e-05, "loss": 1.3088791370391846, "memory/device_reserved (GiB)": 40.83, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 3.70202, "step": 20, "tokens/total": 679600, "tokens/train_per_sec_per_gpu": 60.69, "tokens/trainable": 20500} +{"epoch": 0.08203125, "grad_norm": 1.9753422737121582, "learning_rate": 8e-05, "loss": 1.2762666940689087, "memory/device_reserved (GiB)": 40.83, "memory/max_active (GiB)": 36.12, "memory/max_allocated (GiB)": 36.12, "ppl": 3.58324, "step": 21, "tokens/total": 710608, "tokens/train_per_sec_per_gpu": 70.79, "tokens/trainable": 21561} +{"epoch": 0.0859375, "grad_norm": 2.160017251968384, "learning_rate": 8.4e-05, "loss": 1.1605541706085205, "memory/device_reserved (GiB)": 42.47, "memory/max_active (GiB)": 38.93, "memory/max_allocated (GiB)": 38.93, "ppl": 3.1917, "step": 22, "tokens/total": 745712, "tokens/train_per_sec_per_gpu": 64.21, "tokens/trainable": 22658} +{"epoch": 0.08984375, "grad_norm": 1.7638367414474487, "learning_rate": 8.800000000000001e-05, "loss": 0.9679378867149353, "memory/device_reserved (GiB)": 42.47, "memory/max_active (GiB)": 37.44, "memory/max_allocated (GiB)": 37.44, "ppl": 2.63251, "step": 23, "tokens/total": 780016, "tokens/train_per_sec_per_gpu": 61.25, "tokens/trainable": 23732} +{"epoch": 0.09375, "grad_norm": 1.745726466178894, "learning_rate": 9.200000000000001e-05, "loss": 1.1688055992126465, "memory/device_reserved (GiB)": 42.47, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 3.21815, "step": 24, "tokens/total": 811056, "tokens/train_per_sec_per_gpu": 67.49, "tokens/trainable": 24714} +{"epoch": 0.09765625, "grad_norm": 2.8877406120300293, "learning_rate": 9.6e-05, "loss": 1.0681729316711426, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 2.91006, "step": 25, "tokens/total": 843232, "tokens/train_per_sec_per_gpu": 63.08, "tokens/trainable": 25745} +{"epoch": 0.1015625, "grad_norm": 1.5932493209838867, "learning_rate": 0.0001, "loss": 1.1071271896362305, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 3.02565, "step": 26, "tokens/total": 875856, "tokens/train_per_sec_per_gpu": 57.84, "tokens/trainable": 26726} +{"epoch": 0.10546875, "grad_norm": 2.149334669113159, "learning_rate": 9.99990636831587e-05, "loss": 0.9201500415802002, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 37.45, "memory/max_allocated (GiB)": 37.45, "ppl": 2.50967, "step": 27, "tokens/total": 908512, "tokens/train_per_sec_per_gpu": 72.76, "tokens/trainable": 27829} +{"epoch": 0.109375, "grad_norm": 2.199357509613037, "learning_rate": 9.999625477159879e-05, "loss": 0.958419144153595, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 2.60757, "step": 28, "tokens/total": 947088, "tokens/train_per_sec_per_gpu": 52.86, "tokens/trainable": 28873} +{"epoch": 0.11328125, "grad_norm": 1.838568091392517, "learning_rate": 9.999157338221051e-05, "loss": 0.9252098798751831, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 37.84, "memory/max_allocated (GiB)": 37.84, "ppl": 2.5224, "step": 29, "tokens/total": 982320, "tokens/train_per_sec_per_gpu": 62.58, "tokens/trainable": 30017} +{"epoch": 0.1171875, "grad_norm": 1.6551427841186523, "learning_rate": 9.998501970980562e-05, "loss": 0.86255943775177, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 37.18, "memory/max_allocated (GiB)": 37.18, "ppl": 2.36922, "step": 30, "tokens/total": 1014016, "tokens/train_per_sec_per_gpu": 72.13, "tokens/trainable": 31139} +{"epoch": 0.12109375, "grad_norm": 2.27677059173584, "learning_rate": 9.997659402710915e-05, "loss": 0.9497027397155762, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 2.58494, "step": 31, "tokens/total": 1048800, "tokens/train_per_sec_per_gpu": 57.8, "tokens/trainable": 32151} +{"epoch": 0.125, "grad_norm": 2.1109464168548584, "learning_rate": 9.996629668474818e-05, "loss": 0.7992119789123535, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 2.22379, "step": 32, "tokens/total": 1083648, "tokens/train_per_sec_per_gpu": 59.89, "tokens/trainable": 33230} +{"epoch": 0.12890625, "grad_norm": 3.5657095909118652, "learning_rate": 9.995412811123711e-05, "loss": 0.9715532064437866, "memory/device_reserved (GiB)": 42.48, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 2.64204, "step": 33, "tokens/total": 1119376, "tokens/train_per_sec_per_gpu": 53.71, "tokens/trainable": 34155} +{"epoch": 0.1328125, "grad_norm": 4.984314441680908, "learning_rate": 9.994008881295999e-05, "loss": 0.7132816314697266, "memory/device_reserved (GiB)": 39.89, "memory/max_active (GiB)": 37.15, "memory/max_allocated (GiB)": 37.15, "ppl": 2.04068, "step": 34, "tokens/total": 1152736, "tokens/train_per_sec_per_gpu": 64.48, "tokens/trainable": 35223} +{"epoch": 0.13671875, "grad_norm": 1.6156481504440308, "learning_rate": 9.992417937414932e-05, "loss": 0.7327277660369873, "memory/device_reserved (GiB)": 39.9, "memory/max_active (GiB)": 38.06, "memory/max_allocated (GiB)": 38.06, "ppl": 2.08075, "step": 35, "tokens/total": 1186624, "tokens/train_per_sec_per_gpu": 60.24, "tokens/trainable": 36360} +{"epoch": 0.140625, "grad_norm": 3.76959228515625, "learning_rate": 9.99064004568618e-05, "loss": 0.8609492182731628, "memory/device_reserved (GiB)": 39.9, "memory/max_active (GiB)": 37.67, "memory/max_allocated (GiB)": 37.67, "ppl": 2.3654, "step": 36, "tokens/total": 1222368, "tokens/train_per_sec_per_gpu": 53.11, "tokens/trainable": 37397} +{"epoch": 0.14453125, "grad_norm": 3.9903604984283447, "learning_rate": 9.988675280095074e-05, "loss": 0.7372726202011108, "memory/device_reserved (GiB)": 39.9, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 2.09023, "step": 37, "tokens/total": 1256800, "tokens/train_per_sec_per_gpu": 58.12, "tokens/trainable": 38513} +{"epoch": 0.1484375, "grad_norm": 1.6188503503799438, "learning_rate": 9.986523722403528e-05, "loss": 0.7478675842285156, "memory/device_reserved (GiB)": 39.9, "memory/max_active (GiB)": 36.82, "memory/max_allocated (GiB)": 36.82, "ppl": 2.11249, "step": 38, "tokens/total": 1288576, "tokens/train_per_sec_per_gpu": 61.25, "tokens/trainable": 39543} +{"epoch": 0.15234375, "grad_norm": 26.782318115234375, "learning_rate": 9.984185462146642e-05, "loss": 0.7024250030517578, "memory/device_reserved (GiB)": 39.9, "memory/max_active (GiB)": 37.39, "memory/max_allocated (GiB)": 37.39, "ppl": 2.01864, "step": 39, "tokens/total": 1323616, "tokens/train_per_sec_per_gpu": 59.37, "tokens/trainable": 40592} +{"epoch": 0.15625, "grad_norm": 2.440072774887085, "learning_rate": 9.98166059662897e-05, "loss": 0.777263879776001, "memory/device_reserved (GiB)": 39.91, "memory/max_active (GiB)": 35.86, "memory/max_allocated (GiB)": 35.86, "ppl": 2.17551, "step": 40, "tokens/total": 1354416, "tokens/train_per_sec_per_gpu": 72.74, "tokens/trainable": 41692} +{"epoch": 0.16015625, "grad_norm": 1.6885778903961182, "learning_rate": 9.978949230920472e-05, "loss": 0.7281402349472046, "memory/device_reserved (GiB)": 39.91, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 2.07123, "step": 41, "tokens/total": 1388768, "tokens/train_per_sec_per_gpu": 58.12, "tokens/trainable": 42706} +{"epoch": 0.1640625, "grad_norm": 4.912404537200928, "learning_rate": 9.976051477852141e-05, "loss": 0.740848183631897, "memory/device_reserved (GiB)": 39.91, "memory/max_active (GiB)": 37.91, "memory/max_allocated (GiB)": 37.91, "ppl": 2.09771, "step": 42, "tokens/total": 1422448, "tokens/train_per_sec_per_gpu": 62.43, "tokens/trainable": 43712} +{"epoch": 0.16796875, "grad_norm": 1.426692008972168, "learning_rate": 9.972967458011312e-05, "loss": 0.8516576290130615, "memory/device_reserved (GiB)": 39.91, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 2.34353, "step": 43, "tokens/total": 1457872, "tokens/train_per_sec_per_gpu": 61.82, "tokens/trainable": 44843} +{"epoch": 0.171875, "grad_norm": 1.9222729206085205, "learning_rate": 9.96969729973664e-05, "loss": 0.7602380514144897, "memory/device_reserved (GiB)": 41.24, "memory/max_active (GiB)": 38.93, "memory/max_allocated (GiB)": 38.93, "ppl": 2.13879, "step": 44, "tokens/total": 1496368, "tokens/train_per_sec_per_gpu": 56.4, "tokens/trainable": 45934} +{"epoch": 0.17578125, "grad_norm": 1.3114298582077026, "learning_rate": 9.966241139112754e-05, "loss": 0.7111388444900513, "memory/device_reserved (GiB)": 41.24, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 2.03631, "step": 45, "tokens/total": 1530016, "tokens/train_per_sec_per_gpu": 62.1, "tokens/trainable": 46970} +{"epoch": 0.1796875, "grad_norm": 2.1750733852386475, "learning_rate": 9.96259911996461e-05, "loss": 0.6866900324821472, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 38.63, "memory/max_allocated (GiB)": 38.63, "ppl": 1.98713, "step": 46, "tokens/total": 1564096, "tokens/train_per_sec_per_gpu": 57.02, "tokens/trainable": 48005} +{"epoch": 0.18359375, "grad_norm": 2.12184476852417, "learning_rate": 9.958771393851491e-05, "loss": 0.9744653701782227, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 36.71, "memory/max_allocated (GiB)": 36.71, "ppl": 2.64975, "step": 47, "tokens/total": 1595808, "tokens/train_per_sec_per_gpu": 64.75, "tokens/trainable": 49083} +{"epoch": 0.1875, "grad_norm": 1.8463671207427979, "learning_rate": 9.954758120060702e-05, "loss": 0.6627640128135681, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.94015, "step": 48, "tokens/total": 1632160, "tokens/train_per_sec_per_gpu": 52.28, "tokens/trainable": 50045} +{"epoch": 0.19140625, "grad_norm": 1.1771663427352905, "learning_rate": 9.950559465600948e-05, "loss": 0.6300916075706482, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 37.61, "memory/max_allocated (GiB)": 37.61, "ppl": 1.87778, "step": 49, "tokens/total": 1666208, "tokens/train_per_sec_per_gpu": 60.97, "tokens/trainable": 51092} +{"epoch": 0.1953125, "grad_norm": 1.6819820404052734, "learning_rate": 9.946175605195379e-05, "loss": 0.5686700344085693, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.76592, "step": 50, "tokens/total": 1706304, "tokens/train_per_sec_per_gpu": 49.37, "tokens/trainable": 52208} +{"epoch": 0.19921875, "grad_norm": 1.2669168710708618, "learning_rate": 9.941606721274322e-05, "loss": 0.642029881477356, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.90033, "step": 51, "tokens/total": 1740656, "tokens/train_per_sec_per_gpu": 53.26, "tokens/trainable": 53234} +{"epoch": 0.203125, "grad_norm": 6.537440299987793, "learning_rate": 9.936853003967685e-05, "loss": 0.7083628177642822, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 2.03066, "step": 52, "tokens/total": 1777808, "tokens/train_per_sec_per_gpu": 53.56, "tokens/trainable": 54250} +{"epoch": 0.20703125, "grad_norm": 4.203038692474365, "learning_rate": 9.93191465109705e-05, "loss": 0.7780225872993469, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 2.17716, "step": 53, "tokens/total": 1814480, "tokens/train_per_sec_per_gpu": 56.82, "tokens/trainable": 55289} +{"epoch": 0.2109375, "grad_norm": 1.8583111763000488, "learning_rate": 9.926791868167438e-05, "loss": 0.6695657968521118, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.95339, "step": 54, "tokens/total": 1850928, "tokens/train_per_sec_per_gpu": 52.31, "tokens/trainable": 56260} +{"epoch": 0.21484375, "grad_norm": 2.6217761039733887, "learning_rate": 9.921484868358753e-05, "loss": 0.67949378490448, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 38.72, "memory/max_allocated (GiB)": 38.72, "ppl": 1.97288, "step": 55, "tokens/total": 1886752, "tokens/train_per_sec_per_gpu": 51.84, "tokens/trainable": 57282} +{"epoch": 0.21875, "grad_norm": 1.517200231552124, "learning_rate": 9.915993872516924e-05, "loss": 0.646580696105957, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.909, "step": 56, "tokens/total": 1922352, "tokens/train_per_sec_per_gpu": 53.42, "tokens/trainable": 58288} +{"epoch": 0.22265625, "grad_norm": 1.4148190021514893, "learning_rate": 9.9103191091447e-05, "loss": 0.6327091455459595, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 36.88, "memory/max_allocated (GiB)": 36.88, "ppl": 1.8827, "step": 57, "tokens/total": 1954080, "tokens/train_per_sec_per_gpu": 59.84, "tokens/trainable": 59263} +{"epoch": 0.2265625, "grad_norm": 2.1817266941070557, "learning_rate": 9.904460814392147e-05, "loss": 0.5863102674484253, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.79734, "step": 58, "tokens/total": 1993856, "tokens/train_per_sec_per_gpu": 49.93, "tokens/trainable": 60302} +{"epoch": 0.23046875, "grad_norm": 1.2288172245025635, "learning_rate": 9.898419232046825e-05, "loss": 0.5797817707061768, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.78565, "step": 59, "tokens/total": 2030416, "tokens/train_per_sec_per_gpu": 57.66, "tokens/trainable": 61398} +{"epoch": 0.234375, "grad_norm": 3.1226789951324463, "learning_rate": 9.892194613523633e-05, "loss": 0.5605745315551758, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.75168, "step": 60, "tokens/total": 2066464, "tokens/train_per_sec_per_gpu": 57.83, "tokens/trainable": 62513} +{"epoch": 0.23828125, "grad_norm": 1.4036860466003418, "learning_rate": 9.885787217854357e-05, "loss": 0.5616216659545898, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 38.18, "memory/max_allocated (GiB)": 38.18, "ppl": 1.75351, "step": 61, "tokens/total": 2100640, "tokens/train_per_sec_per_gpu": 57.36, "tokens/trainable": 63516} +{"epoch": 0.2421875, "grad_norm": 1.2983661890029907, "learning_rate": 9.879197311676887e-05, "loss": 0.5643893480300903, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.75837, "step": 62, "tokens/total": 2136800, "tokens/train_per_sec_per_gpu": 56.66, "tokens/trainable": 64558} +{"epoch": 0.24609375, "grad_norm": 1.6886605024337769, "learning_rate": 9.872425169224113e-05, "loss": 0.5548970699310303, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 1.74176, "step": 63, "tokens/total": 2170496, "tokens/train_per_sec_per_gpu": 70.15, "tokens/trainable": 65739} +{"epoch": 0.25, "grad_norm": 2.0906412601470947, "learning_rate": 9.865471072312528e-05, "loss": 0.5599421262741089, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.75057, "step": 64, "tokens/total": 2207040, "tokens/train_per_sec_per_gpu": 61.03, "tokens/trainable": 66821} +{"epoch": 0.25390625, "grad_norm": 1.60394287109375, "learning_rate": 9.858335310330492e-05, "loss": 0.5087047815322876, "memory/device_reserved (GiB)": 41.36, "memory/max_active (GiB)": 37.09, "memory/max_allocated (GiB)": 37.09, "ppl": 1.66314, "step": 65, "tokens/total": 2239632, "tokens/train_per_sec_per_gpu": 63.63, "tokens/trainable": 67900} +{"epoch": 0.2578125, "grad_norm": 3.055310010910034, "learning_rate": 9.851018180226185e-05, "loss": 0.5083943605422974, "memory/device_reserved (GiB)": 41.42, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.66262, "step": 66, "tokens/total": 2278304, "tokens/train_per_sec_per_gpu": 54.25, "tokens/trainable": 68970} +{"epoch": 0.26171875, "grad_norm": 4.76242208480835, "learning_rate": 9.843519986495259e-05, "loss": 0.5751352310180664, "memory/device_reserved (GiB)": 41.42, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.77737, "step": 67, "tokens/total": 2313248, "tokens/train_per_sec_per_gpu": 60.18, "tokens/trainable": 70021} +{"epoch": 0.265625, "grad_norm": 2.465327262878418, "learning_rate": 9.835841041168162e-05, "loss": 0.5294528007507324, "memory/device_reserved (GiB)": 41.42, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.698, "step": 68, "tokens/total": 2350656, "tokens/train_per_sec_per_gpu": 60.91, "tokens/trainable": 71127} +{"epoch": 0.26953125, "grad_norm": 1.3505886793136597, "learning_rate": 9.82798166379715e-05, "loss": 0.5012938380241394, "memory/device_reserved (GiB)": 41.42, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.65086, "step": 69, "tokens/total": 2383856, "tokens/train_per_sec_per_gpu": 61.94, "tokens/trainable": 72211} +{"epoch": 0.2734375, "grad_norm": 1.489576816558838, "learning_rate": 9.819942181443002e-05, "loss": 0.646255612373352, "memory/device_reserved (GiB)": 41.42, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.90838, "step": 70, "tokens/total": 2421120, "tokens/train_per_sec_per_gpu": 58.58, "tokens/trainable": 73295} +{"epoch": 0.27734375, "grad_norm": 1.538165807723999, "learning_rate": 9.811722928661392e-05, "loss": 0.6091792583465576, "memory/device_reserved (GiB)": 41.43, "memory/max_active (GiB)": 35.8, "memory/max_allocated (GiB)": 35.8, "ppl": 1.83892, "step": 71, "tokens/total": 2449904, "tokens/train_per_sec_per_gpu": 72.29, "tokens/trainable": 74322} +{"epoch": 0.28125, "grad_norm": 1.3721330165863037, "learning_rate": 9.803324247488975e-05, "loss": 0.4970642924308777, "memory/device_reserved (GiB)": 41.43, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.64389, "step": 72, "tokens/total": 2485936, "tokens/train_per_sec_per_gpu": 57.77, "tokens/trainable": 75392} +{"epoch": 0.28515625, "grad_norm": 1.7424070835113525, "learning_rate": 9.794746487429161e-05, "loss": 0.48756375908851624, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.04, "memory/max_allocated (GiB)": 37.04, "ppl": 1.62834, "step": 73, "tokens/total": 2519024, "tokens/train_per_sec_per_gpu": 67.79, "tokens/trainable": 76431} +{"epoch": 0.2890625, "grad_norm": 1.3134344816207886, "learning_rate": 9.785990005437554e-05, "loss": 0.5653057098388672, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.81, "memory/max_allocated (GiB)": 37.81, "ppl": 1.75999, "step": 74, "tokens/total": 2554864, "tokens/train_per_sec_per_gpu": 61.44, "tokens/trainable": 77515} +{"epoch": 0.29296875, "grad_norm": 25.715065002441406, "learning_rate": 9.777055165907117e-05, "loss": 0.522104024887085, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.68557, "step": 75, "tokens/total": 2588048, "tokens/train_per_sec_per_gpu": 66.63, "tokens/trainable": 78562} +{"epoch": 0.296875, "grad_norm": 4.217231750488281, "learning_rate": 9.767942340652993e-05, "loss": 0.5786105394363403, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.05, "memory/max_allocated (GiB)": 38.05, "ppl": 1.78356, "step": 76, "tokens/total": 2622784, "tokens/train_per_sec_per_gpu": 55.23, "tokens/trainable": 79581} +{"epoch": 0.30078125, "grad_norm": 4.907668590545654, "learning_rate": 9.758651908897035e-05, "loss": 0.5356509685516357, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.70856, "step": 77, "tokens/total": 2656416, "tokens/train_per_sec_per_gpu": 62.3, "tokens/trainable": 80653} +{"epoch": 0.3046875, "grad_norm": 3.1586804389953613, "learning_rate": 9.749184257252033e-05, "loss": 0.5833450555801392, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.79202, "step": 78, "tokens/total": 2690672, "tokens/train_per_sec_per_gpu": 61.97, "tokens/trainable": 81654} +{"epoch": 0.30859375, "grad_norm": 1.5110328197479248, "learning_rate": 9.739539779705614e-05, "loss": 0.46050184965133667, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.58487, "step": 79, "tokens/total": 2729200, "tokens/train_per_sec_per_gpu": 58.55, "tokens/trainable": 82851} +{"epoch": 0.3125, "grad_norm": 4.235539436340332, "learning_rate": 9.729718877603861e-05, "loss": 0.573461651802063, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.7744, "step": 80, "tokens/total": 2762784, "tokens/train_per_sec_per_gpu": 69.07, "tokens/trainable": 83909} +{"epoch": 0.31640625, "grad_norm": 1.4623039960861206, "learning_rate": 9.719721959634592e-05, "loss": 0.4465371370315552, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.56289, "step": 81, "tokens/total": 2798304, "tokens/train_per_sec_per_gpu": 61.41, "tokens/trainable": 84958} +{"epoch": 0.3203125, "grad_norm": 2.292548418045044, "learning_rate": 9.709549441810375e-05, "loss": 0.48102691769599915, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.61773, "step": 82, "tokens/total": 2835712, "tokens/train_per_sec_per_gpu": 60.57, "tokens/trainable": 86038} +{"epoch": 0.32421875, "grad_norm": 2.708446741104126, "learning_rate": 9.699201747451195e-05, "loss": 0.497903436422348, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.54, "memory/max_allocated (GiB)": 36.54, "ppl": 1.64527, "step": 83, "tokens/total": 2868224, "tokens/train_per_sec_per_gpu": 58.73, "tokens/trainable": 87091} +{"epoch": 0.328125, "grad_norm": 1.7877753973007202, "learning_rate": 9.688679307166854e-05, "loss": 0.5402355194091797, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.71641, "step": 84, "tokens/total": 2900736, "tokens/train_per_sec_per_gpu": 63.63, "tokens/trainable": 88143} +{"epoch": 0.33203125, "grad_norm": 1.4409058094024658, "learning_rate": 9.677982558839042e-05, "loss": 0.5496367812156677, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.82, "memory/max_allocated (GiB)": 36.82, "ppl": 1.73262, "step": 85, "tokens/total": 2933904, "tokens/train_per_sec_per_gpu": 56.55, "tokens/trainable": 89198} +{"epoch": 0.3359375, "grad_norm": 1.792243242263794, "learning_rate": 9.66711194760312e-05, "loss": 0.44553831219673157, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 39.04, "memory/max_allocated (GiB)": 39.04, "ppl": 1.56133, "step": 86, "tokens/total": 2971344, "tokens/train_per_sec_per_gpu": 55.95, "tokens/trainable": 90327} +{"epoch": 0.33984375, "grad_norm": 1.530079960823059, "learning_rate": 9.656067925829593e-05, "loss": 0.4971941113471985, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.6441, "step": 87, "tokens/total": 3001936, "tokens/train_per_sec_per_gpu": 71.51, "tokens/trainable": 91361} +{"epoch": 0.34375, "grad_norm": 1.5389336347579956, "learning_rate": 9.644850953105288e-05, "loss": 0.5172426700592041, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 35.88, "memory/max_allocated (GiB)": 35.88, "ppl": 1.6774, "step": 88, "tokens/total": 3032944, "tokens/train_per_sec_per_gpu": 68.37, "tokens/trainable": 92445} +{"epoch": 0.34765625, "grad_norm": 2.8289437294006348, "learning_rate": 9.633461496214225e-05, "loss": 0.44299471378326416, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.55736, "step": 89, "tokens/total": 3065584, "tokens/train_per_sec_per_gpu": 62.56, "tokens/trainable": 93502} +{"epoch": 0.3515625, "grad_norm": 2.305694580078125, "learning_rate": 9.621900029118195e-05, "loss": 0.5768658518791199, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.1, "memory/max_allocated (GiB)": 36.1, "ppl": 1.78045, "step": 90, "tokens/total": 3095248, "tokens/train_per_sec_per_gpu": 69.45, "tokens/trainable": 94544} +{"epoch": 0.35546875, "grad_norm": 3.3755929470062256, "learning_rate": 9.610167032937036e-05, "loss": 0.47213467955589294, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.60341, "step": 91, "tokens/total": 3133456, "tokens/train_per_sec_per_gpu": 54.54, "tokens/trainable": 95597} +{"epoch": 0.359375, "grad_norm": 1.2387703657150269, "learning_rate": 9.598262995928611e-05, "loss": 0.46297916769981384, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.5888, "step": 92, "tokens/total": 3164624, "tokens/train_per_sec_per_gpu": 70.58, "tokens/trainable": 96662} +{"epoch": 0.36328125, "grad_norm": 8.585007667541504, "learning_rate": 9.586188413468492e-05, "loss": 0.4456758499145508, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.8, "memory/max_allocated (GiB)": 38.8, "ppl": 1.56155, "step": 93, "tokens/total": 3203008, "tokens/train_per_sec_per_gpu": 49.7, "tokens/trainable": 97715} +{"epoch": 0.3671875, "grad_norm": 2.1850242614746094, "learning_rate": 9.57394378802934e-05, "loss": 0.4386202096939087, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.55057, "step": 94, "tokens/total": 3236448, "tokens/train_per_sec_per_gpu": 61.01, "tokens/trainable": 98761} +{"epoch": 0.37109375, "grad_norm": 1.405388593673706, "learning_rate": 9.56152962916e-05, "loss": 0.4582071304321289, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.58124, "step": 95, "tokens/total": 3267104, "tokens/train_per_sec_per_gpu": 66.51, "tokens/trainable": 99757} +{"epoch": 0.375, "grad_norm": 1.4918923377990723, "learning_rate": 9.548946453464296e-05, "loss": 0.38246408104896545, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 1.46589, "step": 96, "tokens/total": 3301440, "tokens/train_per_sec_per_gpu": 62.95, "tokens/trainable": 100862} +{"epoch": 0.37890625, "grad_norm": 10.729366302490234, "learning_rate": 9.53619478457953e-05, "loss": 0.506010115146637, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.65866, "step": 97, "tokens/total": 3336112, "tokens/train_per_sec_per_gpu": 52.71, "tokens/trainable": 101868} +{"epoch": 0.3828125, "grad_norm": 1.5747276544570923, "learning_rate": 9.523275153154695e-05, "loss": 0.4825636148452759, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.62022, "step": 98, "tokens/total": 3368624, "tokens/train_per_sec_per_gpu": 58.08, "tokens/trainable": 102828} +{"epoch": 0.38671875, "grad_norm": 1.7378532886505127, "learning_rate": 9.51018809682839e-05, "loss": 0.41804274916648865, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.51899, "step": 99, "tokens/total": 3406448, "tokens/train_per_sec_per_gpu": 53.13, "tokens/trainable": 103845} +{"epoch": 0.390625, "grad_norm": 1.8017516136169434, "learning_rate": 9.49693416020645e-05, "loss": 0.4844069480895996, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.51, "memory/max_allocated (GiB)": 38.51, "ppl": 1.62321, "step": 100, "tokens/total": 3444240, "tokens/train_per_sec_per_gpu": 55.07, "tokens/trainable": 104865} +{"epoch": 0.39453125, "grad_norm": 1.8356019258499146, "learning_rate": 9.483513894839276e-05, "loss": 0.48894524574279785, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.6306, "step": 101, "tokens/total": 3477680, "tokens/train_per_sec_per_gpu": 68.27, "tokens/trainable": 105968} +{"epoch": 0.3984375, "grad_norm": 4.014509677886963, "learning_rate": 9.469927859198888e-05, "loss": 0.45534947514533997, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.57672, "step": 102, "tokens/total": 3513616, "tokens/train_per_sec_per_gpu": 58.8, "tokens/trainable": 106968} +{"epoch": 0.40234375, "grad_norm": 1.329097867012024, "learning_rate": 9.456176618655689e-05, "loss": 0.430510014295578, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.51, "memory/max_allocated (GiB)": 36.51, "ppl": 1.53804, "step": 103, "tokens/total": 3545584, "tokens/train_per_sec_per_gpu": 60.68, "tokens/trainable": 107964} +{"epoch": 0.40625, "grad_norm": 1.5829964876174927, "learning_rate": 9.442260745454927e-05, "loss": 0.43621063232421875, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.54683, "step": 104, "tokens/total": 3580768, "tokens/train_per_sec_per_gpu": 59.33, "tokens/trainable": 109035} +{"epoch": 0.41015625, "grad_norm": 1.482440710067749, "learning_rate": 9.428180818692884e-05, "loss": 0.4167865812778473, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.51708, "step": 105, "tokens/total": 3617008, "tokens/train_per_sec_per_gpu": 52.69, "tokens/trainable": 110095} +{"epoch": 0.4140625, "grad_norm": 1.642553687095642, "learning_rate": 9.413937424292791e-05, "loss": 0.5113812685012817, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.66759, "step": 106, "tokens/total": 3655936, "tokens/train_per_sec_per_gpu": 52.58, "tokens/trainable": 111135} +{"epoch": 0.41796875, "grad_norm": 3.5985357761383057, "learning_rate": 9.399531154980424e-05, "loss": 0.40825915336608887, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.5042, "step": 107, "tokens/total": 3692736, "tokens/train_per_sec_per_gpu": 51.75, "tokens/trainable": 112140} +{"epoch": 0.421875, "grad_norm": 1.4516206979751587, "learning_rate": 9.384962610259455e-05, "loss": 0.4549720883369446, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.57613, "step": 108, "tokens/total": 3727248, "tokens/train_per_sec_per_gpu": 55.01, "tokens/trainable": 113165} +{"epoch": 0.42578125, "grad_norm": 1.4002677202224731, "learning_rate": 9.370232396386494e-05, "loss": 0.4700906276702881, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.60014, "step": 109, "tokens/total": 3760464, "tokens/train_per_sec_per_gpu": 61.11, "tokens/trainable": 114212} +{"epoch": 0.4296875, "grad_norm": 2.06840443611145, "learning_rate": 9.355341126345868e-05, "loss": 0.38585835695266724, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.47088, "step": 110, "tokens/total": 3795952, "tokens/train_per_sec_per_gpu": 61.42, "tokens/trainable": 115312} +{"epoch": 0.43359375, "grad_norm": 1.329538106918335, "learning_rate": 9.340289419824107e-05, "loss": 0.3628474175930023, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.74, "memory/max_allocated (GiB)": 36.74, "ppl": 1.43742, "step": 111, "tokens/total": 3828144, "tokens/train_per_sec_per_gpu": 69.24, "tokens/trainable": 116343} +{"epoch": 0.4375, "grad_norm": 2.5097227096557617, "learning_rate": 9.325077903184159e-05, "loss": 0.5993927717208862, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.82101, "step": 112, "tokens/total": 3862992, "tokens/train_per_sec_per_gpu": 62.14, "tokens/trainable": 117463} +{"epoch": 0.44140625, "grad_norm": 1.5980772972106934, "learning_rate": 9.30970720943932e-05, "loss": 0.38238316774368286, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.43, "memory/max_allocated (GiB)": 38.43, "ppl": 1.46577, "step": 113, "tokens/total": 3898416, "tokens/train_per_sec_per_gpu": 59.29, "tokens/trainable": 118557} +{"epoch": 0.4453125, "grad_norm": 1.5903464555740356, "learning_rate": 9.2941779782269e-05, "loss": 0.416101336479187, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.51604, "step": 114, "tokens/total": 3932176, "tokens/train_per_sec_per_gpu": 61.32, "tokens/trainable": 119622} +{"epoch": 0.44921875, "grad_norm": 1.3836725950241089, "learning_rate": 9.278490855781596e-05, "loss": 0.4407464563846588, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.55387, "step": 115, "tokens/total": 3964384, "tokens/train_per_sec_per_gpu": 71.51, "tokens/trainable": 120662} +{"epoch": 0.453125, "grad_norm": 1.8447723388671875, "learning_rate": 9.262646494908604e-05, "loss": 0.4638434052467346, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 39.14, "memory/max_allocated (GiB)": 39.14, "ppl": 1.59017, "step": 116, "tokens/total": 4002960, "tokens/train_per_sec_per_gpu": 52.13, "tokens/trainable": 121672} +{"epoch": 0.45703125, "grad_norm": 2.2597692012786865, "learning_rate": 9.246645554956457e-05, "loss": 0.43707969784736633, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.54818, "step": 117, "tokens/total": 4037552, "tokens/train_per_sec_per_gpu": 62.54, "tokens/trainable": 122725} +{"epoch": 0.4609375, "grad_norm": 1.6953067779541016, "learning_rate": 9.230488701789578e-05, "loss": 0.5201123952865601, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.7, "memory/max_allocated (GiB)": 38.7, "ppl": 1.68222, "step": 118, "tokens/total": 4075072, "tokens/train_per_sec_per_gpu": 59.07, "tokens/trainable": 123843} +{"epoch": 0.46484375, "grad_norm": 1.7751185894012451, "learning_rate": 9.214176607760577e-05, "loss": 0.36699169874191284, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.44339, "step": 119, "tokens/total": 4113744, "tokens/train_per_sec_per_gpu": 58.62, "tokens/trainable": 124974} +{"epoch": 0.46875, "grad_norm": 3.007453441619873, "learning_rate": 9.197709951682268e-05, "loss": 0.37369304895401, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.45309, "step": 120, "tokens/total": 4146032, "tokens/train_per_sec_per_gpu": 65.23, "tokens/trainable": 126039} +{"epoch": 0.47265625, "grad_norm": 1.731414556503296, "learning_rate": 9.181089418799428e-05, "loss": 0.4052283763885498, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 39.39, "memory/max_allocated (GiB)": 39.39, "ppl": 1.49964, "step": 121, "tokens/total": 4182832, "tokens/train_per_sec_per_gpu": 55.09, "tokens/trainable": 127078} +{"epoch": 0.4765625, "grad_norm": 1.5586636066436768, "learning_rate": 9.164315700760271e-05, "loss": 0.3281603157520294, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.38841, "step": 122, "tokens/total": 4218640, "tokens/train_per_sec_per_gpu": 58.5, "tokens/trainable": 128122} +{"epoch": 0.48046875, "grad_norm": 1.8894630670547485, "learning_rate": 9.147389495587671e-05, "loss": 0.45857876539230347, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 1.58182, "step": 123, "tokens/total": 4249792, "tokens/train_per_sec_per_gpu": 69.41, "tokens/trainable": 129160} +{"epoch": 0.484375, "grad_norm": 1.4898499250411987, "learning_rate": 9.130311507650116e-05, "loss": 0.34617379307746887, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 1.41365, "step": 124, "tokens/total": 4282464, "tokens/train_per_sec_per_gpu": 61.79, "tokens/trainable": 130215} +{"epoch": 0.48828125, "grad_norm": 1.4212357997894287, "learning_rate": 9.113082447632394e-05, "loss": 0.36718493700027466, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.44366, "step": 125, "tokens/total": 4314608, "tokens/train_per_sec_per_gpu": 79.75, "tokens/trainable": 131358} +{"epoch": 0.4921875, "grad_norm": 1.7930868864059448, "learning_rate": 9.09570303250602e-05, "loss": 0.4044566750526428, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.47, "memory/max_allocated (GiB)": 36.47, "ppl": 1.49849, "step": 126, "tokens/total": 4346416, "tokens/train_per_sec_per_gpu": 63.32, "tokens/trainable": 132421} +{"epoch": 0.49609375, "grad_norm": 1.8330329656600952, "learning_rate": 9.078173985499394e-05, "loss": 0.37108322978019714, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 35.79, "memory/max_allocated (GiB)": 35.79, "ppl": 1.4493, "step": 127, "tokens/total": 4376400, "tokens/train_per_sec_per_gpu": 65.58, "tokens/trainable": 133425} +{"epoch": 0.5, "grad_norm": 1.4855762720108032, "learning_rate": 9.060496036067713e-05, "loss": 0.3969651162624359, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 38.72, "memory/max_allocated (GiB)": 38.72, "ppl": 1.4873, "step": 128, "tokens/total": 4413824, "tokens/train_per_sec_per_gpu": 48.2, "tokens/trainable": 134439} +{"epoch": 0.50390625, "grad_norm": 1.877728819847107, "learning_rate": 9.042669919862615e-05, "loss": 0.4705854654312134, "memory/device_reserved (GiB)": 41.92, "memory/max_active (GiB)": 36.82, "memory/max_allocated (GiB)": 36.82, "ppl": 1.60093, "step": 129, "tokens/total": 4445936, "tokens/train_per_sec_per_gpu": 67.67, "tokens/trainable": 135505} +{"epoch": 0.5078125, "grad_norm": 1.3678077459335327, "learning_rate": 9.024696378701557e-05, "loss": 0.3702026605606079, "memory/device_reserved (GiB)": 41.42, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.44803, "step": 130, "tokens/total": 4481488, "tokens/train_per_sec_per_gpu": 58.61, "tokens/trainable": 136588} +{"epoch": 0.51171875, "grad_norm": 2.030912160873413, "learning_rate": 9.006576160536948e-05, "loss": 0.40476423501968384, "memory/device_reserved (GiB)": 41.42, "memory/max_active (GiB)": 37.43, "memory/max_allocated (GiB)": 37.43, "ppl": 1.49895, "step": 131, "tokens/total": 4516048, "tokens/train_per_sec_per_gpu": 56.11, "tokens/trainable": 137590} +{"epoch": 0.515625, "grad_norm": 1.4746774435043335, "learning_rate": 8.988310019425035e-05, "loss": 0.37946993112564087, "memory/device_reserved (GiB)": 41.71, "memory/max_active (GiB)": 39.7, "memory/max_allocated (GiB)": 39.7, "ppl": 1.46151, "step": 132, "tokens/total": 4553904, "tokens/train_per_sec_per_gpu": 51.56, "tokens/trainable": 138600} +{"epoch": 0.51953125, "grad_norm": 1.5867228507995605, "learning_rate": 8.969898715494506e-05, "loss": 0.3613826334476471, "memory/device_reserved (GiB)": 41.73, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.43531, "step": 133, "tokens/total": 4590736, "tokens/train_per_sec_per_gpu": 61.05, "tokens/trainable": 139690} +{"epoch": 0.5234375, "grad_norm": 1.7150899171829224, "learning_rate": 8.951343014914869e-05, "loss": 0.3498016595840454, "memory/device_reserved (GiB)": 41.73, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.41879, "step": 134, "tokens/total": 4627392, "tokens/train_per_sec_per_gpu": 57.28, "tokens/trainable": 140782} +{"epoch": 0.52734375, "grad_norm": 1.5689284801483154, "learning_rate": 8.932643689864568e-05, "loss": 0.3503245711326599, "memory/device_reserved (GiB)": 41.73, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.41953, "step": 135, "tokens/total": 4663968, "tokens/train_per_sec_per_gpu": 62.15, "tokens/trainable": 141899} +{"epoch": 0.53125, "grad_norm": 1.6985130310058594, "learning_rate": 8.913801518498845e-05, "loss": 0.32164663076400757, "memory/device_reserved (GiB)": 41.73, "memory/max_active (GiB)": 39.68, "memory/max_allocated (GiB)": 39.68, "ppl": 1.3794, "step": 136, "tokens/total": 4703344, "tokens/train_per_sec_per_gpu": 48.56, "tokens/trainable": 143011} +{"epoch": 0.53515625, "grad_norm": 1.429890751838684, "learning_rate": 8.894817284917364e-05, "loss": 0.3601451516151428, "memory/device_reserved (GiB)": 41.73, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.43354, "step": 137, "tokens/total": 4740288, "tokens/train_per_sec_per_gpu": 50.26, "tokens/trainable": 144045} +{"epoch": 0.5390625, "grad_norm": 1.3188704252243042, "learning_rate": 8.875691779131569e-05, "loss": 0.38143110275268555, "memory/device_reserved (GiB)": 41.9, "memory/max_active (GiB)": 39.78, "memory/max_allocated (GiB)": 39.78, "ppl": 1.46438, "step": 138, "tokens/total": 4778224, "tokens/train_per_sec_per_gpu": 56.22, "tokens/trainable": 145172} +{"epoch": 0.54296875, "grad_norm": 36.98548889160156, "learning_rate": 8.856425797031829e-05, "loss": 0.29051482677459717, "memory/device_reserved (GiB)": 42.22, "memory/max_active (GiB)": 39.58, "memory/max_allocated (GiB)": 39.58, "ppl": 1.33712, "step": 139, "tokens/total": 4814256, "tokens/train_per_sec_per_gpu": 63.73, "tokens/trainable": 146300} +{"epoch": 0.546875, "grad_norm": 1.6847885847091675, "learning_rate": 8.837020140354295e-05, "loss": 0.3592267334461212, "memory/device_reserved (GiB)": 42.22, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.43222, "step": 140, "tokens/total": 4851072, "tokens/train_per_sec_per_gpu": 55.34, "tokens/trainable": 147307} +{"epoch": 0.55078125, "grad_norm": 1.7236360311508179, "learning_rate": 8.817475616647554e-05, "loss": 0.39077243208885193, "memory/device_reserved (GiB)": 42.22, "memory/max_active (GiB)": 38.17, "memory/max_allocated (GiB)": 38.17, "ppl": 1.47812, "step": 141, "tokens/total": 4885504, "tokens/train_per_sec_per_gpu": 60.07, "tokens/trainable": 148393} +{"epoch": 0.5546875, "grad_norm": 2.1619787216186523, "learning_rate": 8.797793039239017e-05, "loss": 0.3863121569156647, "memory/device_reserved (GiB)": 42.22, "memory/max_active (GiB)": 37.83, "memory/max_allocated (GiB)": 37.83, "ppl": 1.47154, "step": 142, "tokens/total": 4920528, "tokens/train_per_sec_per_gpu": 64.72, "tokens/trainable": 149499} +{"epoch": 0.55859375, "grad_norm": 1.472019076347351, "learning_rate": 8.777973227201069e-05, "loss": 0.3491016626358032, "memory/device_reserved (GiB)": 43.76, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.41779, "step": 143, "tokens/total": 4957440, "tokens/train_per_sec_per_gpu": 60.42, "tokens/trainable": 150562} +{"epoch": 0.5625, "grad_norm": 2.24379563331604, "learning_rate": 8.758017005316988e-05, "loss": 0.3809899091720581, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.46373, "step": 144, "tokens/total": 4990752, "tokens/train_per_sec_per_gpu": 72.74, "tokens/trainable": 151616} +{"epoch": 0.56640625, "grad_norm": 1.4432369470596313, "learning_rate": 8.737925204046629e-05, "loss": 0.41598522663116455, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.51586, "step": 145, "tokens/total": 5025472, "tokens/train_per_sec_per_gpu": 54.72, "tokens/trainable": 152681} +{"epoch": 0.5703125, "grad_norm": 1.2111608982086182, "learning_rate": 8.717698659491851e-05, "loss": 0.32367101311683655, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 38.91, "memory/max_allocated (GiB)": 38.91, "ppl": 1.38219, "step": 146, "tokens/total": 5062304, "tokens/train_per_sec_per_gpu": 57.01, "tokens/trainable": 153785} +{"epoch": 0.57421875, "grad_norm": 2.0112831592559814, "learning_rate": 8.697338213361735e-05, "loss": 0.3633330464363098, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 36.45, "memory/max_allocated (GiB)": 36.45, "ppl": 1.43811, "step": 147, "tokens/total": 5093280, "tokens/train_per_sec_per_gpu": 69.78, "tokens/trainable": 154886} +{"epoch": 0.578125, "grad_norm": 1.0705114603042603, "learning_rate": 8.676844712937552e-05, "loss": 0.30196669697761536, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.35252, "step": 148, "tokens/total": 5133280, "tokens/train_per_sec_per_gpu": 49.17, "tokens/trainable": 155938} +{"epoch": 0.58203125, "grad_norm": 1.2151323556900024, "learning_rate": 8.656219011037509e-05, "loss": 0.3424830436706543, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.40844, "step": 149, "tokens/total": 5170576, "tokens/train_per_sec_per_gpu": 55.62, "tokens/trainable": 157021} +{"epoch": 0.5859375, "grad_norm": 6.580717086791992, "learning_rate": 8.63546196598125e-05, "loss": 0.35557812452316284, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.42701, "step": 150, "tokens/total": 5208160, "tokens/train_per_sec_per_gpu": 55.79, "tokens/trainable": 158173} +{"epoch": 0.58984375, "grad_norm": 1.909930944442749, "learning_rate": 8.614574441554145e-05, "loss": 0.43499788641929626, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 36.95, "memory/max_allocated (GiB)": 36.95, "ppl": 1.54496, "step": 151, "tokens/total": 5240784, "tokens/train_per_sec_per_gpu": 63.19, "tokens/trainable": 159145} +{"epoch": 0.59375, "grad_norm": 1.5423970222473145, "learning_rate": 8.593557306971349e-05, "loss": 0.3243396282196045, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.38312, "step": 152, "tokens/total": 5274096, "tokens/train_per_sec_per_gpu": 58.53, "tokens/trainable": 160178} +{"epoch": 0.59765625, "grad_norm": 2.806018114089966, "learning_rate": 8.572411436841618e-05, "loss": 0.3586214482784271, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 36.9, "memory/max_allocated (GiB)": 36.9, "ppl": 1.43135, "step": 153, "tokens/total": 5307840, "tokens/train_per_sec_per_gpu": 64.36, "tokens/trainable": 161248} +{"epoch": 0.6015625, "grad_norm": 1.743974208831787, "learning_rate": 8.551137711130922e-05, "loss": 0.3433808386325836, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.40971, "step": 154, "tokens/total": 5339616, "tokens/train_per_sec_per_gpu": 59.21, "tokens/trainable": 162266} +{"epoch": 0.60546875, "grad_norm": 1.5566474199295044, "learning_rate": 8.529737015125824e-05, "loss": 0.2839331328868866, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 38.17, "memory/max_allocated (GiB)": 38.17, "ppl": 1.32834, "step": 155, "tokens/total": 5374720, "tokens/train_per_sec_per_gpu": 56.74, "tokens/trainable": 163285} +{"epoch": 0.609375, "grad_norm": 1.3389400243759155, "learning_rate": 8.508210239396639e-05, "loss": 0.34026622772216797, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 39.77, "memory/max_allocated (GiB)": 39.77, "ppl": 1.40532, "step": 156, "tokens/total": 5412608, "tokens/train_per_sec_per_gpu": 57.2, "tokens/trainable": 164302} +{"epoch": 0.61328125, "grad_norm": 1.3804423809051514, "learning_rate": 8.486558279760375e-05, "loss": 0.3483239412307739, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 35.9, "memory/max_allocated (GiB)": 35.9, "ppl": 1.41669, "step": 157, "tokens/total": 5443664, "tokens/train_per_sec_per_gpu": 64.08, "tokens/trainable": 165309} +{"epoch": 0.6171875, "grad_norm": 1.4441436529159546, "learning_rate": 8.464782037243449e-05, "loss": 0.3528158664703369, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.42307, "step": 158, "tokens/total": 5475808, "tokens/train_per_sec_per_gpu": 60.68, "tokens/trainable": 166298} +{"epoch": 0.62109375, "grad_norm": 1.4114998579025269, "learning_rate": 8.442882418044202e-05, "loss": 0.3142455220222473, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.36923, "step": 159, "tokens/total": 5508448, "tokens/train_per_sec_per_gpu": 63.64, "tokens/trainable": 167338} +{"epoch": 0.625, "grad_norm": 1.6489492654800415, "learning_rate": 8.420860333495179e-05, "loss": 0.374631404876709, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.45446, "step": 160, "tokens/total": 5540976, "tokens/train_per_sec_per_gpu": 64.72, "tokens/trainable": 168338} +{"epoch": 0.62890625, "grad_norm": 1.5164738893508911, "learning_rate": 8.398716700025208e-05, "loss": 0.3843270540237427, "memory/device_reserved (GiB)": 43.77, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.46863, "step": 161, "tokens/total": 5574496, "tokens/train_per_sec_per_gpu": 61.42, "tokens/trainable": 169378} +{"epoch": 0.6328125, "grad_norm": 1.6514025926589966, "learning_rate": 8.376452439121266e-05, "loss": 0.42841535806655884, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.05, "memory/max_allocated (GiB)": 36.05, "ppl": 1.53482, "step": 162, "tokens/total": 5604544, "tokens/train_per_sec_per_gpu": 69.78, "tokens/trainable": 170407} +{"epoch": 0.63671875, "grad_norm": 2.0843327045440674, "learning_rate": 8.354068477290124e-05, "loss": 0.34798312187194824, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.41621, "step": 163, "tokens/total": 5638768, "tokens/train_per_sec_per_gpu": 61.3, "tokens/trainable": 171394} +{"epoch": 0.640625, "grad_norm": 1.6335402727127075, "learning_rate": 8.331565746019807e-05, "loss": 0.35140261054039, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.42106, "step": 164, "tokens/total": 5670960, "tokens/train_per_sec_per_gpu": 68.94, "tokens/trainable": 172444} +{"epoch": 0.64453125, "grad_norm": 1.6585966348648071, "learning_rate": 8.308945181740812e-05, "loss": 0.387387216091156, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.47313, "step": 165, "tokens/total": 5705664, "tokens/train_per_sec_per_gpu": 62.15, "tokens/trainable": 173468} +{"epoch": 0.6484375, "grad_norm": 1.1981943845748901, "learning_rate": 8.286207725787153e-05, "loss": 0.2870294153690338, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.33246, "step": 166, "tokens/total": 5742672, "tokens/train_per_sec_per_gpu": 51.85, "tokens/trainable": 174495} +{"epoch": 0.65234375, "grad_norm": 1.9920854568481445, "learning_rate": 8.263354324357182e-05, "loss": 0.31014585494995117, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.12, "memory/max_allocated (GiB)": 39.12, "ppl": 1.36362, "step": 167, "tokens/total": 5777408, "tokens/train_per_sec_per_gpu": 60.24, "tokens/trainable": 175533} +{"epoch": 0.65625, "grad_norm": 1.4061107635498047, "learning_rate": 8.240385928474219e-05, "loss": 0.3703303039073944, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.44821, "step": 168, "tokens/total": 5810224, "tokens/train_per_sec_per_gpu": 58.01, "tokens/trainable": 176493} +{"epoch": 0.66015625, "grad_norm": 1.1416693925857544, "learning_rate": 8.217303493946967e-05, "loss": 0.34752368927001953, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.41556, "step": 169, "tokens/total": 5844432, "tokens/train_per_sec_per_gpu": 58.02, "tokens/trainable": 177502} +{"epoch": 0.6640625, "grad_norm": 0.9960429072380066, "learning_rate": 8.194107981329746e-05, "loss": 0.3164145350456238, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.73, "memory/max_allocated (GiB)": 39.73, "ppl": 1.3722, "step": 170, "tokens/total": 5883008, "tokens/train_per_sec_per_gpu": 51.63, "tokens/trainable": 178523} +{"epoch": 0.66796875, "grad_norm": 1.2896519899368286, "learning_rate": 8.170800355882518e-05, "loss": 0.2891680598258972, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.33532, "step": 171, "tokens/total": 5917440, "tokens/train_per_sec_per_gpu": 66.37, "tokens/trainable": 179562} +{"epoch": 0.671875, "grad_norm": 3.2128381729125977, "learning_rate": 8.147381587530713e-05, "loss": 0.39703378081321716, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.39, "memory/max_allocated (GiB)": 37.39, "ppl": 1.48741, "step": 172, "tokens/total": 5951232, "tokens/train_per_sec_per_gpu": 63.03, "tokens/trainable": 180566} +{"epoch": 0.67578125, "grad_norm": 1.6058439016342163, "learning_rate": 8.123852650824877e-05, "loss": 0.35641980171203613, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.45, "memory/max_allocated (GiB)": 37.45, "ppl": 1.42821, "step": 173, "tokens/total": 5985056, "tokens/train_per_sec_per_gpu": 63.47, "tokens/trainable": 181642} +{"epoch": 0.6796875, "grad_norm": 1.6758099794387817, "learning_rate": 8.100214524900103e-05, "loss": 0.38127601146698, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.46415, "step": 174, "tokens/total": 6020128, "tokens/train_per_sec_per_gpu": 52.07, "tokens/trainable": 182645} +{"epoch": 0.68359375, "grad_norm": 1.5955839157104492, "learning_rate": 8.076468193435301e-05, "loss": 0.34679269790649414, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.41452, "step": 175, "tokens/total": 6050976, "tokens/train_per_sec_per_gpu": 66.68, "tokens/trainable": 183593} +{"epoch": 0.6875, "grad_norm": 1.2694306373596191, "learning_rate": 8.052614644612253e-05, "loss": 0.29586073756217957, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.34428, "step": 176, "tokens/total": 6088816, "tokens/train_per_sec_per_gpu": 49.99, "tokens/trainable": 184643} +{"epoch": 0.69140625, "grad_norm": 2.493555784225464, "learning_rate": 8.028654871074489e-05, "loss": 0.29153722524642944, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.33848, "step": 177, "tokens/total": 6123408, "tokens/train_per_sec_per_gpu": 59.09, "tokens/trainable": 185729} +{"epoch": 0.6953125, "grad_norm": 1.6019831895828247, "learning_rate": 8.004589869885986e-05, "loss": 0.42456793785095215, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.37, "memory/max_allocated (GiB)": 36.37, "ppl": 1.52893, "step": 178, "tokens/total": 6154880, "tokens/train_per_sec_per_gpu": 61.75, "tokens/trainable": 186718} +{"epoch": 0.69921875, "grad_norm": 1.2731668949127197, "learning_rate": 7.980420642489674e-05, "loss": 0.3048350512981415, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.3564, "step": 179, "tokens/total": 6193584, "tokens/train_per_sec_per_gpu": 53.69, "tokens/trainable": 187750} +{"epoch": 0.703125, "grad_norm": 1.2545480728149414, "learning_rate": 7.95614819466576e-05, "loss": 0.33444923162460327, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 1.39717, "step": 180, "tokens/total": 6227296, "tokens/train_per_sec_per_gpu": 61.72, "tokens/trainable": 188825} +{"epoch": 0.70703125, "grad_norm": 1.2708393335342407, "learning_rate": 7.931773536489872e-05, "loss": 0.32794755697250366, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.38812, "step": 181, "tokens/total": 6261888, "tokens/train_per_sec_per_gpu": 62.6, "tokens/trainable": 189884} +{"epoch": 0.7109375, "grad_norm": 1.4100549221038818, "learning_rate": 7.907297682291035e-05, "loss": 0.3217083513736725, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.37948, "step": 182, "tokens/total": 6296544, "tokens/train_per_sec_per_gpu": 60.83, "tokens/trainable": 190987} +{"epoch": 0.71484375, "grad_norm": 1.1796455383300781, "learning_rate": 7.882721650609442e-05, "loss": 0.23649081587791443, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.65, "memory/max_allocated (GiB)": 39.65, "ppl": 1.2668, "step": 183, "tokens/total": 6336960, "tokens/train_per_sec_per_gpu": 53.4, "tokens/trainable": 192165} +{"epoch": 0.71875, "grad_norm": 4.068352699279785, "learning_rate": 7.85804646415409e-05, "loss": 0.29197195172309875, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.33907, "step": 184, "tokens/total": 6372192, "tokens/train_per_sec_per_gpu": 64.21, "tokens/trainable": 193251} +{"epoch": 0.72265625, "grad_norm": 8.984768867492676, "learning_rate": 7.833273149760207e-05, "loss": 0.4102378785610199, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.50718, "step": 185, "tokens/total": 6403056, "tokens/train_per_sec_per_gpu": 62.95, "tokens/trainable": 194276} +{"epoch": 0.7265625, "grad_norm": 1.8207406997680664, "learning_rate": 7.808402738346527e-05, "loss": 0.28287965059280396, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.11, "memory/max_allocated (GiB)": 39.11, "ppl": 1.32695, "step": 186, "tokens/total": 6441840, "tokens/train_per_sec_per_gpu": 58.21, "tokens/trainable": 195434} +{"epoch": 0.73046875, "grad_norm": 1.8595383167266846, "learning_rate": 7.783436264872382e-05, "loss": 0.285876989364624, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.33093, "step": 187, "tokens/total": 6478688, "tokens/train_per_sec_per_gpu": 56.14, "tokens/trainable": 196483} +{"epoch": 0.734375, "grad_norm": 1.4113974571228027, "learning_rate": 7.758374768294647e-05, "loss": 0.2661101818084717, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.16, "memory/max_allocated (GiB)": 38.16, "ppl": 1.30488, "step": 188, "tokens/total": 6514624, "tokens/train_per_sec_per_gpu": 60.72, "tokens/trainable": 197578} +{"epoch": 0.73828125, "grad_norm": 1.6220381259918213, "learning_rate": 7.733219291524489e-05, "loss": 0.3296636939048767, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.53, "memory/max_allocated (GiB)": 37.53, "ppl": 1.3905, "step": 189, "tokens/total": 6548048, "tokens/train_per_sec_per_gpu": 64.85, "tokens/trainable": 198635} +{"epoch": 0.7421875, "grad_norm": 1.5284607410430908, "learning_rate": 7.707970881383977e-05, "loss": 0.40542563796043396, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.45, "memory/max_allocated (GiB)": 36.45, "ppl": 1.49994, "step": 190, "tokens/total": 6579760, "tokens/train_per_sec_per_gpu": 71.84, "tokens/trainable": 199697} +{"epoch": 0.74609375, "grad_norm": 1.8781393766403198, "learning_rate": 7.682630588562518e-05, "loss": 0.29098808765411377, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.7, "memory/max_allocated (GiB)": 38.7, "ppl": 1.33775, "step": 191, "tokens/total": 6614800, "tokens/train_per_sec_per_gpu": 57.64, "tokens/trainable": 200758} +{"epoch": 0.75, "grad_norm": 2.5942044258117676, "learning_rate": 7.657199467573129e-05, "loss": 0.29510408639907837, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.34327, "step": 192, "tokens/total": 6648928, "tokens/train_per_sec_per_gpu": 57.34, "tokens/trainable": 201783} +{"epoch": 0.75390625, "grad_norm": 1.743707299232483, "learning_rate": 7.631678576708561e-05, "loss": 0.24321424961090088, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.16, "memory/max_allocated (GiB)": 38.16, "ppl": 1.27534, "step": 193, "tokens/total": 6683328, "tokens/train_per_sec_per_gpu": 67.39, "tokens/trainable": 202877} +{"epoch": 0.7578125, "grad_norm": 1.3846039772033691, "learning_rate": 7.606068977997255e-05, "loss": 0.3071694076061249, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.64, "memory/max_allocated (GiB)": 39.64, "ppl": 1.35957, "step": 194, "tokens/total": 6720928, "tokens/train_per_sec_per_gpu": 66.21, "tokens/trainable": 204053} +{"epoch": 0.76171875, "grad_norm": 4.75140905380249, "learning_rate": 7.580371737159148e-05, "loss": 0.3186873197555542, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.37532, "step": 195, "tokens/total": 6757024, "tokens/train_per_sec_per_gpu": 58.06, "tokens/trainable": 205111} +{"epoch": 0.765625, "grad_norm": 1.5715601444244385, "learning_rate": 7.554587923561324e-05, "loss": 0.28336071968078613, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.99, "memory/max_allocated (GiB)": 38.99, "ppl": 1.32758, "step": 196, "tokens/total": 6791728, "tokens/train_per_sec_per_gpu": 51.83, "tokens/trainable": 206083} +{"epoch": 0.76953125, "grad_norm": 1.5682258605957031, "learning_rate": 7.528718610173511e-05, "loss": 0.28464475274086, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.32929, "step": 197, "tokens/total": 6827072, "tokens/train_per_sec_per_gpu": 63.24, "tokens/trainable": 207177} +{"epoch": 0.7734375, "grad_norm": 1.5674622058868408, "learning_rate": 7.502764873523431e-05, "loss": 0.36589154601097107, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.4418, "step": 198, "tokens/total": 6861120, "tokens/train_per_sec_per_gpu": 62.62, "tokens/trainable": 208214} +{"epoch": 0.77734375, "grad_norm": 1.043484091758728, "learning_rate": 7.476727793652011e-05, "loss": 0.2617405951023102, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.72, "memory/max_allocated (GiB)": 36.72, "ppl": 1.29919, "step": 199, "tokens/total": 6894352, "tokens/train_per_sec_per_gpu": 62.68, "tokens/trainable": 209273} +{"epoch": 0.78125, "grad_norm": 1.4997425079345703, "learning_rate": 7.450608454068415e-05, "loss": 0.2672302722930908, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.30634, "step": 200, "tokens/total": 6928848, "tokens/train_per_sec_per_gpu": 57.9, "tokens/trainable": 210335} +{"epoch": 0.78515625, "grad_norm": 1.83994460105896, "learning_rate": 7.424407941704987e-05, "loss": 0.23084846138954163, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.49, "memory/max_allocated (GiB)": 38.49, "ppl": 1.25967, "step": 201, "tokens/total": 6965504, "tokens/train_per_sec_per_gpu": 67.25, "tokens/trainable": 211480} +{"epoch": 0.7890625, "grad_norm": 1.3494921922683716, "learning_rate": 7.398127346871986e-05, "loss": 0.25240740180015564, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.61, "memory/max_allocated (GiB)": 39.61, "ppl": 1.28712, "step": 202, "tokens/total": 7002448, "tokens/train_per_sec_per_gpu": 55.05, "tokens/trainable": 212523} +{"epoch": 0.79296875, "grad_norm": 2.6964609622955322, "learning_rate": 7.371767763212238e-05, "loss": 0.2722027897834778, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.52, "memory/max_allocated (GiB)": 39.52, "ppl": 1.31285, "step": 203, "tokens/total": 7037824, "tokens/train_per_sec_per_gpu": 62.56, "tokens/trainable": 213574} +{"epoch": 0.796875, "grad_norm": 1.6556199789047241, "learning_rate": 7.345330287655617e-05, "loss": 0.3150217533111572, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.09, "memory/max_allocated (GiB)": 39.09, "ppl": 1.37029, "step": 204, "tokens/total": 7075920, "tokens/train_per_sec_per_gpu": 48.7, "tokens/trainable": 214633} +{"epoch": 0.80078125, "grad_norm": 9.789473533630371, "learning_rate": 7.31881602037339e-05, "loss": 0.34563204646110535, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.69, "memory/max_allocated (GiB)": 36.69, "ppl": 1.41288, "step": 205, "tokens/total": 7107888, "tokens/train_per_sec_per_gpu": 63.59, "tokens/trainable": 215651} +{"epoch": 0.8046875, "grad_norm": 1.983750343322754, "learning_rate": 7.29222606473245e-05, "loss": 0.27007806301116943, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.31007, "step": 206, "tokens/total": 7140720, "tokens/train_per_sec_per_gpu": 61.44, "tokens/trainable": 216669} +{"epoch": 0.80859375, "grad_norm": 3.2563843727111816, "learning_rate": 7.265561527249383e-05, "loss": 0.3942945599555969, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.48334, "step": 207, "tokens/total": 7174720, "tokens/train_per_sec_per_gpu": 63.46, "tokens/trainable": 217727} +{"epoch": 0.8125, "grad_norm": 1.1126151084899902, "learning_rate": 7.238823517544436e-05, "loss": 0.2628099322319031, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.30058, "step": 208, "tokens/total": 7211872, "tokens/train_per_sec_per_gpu": 67.72, "tokens/trainable": 218899} +{"epoch": 0.81640625, "grad_norm": 1.5376079082489014, "learning_rate": 7.212013148295333e-05, "loss": 0.25855889916419983, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.29506, "step": 209, "tokens/total": 7245856, "tokens/train_per_sec_per_gpu": 63.24, "tokens/trainable": 219978} +{"epoch": 0.8203125, "grad_norm": 3.097480297088623, "learning_rate": 7.185131535190975e-05, "loss": 0.2973684072494507, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.06, "memory/max_allocated (GiB)": 37.06, "ppl": 1.34631, "step": 210, "tokens/total": 7278448, "tokens/train_per_sec_per_gpu": 65.98, "tokens/trainable": 221067} +{"epoch": 0.82421875, "grad_norm": 1.1498498916625977, "learning_rate": 7.158179796885005e-05, "loss": 0.2404199093580246, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.27178, "step": 211, "tokens/total": 7313072, "tokens/train_per_sec_per_gpu": 62.16, "tokens/trainable": 222120} +{"epoch": 0.828125, "grad_norm": 1.4445598125457764, "learning_rate": 7.131159054949273e-05, "loss": 0.2797163426876068, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.32275, "step": 212, "tokens/total": 7351456, "tokens/train_per_sec_per_gpu": 54.86, "tokens/trainable": 223159} +{"epoch": 0.83203125, "grad_norm": 1.2356138229370117, "learning_rate": 7.104070433827139e-05, "loss": 0.29981788992881775, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.34961, "step": 213, "tokens/total": 7386880, "tokens/train_per_sec_per_gpu": 52.47, "tokens/trainable": 224113} +{"epoch": 0.8359375, "grad_norm": 1.2562329769134521, "learning_rate": 7.076915060786705e-05, "loss": 0.26322847604751587, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.30112, "step": 214, "tokens/total": 7419696, "tokens/train_per_sec_per_gpu": 64.73, "tokens/trainable": 225176} +{"epoch": 0.83984375, "grad_norm": 1.5657415390014648, "learning_rate": 7.049694065873882e-05, "loss": 0.2721453309059143, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.07, "memory/max_allocated (GiB)": 38.07, "ppl": 1.31278, "step": 215, "tokens/total": 7454352, "tokens/train_per_sec_per_gpu": 56.06, "tokens/trainable": 226219} +{"epoch": 0.84375, "grad_norm": 1.2515895366668701, "learning_rate": 7.022408581865382e-05, "loss": 0.23623207211494446, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.26647, "step": 216, "tokens/total": 7488496, "tokens/train_per_sec_per_gpu": 57.86, "tokens/trainable": 227296} +{"epoch": 0.84765625, "grad_norm": 4.467289924621582, "learning_rate": 6.99505974422157e-05, "loss": 0.23350127041339874, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.26301, "step": 217, "tokens/total": 7522976, "tokens/train_per_sec_per_gpu": 65.63, "tokens/trainable": 228383} +{"epoch": 0.8515625, "grad_norm": 1.226471185684204, "learning_rate": 6.967648691039213e-05, "loss": 0.2672448754310608, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.58, "memory/max_allocated (GiB)": 36.58, "ppl": 1.30636, "step": 218, "tokens/total": 7554848, "tokens/train_per_sec_per_gpu": 67.59, "tokens/trainable": 229432} +{"epoch": 0.85546875, "grad_norm": 1.660046100616455, "learning_rate": 6.940176563004123e-05, "loss": 0.2665237486362457, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.30542, "step": 219, "tokens/total": 7588896, "tokens/train_per_sec_per_gpu": 57.48, "tokens/trainable": 230405} +{"epoch": 0.859375, "grad_norm": 1.5469764471054077, "learning_rate": 6.912644503343682e-05, "loss": 0.26892921328544617, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.30856, "step": 220, "tokens/total": 7627056, "tokens/train_per_sec_per_gpu": 55.59, "tokens/trainable": 231414} +{"epoch": 0.86328125, "grad_norm": 2.24670147895813, "learning_rate": 6.885053657779273e-05, "loss": 0.27872514724731445, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 36.28, "memory/max_allocated (GiB)": 36.28, "ppl": 1.32144, "step": 221, "tokens/total": 7658368, "tokens/train_per_sec_per_gpu": 62.56, "tokens/trainable": 232449} +{"epoch": 0.8671875, "grad_norm": 1.4588860273361206, "learning_rate": 6.857405174478604e-05, "loss": 0.2577178478240967, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.29397, "step": 222, "tokens/total": 7693072, "tokens/train_per_sec_per_gpu": 57.97, "tokens/trainable": 233459} +{"epoch": 0.87109375, "grad_norm": 1.3122183084487915, "learning_rate": 6.82970020400792e-05, "loss": 0.28066107630729675, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.324, "step": 223, "tokens/total": 7730288, "tokens/train_per_sec_per_gpu": 54.2, "tokens/trainable": 234491} +{"epoch": 0.875, "grad_norm": 1.2428213357925415, "learning_rate": 6.801939899284132e-05, "loss": 0.20686869323253632, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.22982, "step": 224, "tokens/total": 7766304, "tokens/train_per_sec_per_gpu": 58.5, "tokens/trainable": 235570} +{"epoch": 0.87890625, "grad_norm": 1.594831109046936, "learning_rate": 6.774125415526827e-05, "loss": 0.21444553136825562, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.23917, "step": 225, "tokens/total": 7799680, "tokens/train_per_sec_per_gpu": 70.85, "tokens/trainable": 236708} +{"epoch": 0.8828125, "grad_norm": 1.1596966981887817, "learning_rate": 6.746257910210214e-05, "loss": 0.25412654876708984, "memory/device_reserved (GiB)": 43.78, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.28933, "step": 226, "tokens/total": 7832912, "tokens/train_per_sec_per_gpu": 61.28, "tokens/trainable": 237735} +{"epoch": 0.88671875, "grad_norm": 1.420740008354187, "learning_rate": 6.718338543014937e-05, "loss": 0.3161396384239197, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 35.66, "memory/max_allocated (GiB)": 35.66, "ppl": 1.37182, "step": 227, "tokens/total": 7862912, "tokens/train_per_sec_per_gpu": 70.02, "tokens/trainable": 238769} +{"epoch": 0.890625, "grad_norm": 0.916044294834137, "learning_rate": 6.69036847577983e-05, "loss": 0.22653917968273163, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.25425, "step": 228, "tokens/total": 7896816, "tokens/train_per_sec_per_gpu": 61.23, "tokens/trainable": 239798} +{"epoch": 0.89453125, "grad_norm": 1.2110626697540283, "learning_rate": 6.662348872453553e-05, "loss": 0.20418605208396912, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.46, "memory/max_allocated (GiB)": 38.46, "ppl": 1.22653, "step": 229, "tokens/total": 7933376, "tokens/train_per_sec_per_gpu": 60.97, "tokens/trainable": 240866} +{"epoch": 0.8984375, "grad_norm": 0.9950777888298035, "learning_rate": 6.63428089904618e-05, "loss": 0.2336106151342392, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.26315, "step": 230, "tokens/total": 7972544, "tokens/train_per_sec_per_gpu": 52.54, "tokens/trainable": 241897} +{"epoch": 0.90234375, "grad_norm": 1.1826196908950806, "learning_rate": 6.60616572358065e-05, "loss": 0.2842276692390442, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.32874, "step": 231, "tokens/total": 8007008, "tokens/train_per_sec_per_gpu": 62.03, "tokens/trainable": 242876} +{"epoch": 0.90625, "grad_norm": 1.0795729160308838, "learning_rate": 6.578004516044172e-05, "loss": 0.2350185513496399, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.26493, "step": 232, "tokens/total": 8042864, "tokens/train_per_sec_per_gpu": 63.03, "tokens/trainable": 243921} +{"epoch": 0.91015625, "grad_norm": 2.363401412963867, "learning_rate": 6.549798448339548e-05, "loss": 0.23657767474651337, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.6, "memory/max_allocated (GiB)": 38.6, "ppl": 1.26691, "step": 233, "tokens/total": 8080272, "tokens/train_per_sec_per_gpu": 60.39, "tokens/trainable": 244977} +{"epoch": 0.9140625, "grad_norm": 1.6960421800613403, "learning_rate": 6.521548694236384e-05, "loss": 0.2605664134025574, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.29766, "step": 234, "tokens/total": 8117584, "tokens/train_per_sec_per_gpu": 55.0, "tokens/trainable": 246030} +{"epoch": 0.91796875, "grad_norm": 1.279126763343811, "learning_rate": 6.493256429322259e-05, "loss": 0.24359136819839478, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.27582, "step": 235, "tokens/total": 8153712, "tokens/train_per_sec_per_gpu": 55.62, "tokens/trainable": 247057} +{"epoch": 0.921875, "grad_norm": 3.107456922531128, "learning_rate": 6.464922830953799e-05, "loss": 0.26044026017189026, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.2975, "step": 236, "tokens/total": 8192928, "tokens/train_per_sec_per_gpu": 53.12, "tokens/trainable": 248105} +{"epoch": 0.92578125, "grad_norm": 1.2698794603347778, "learning_rate": 6.436549078207688e-05, "loss": 0.24247226119041443, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.21, "memory/max_allocated (GiB)": 38.21, "ppl": 1.2744, "step": 237, "tokens/total": 8226016, "tokens/train_per_sec_per_gpu": 62.36, "tokens/trainable": 249105} +{"epoch": 0.9296875, "grad_norm": 1.0101531744003296, "learning_rate": 6.408136351831592e-05, "loss": 0.23605960607528687, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 37.15, "memory/max_allocated (GiB)": 37.15, "ppl": 1.26625, "step": 238, "tokens/total": 8258384, "tokens/train_per_sec_per_gpu": 60.3, "tokens/trainable": 250102} +{"epoch": 0.93359375, "grad_norm": 2.958418607711792, "learning_rate": 6.379685834195036e-05, "loss": 0.23336079716682434, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.26284, "step": 239, "tokens/total": 8297888, "tokens/train_per_sec_per_gpu": 54.45, "tokens/trainable": 251272} +{"epoch": 0.9375, "grad_norm": 18.5676212310791, "learning_rate": 6.351198709240186e-05, "loss": 0.2514111399650574, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.61, "memory/max_allocated (GiB)": 39.61, "ppl": 1.28584, "step": 240, "tokens/total": 8338016, "tokens/train_per_sec_per_gpu": 54.08, "tokens/trainable": 252361} +{"epoch": 0.94140625, "grad_norm": 2.078204393386841, "learning_rate": 6.32267616243259e-05, "loss": 0.2571539282798767, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.29324, "step": 241, "tokens/total": 8370608, "tokens/train_per_sec_per_gpu": 63.69, "tokens/trainable": 253391} +{"epoch": 0.9453125, "grad_norm": 1.547778844833374, "learning_rate": 6.294119380711849e-05, "loss": 0.28525179624557495, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.3301, "step": 242, "tokens/total": 8405040, "tokens/train_per_sec_per_gpu": 53.44, "tokens/trainable": 254355} +{"epoch": 0.94921875, "grad_norm": 1.3277722597122192, "learning_rate": 6.265529552442209e-05, "loss": 0.2573148012161255, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.29345, "step": 243, "tokens/total": 8440400, "tokens/train_per_sec_per_gpu": 63.1, "tokens/trainable": 255455} +{"epoch": 0.953125, "grad_norm": 1.179445505142212, "learning_rate": 6.236907867363127e-05, "loss": 0.23238898813724518, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.14, "memory/max_allocated (GiB)": 38.14, "ppl": 1.26161, "step": 244, "tokens/total": 8474736, "tokens/train_per_sec_per_gpu": 63.43, "tokens/trainable": 256489} +{"epoch": 0.95703125, "grad_norm": 1.010067343711853, "learning_rate": 6.208255516539749e-05, "loss": 0.2132769525051117, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 36.34, "memory/max_allocated (GiB)": 36.34, "ppl": 1.23773, "step": 245, "tokens/total": 8506944, "tokens/train_per_sec_per_gpu": 60.62, "tokens/trainable": 257546} +{"epoch": 0.9609375, "grad_norm": 1.9483128786087036, "learning_rate": 6.179573692313344e-05, "loss": 0.242621511220932, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 1.27459, "step": 246, "tokens/total": 8544064, "tokens/train_per_sec_per_gpu": 57.92, "tokens/trainable": 258572} +{"epoch": 0.96484375, "grad_norm": 5.000629901885986, "learning_rate": 6.150863588251694e-05, "loss": 0.2504868805408478, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.28465, "step": 247, "tokens/total": 8584336, "tokens/train_per_sec_per_gpu": 50.32, "tokens/trainable": 259638} +{"epoch": 0.96875, "grad_norm": 1.266863226890564, "learning_rate": 6.122126399099419e-05, "loss": 0.2071881890296936, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.78, "memory/max_allocated (GiB)": 39.78, "ppl": 1.23021, "step": 248, "tokens/total": 8624304, "tokens/train_per_sec_per_gpu": 66.47, "tokens/trainable": 260894} +{"epoch": 0.97265625, "grad_norm": 1.6223421096801758, "learning_rate": 6.0933633207282615e-05, "loss": 0.264302134513855, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.30252, "step": 249, "tokens/total": 8660672, "tokens/train_per_sec_per_gpu": 54.91, "tokens/trainable": 261961} +{"epoch": 0.9765625, "grad_norm": 1.1754522323608398, "learning_rate": 6.064575550087316e-05, "loss": 0.2556946873664856, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.63, "memory/max_allocated (GiB)": 39.63, "ppl": 1.29136, "step": 250, "tokens/total": 8697680, "tokens/train_per_sec_per_gpu": 61.09, "tokens/trainable": 263037} +{"epoch": 0.98046875, "grad_norm": 1.084790825843811, "learning_rate": 6.0357642851532245e-05, "loss": 0.2168481945991516, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.24216, "step": 251, "tokens/total": 8734112, "tokens/train_per_sec_per_gpu": 54.93, "tokens/trainable": 264105} +{"epoch": 0.984375, "grad_norm": 1.2090396881103516, "learning_rate": 6.0069307248803294e-05, "loss": 0.29137200117111206, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.33826, "step": 252, "tokens/total": 8769440, "tokens/train_per_sec_per_gpu": 50.52, "tokens/trainable": 265091} +{"epoch": 0.98828125, "grad_norm": 1.1875697374343872, "learning_rate": 5.9780760691507635e-05, "loss": 0.26334214210510254, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 36.25, "memory/max_allocated (GiB)": 36.25, "ppl": 1.30127, "step": 253, "tokens/total": 8800688, "tokens/train_per_sec_per_gpu": 64.02, "tokens/trainable": 266053} +{"epoch": 0.9921875, "grad_norm": 16.11334800720215, "learning_rate": 5.9492015187245334e-05, "loss": 0.2205744832754135, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 38.72, "memory/max_allocated (GiB)": 38.72, "ppl": 1.24679, "step": 254, "tokens/total": 8838272, "tokens/train_per_sec_per_gpu": 54.75, "tokens/trainable": 267018} +{"epoch": 0.99609375, "grad_norm": 1.045760989189148, "learning_rate": 5.920308275189541e-05, "loss": 0.2262175977230072, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 36.6, "memory/max_allocated (GiB)": 36.6, "ppl": 1.25385, "step": 255, "tokens/total": 8869952, "tokens/train_per_sec_per_gpu": 65.64, "tokens/trainable": 268040} +{"epoch": 1.0, "grad_norm": 0.7919358015060425, "learning_rate": 5.8913975409115874e-05, "loss": 0.20814141631126404, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.23139, "step": 256, "tokens/total": 8903312, "tokens/train_per_sec_per_gpu": 62.45, "tokens/trainable": 269099} +{"epoch": 1.00390625, "grad_norm": 0.8366561532020569, "learning_rate": 5.8624705189843395e-05, "loss": 0.18596124649047852, "memory/device_reserved (GiB)": 43.79, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.20438, "step": 257, "tokens/total": 8940768, "tokens/train_per_sec_per_gpu": 55.47, "tokens/trainable": 270206} +{"epoch": 1.0078125, "grad_norm": 1.026794195175171, "learning_rate": 5.833528413179249e-05, "loss": 0.19512814283370972, "memory/device_reserved (GiB)": 40.67, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.21547, "step": 258, "tokens/total": 8977392, "tokens/train_per_sec_per_gpu": 58.64, "tokens/trainable": 271305} +{"epoch": 1.01171875, "grad_norm": 1.4086321592330933, "learning_rate": 5.80457242789548e-05, "loss": 0.1796715408563614, "memory/device_reserved (GiB)": 42.74, "memory/max_active (GiB)": 39.65, "memory/max_allocated (GiB)": 39.65, "ppl": 1.19682, "step": 259, "tokens/total": 9017568, "tokens/train_per_sec_per_gpu": 57.35, "tokens/trainable": 272486} +{"epoch": 1.015625, "grad_norm": 1.4933545589447021, "learning_rate": 5.77560376810977e-05, "loss": 0.17971670627593994, "memory/device_reserved (GiB)": 42.75, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.19688, "step": 260, "tokens/total": 9050928, "tokens/train_per_sec_per_gpu": 70.99, "tokens/trainable": 273635} +{"epoch": 1.01953125, "grad_norm": 1.7674061059951782, "learning_rate": 5.7466236393263005e-05, "loss": 0.21022991836071014, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.23396, "step": 261, "tokens/total": 9085552, "tokens/train_per_sec_per_gpu": 63.56, "tokens/trainable": 274678} +{"epoch": 1.0234375, "grad_norm": 1.4235285520553589, "learning_rate": 5.717633247526522e-05, "loss": 0.22653651237487793, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 36.74, "memory/max_allocated (GiB)": 36.74, "ppl": 1.25425, "step": 262, "tokens/total": 9116240, "tokens/train_per_sec_per_gpu": 60.15, "tokens/trainable": 275631} +{"epoch": 1.02734375, "grad_norm": 2.4623475074768066, "learning_rate": 5.688633799118971e-05, "loss": 0.25216415524482727, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.28681, "step": 263, "tokens/total": 9148688, "tokens/train_per_sec_per_gpu": 55.82, "tokens/trainable": 276571} +{"epoch": 1.03125, "grad_norm": 0.9971867203712463, "learning_rate": 5.659626500889066e-05, "loss": 0.17057937383651733, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.09, "memory/max_allocated (GiB)": 37.09, "ppl": 1.18599, "step": 264, "tokens/total": 9182704, "tokens/train_per_sec_per_gpu": 64.73, "tokens/trainable": 277697} +{"epoch": 1.03515625, "grad_norm": 1.0917084217071533, "learning_rate": 5.6306125599488905e-05, "loss": 0.202936053276062, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 36.88, "memory/max_allocated (GiB)": 36.88, "ppl": 1.22499, "step": 265, "tokens/total": 9213552, "tokens/train_per_sec_per_gpu": 69.68, "tokens/trainable": 278717} +{"epoch": 1.0390625, "grad_norm": 1.0945912599563599, "learning_rate": 5.601593183686955e-05, "loss": 0.1657162755727768, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.18024, "step": 266, "tokens/total": 9250768, "tokens/train_per_sec_per_gpu": 59.98, "tokens/trainable": 279836} +{"epoch": 1.04296875, "grad_norm": 0.8574339151382446, "learning_rate": 5.572569579717961e-05, "loss": 0.18849673867225647, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.20743, "step": 267, "tokens/total": 9286176, "tokens/train_per_sec_per_gpu": 64.25, "tokens/trainable": 280948} +{"epoch": 1.046875, "grad_norm": 1.0759631395339966, "learning_rate": 5.543542955832538e-05, "loss": 0.18768426775932312, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.20645, "step": 268, "tokens/total": 9326128, "tokens/train_per_sec_per_gpu": 52.64, "tokens/trainable": 282031} +{"epoch": 1.05078125, "grad_norm": 2.8513147830963135, "learning_rate": 5.514514519946986e-05, "loss": 0.1891578733921051, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.57, "memory/max_allocated (GiB)": 39.57, "ppl": 1.20823, "step": 269, "tokens/total": 9366544, "tokens/train_per_sec_per_gpu": 56.96, "tokens/trainable": 283178} +{"epoch": 1.0546875, "grad_norm": 1.3901197910308838, "learning_rate": 5.485485480053015e-05, "loss": 0.22051061689853668, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.52, "memory/max_allocated (GiB)": 39.52, "ppl": 1.24671, "step": 270, "tokens/total": 9404928, "tokens/train_per_sec_per_gpu": 49.43, "tokens/trainable": 284180} +{"epoch": 1.05859375, "grad_norm": 1.185436725616455, "learning_rate": 5.4564570441674645e-05, "loss": 0.20481351017951965, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.2273, "step": 271, "tokens/total": 9438592, "tokens/train_per_sec_per_gpu": 67.61, "tokens/trainable": 285227} +{"epoch": 1.0625, "grad_norm": 1.172336220741272, "learning_rate": 5.42743042028204e-05, "loss": 0.21419653296470642, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.83, "memory/max_allocated (GiB)": 37.83, "ppl": 1.23887, "step": 272, "tokens/total": 9474144, "tokens/train_per_sec_per_gpu": 48.18, "tokens/trainable": 286192} +{"epoch": 1.06640625, "grad_norm": 1.0024352073669434, "learning_rate": 5.3984068163130464e-05, "loss": 0.1755952537059784, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.19196, "step": 273, "tokens/total": 9507904, "tokens/train_per_sec_per_gpu": 69.31, "tokens/trainable": 287369} +{"epoch": 1.0703125, "grad_norm": 1.2823853492736816, "learning_rate": 5.369387440051111e-05, "loss": 0.20282281935214996, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.18, "memory/max_allocated (GiB)": 37.18, "ppl": 1.22486, "step": 274, "tokens/total": 9541616, "tokens/train_per_sec_per_gpu": 58.9, "tokens/trainable": 288371} +{"epoch": 1.07421875, "grad_norm": 2.1487300395965576, "learning_rate": 5.340373499110935e-05, "loss": 0.20695683360099792, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.22993, "step": 275, "tokens/total": 9577424, "tokens/train_per_sec_per_gpu": 59.71, "tokens/trainable": 289456} +{"epoch": 1.078125, "grad_norm": 3.0479297637939453, "learning_rate": 5.3113662008810304e-05, "loss": 0.17157617211341858, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.18717, "step": 276, "tokens/total": 9617760, "tokens/train_per_sec_per_gpu": 50.41, "tokens/trainable": 290516} +{"epoch": 1.08203125, "grad_norm": 0.8651800155639648, "learning_rate": 5.282366752473479e-05, "loss": 0.18016508221626282, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.29, "memory/max_allocated (GiB)": 38.29, "ppl": 1.19742, "step": 277, "tokens/total": 9655008, "tokens/train_per_sec_per_gpu": 60.19, "tokens/trainable": 291636} +{"epoch": 1.0859375, "grad_norm": 1.258041262626648, "learning_rate": 5.2533763606737005e-05, "loss": 0.25224247574806213, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.28, "memory/max_allocated (GiB)": 39.28, "ppl": 1.28691, "step": 278, "tokens/total": 9694896, "tokens/train_per_sec_per_gpu": 49.86, "tokens/trainable": 292677} +{"epoch": 1.08984375, "grad_norm": 1.355000376701355, "learning_rate": 5.224396231890232e-05, "loss": 0.1853482723236084, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.23, "memory/max_allocated (GiB)": 37.23, "ppl": 1.20364, "step": 279, "tokens/total": 9726672, "tokens/train_per_sec_per_gpu": 67.95, "tokens/trainable": 293800} +{"epoch": 1.09375, "grad_norm": 2.3430726528167725, "learning_rate": 5.195427572104522e-05, "loss": 0.21335217356681824, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.23782, "step": 280, "tokens/total": 9761136, "tokens/train_per_sec_per_gpu": 60.84, "tokens/trainable": 294843} +{"epoch": 1.09765625, "grad_norm": 1.6529539823532104, "learning_rate": 5.166471586820751e-05, "loss": 0.1985906958580017, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.21968, "step": 281, "tokens/total": 9797888, "tokens/train_per_sec_per_gpu": 61.22, "tokens/trainable": 295941} +{"epoch": 1.1015625, "grad_norm": 0.8027838468551636, "learning_rate": 5.1375294810156615e-05, "loss": 0.1855439841747284, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.20387, "step": 282, "tokens/total": 9832528, "tokens/train_per_sec_per_gpu": 59.81, "tokens/trainable": 296963} +{"epoch": 1.10546875, "grad_norm": 1.0903408527374268, "learning_rate": 5.1086024590884144e-05, "loss": 0.20216841995716095, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.28, "memory/max_allocated (GiB)": 39.28, "ppl": 1.22405, "step": 283, "tokens/total": 9870784, "tokens/train_per_sec_per_gpu": 52.85, "tokens/trainable": 297964} +{"epoch": 1.109375, "grad_norm": 0.9085438847541809, "learning_rate": 5.079691724810461e-05, "loss": 0.18467067182064056, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.20282, "step": 284, "tokens/total": 9906656, "tokens/train_per_sec_per_gpu": 49.31, "tokens/trainable": 298933} +{"epoch": 1.11328125, "grad_norm": 11.072591781616211, "learning_rate": 5.0507984812754684e-05, "loss": 0.19214460253715515, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.21185, "step": 285, "tokens/total": 9940208, "tokens/train_per_sec_per_gpu": 59.79, "tokens/trainable": 299980} +{"epoch": 1.1171875, "grad_norm": 0.9860714077949524, "learning_rate": 5.021923930849237e-05, "loss": 0.195688396692276, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.21615, "step": 286, "tokens/total": 9974096, "tokens/train_per_sec_per_gpu": 56.85, "tokens/trainable": 300993} +{"epoch": 1.12109375, "grad_norm": 0.6943675875663757, "learning_rate": 4.99306927511967e-05, "loss": 0.18487140536308289, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.20306, "step": 287, "tokens/total": 10007568, "tokens/train_per_sec_per_gpu": 62.39, "tokens/trainable": 302000} +{"epoch": 1.125, "grad_norm": 2.7749617099761963, "learning_rate": 4.964235714846775e-05, "loss": 0.21530434489250183, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 36.44, "memory/max_allocated (GiB)": 36.44, "ppl": 1.24024, "step": 288, "tokens/total": 10039280, "tokens/train_per_sec_per_gpu": 66.48, "tokens/trainable": 302994} +{"epoch": 1.12890625, "grad_norm": 1.2119228839874268, "learning_rate": 4.9354244499126866e-05, "loss": 0.22073781490325928, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 36.23, "memory/max_allocated (GiB)": 36.23, "ppl": 1.247, "step": 289, "tokens/total": 10070944, "tokens/train_per_sec_per_gpu": 63.18, "tokens/trainable": 304026} +{"epoch": 1.1328125, "grad_norm": 0.9375731348991394, "learning_rate": 4.90663667927174e-05, "loss": 0.19277501106262207, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.21261, "step": 290, "tokens/total": 10103776, "tokens/train_per_sec_per_gpu": 71.27, "tokens/trainable": 305053} +{"epoch": 1.13671875, "grad_norm": 1.1479817628860474, "learning_rate": 4.877873600900581e-05, "loss": 0.1767859160900116, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.07, "memory/max_allocated (GiB)": 38.07, "ppl": 1.19338, "step": 291, "tokens/total": 10139888, "tokens/train_per_sec_per_gpu": 55.79, "tokens/trainable": 306089} +{"epoch": 1.140625, "grad_norm": 1.4056569337844849, "learning_rate": 4.849136411748306e-05, "loss": 0.16973409056663513, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.04, "memory/max_allocated (GiB)": 39.04, "ppl": 1.18499, "step": 292, "tokens/total": 10177936, "tokens/train_per_sec_per_gpu": 53.8, "tokens/trainable": 307146} +{"epoch": 1.14453125, "grad_norm": 0.7692112326622009, "learning_rate": 4.8204263076866574e-05, "loss": 0.18244579434394836, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.20015, "step": 293, "tokens/total": 10213856, "tokens/train_per_sec_per_gpu": 57.69, "tokens/trainable": 308208} +{"epoch": 1.1484375, "grad_norm": 4.251461505889893, "learning_rate": 4.791744483460251e-05, "loss": 0.2296704649925232, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.74, "memory/max_allocated (GiB)": 37.74, "ppl": 1.25819, "step": 294, "tokens/total": 10248352, "tokens/train_per_sec_per_gpu": 51.84, "tokens/trainable": 309202} +{"epoch": 1.15234375, "grad_norm": 1.0474848747253418, "learning_rate": 4.7630921326368736e-05, "loss": 0.19188854098320007, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.21154, "step": 295, "tokens/total": 10287072, "tokens/train_per_sec_per_gpu": 50.12, "tokens/trainable": 310184} +{"epoch": 1.15625, "grad_norm": 0.9002067446708679, "learning_rate": 4.7344704475577916e-05, "loss": 0.18091103434562683, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 35.7, "memory/max_allocated (GiB)": 35.7, "ppl": 1.19831, "step": 296, "tokens/total": 10316640, "tokens/train_per_sec_per_gpu": 68.54, "tokens/trainable": 311210} +{"epoch": 1.16015625, "grad_norm": 1.534852385520935, "learning_rate": 4.705880619288153e-05, "loss": 0.20696693658828735, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.37, "memory/max_allocated (GiB)": 38.37, "ppl": 1.22994, "step": 297, "tokens/total": 10352864, "tokens/train_per_sec_per_gpu": 53.11, "tokens/trainable": 312195} +{"epoch": 1.1640625, "grad_norm": 0.8968140482902527, "learning_rate": 4.677323837567412e-05, "loss": 0.17951714992523193, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.19664, "step": 298, "tokens/total": 10385696, "tokens/train_per_sec_per_gpu": 64.75, "tokens/trainable": 313251} +{"epoch": 1.16796875, "grad_norm": 1.1017587184906006, "learning_rate": 4.6488012907598146e-05, "loss": 0.20543774962425232, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 36.4, "memory/max_allocated (GiB)": 36.4, "ppl": 1.22806, "step": 299, "tokens/total": 10417904, "tokens/train_per_sec_per_gpu": 57.52, "tokens/trainable": 314206} +{"epoch": 1.171875, "grad_norm": 0.8907317519187927, "learning_rate": 4.620314165804964e-05, "loss": 0.16755330562591553, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.18241, "step": 300, "tokens/total": 10453728, "tokens/train_per_sec_per_gpu": 58.67, "tokens/trainable": 315302} +{"epoch": 1.17578125, "grad_norm": 1.6740291118621826, "learning_rate": 4.591863648168407e-05, "loss": 0.17964404821395874, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.19679, "step": 301, "tokens/total": 10488288, "tokens/train_per_sec_per_gpu": 59.43, "tokens/trainable": 316419} +{"epoch": 1.1796875, "grad_norm": 0.8467761278152466, "learning_rate": 4.5634509217923135e-05, "loss": 0.18975698947906494, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.96, "memory/max_allocated (GiB)": 37.96, "ppl": 1.20896, "step": 302, "tokens/total": 10521824, "tokens/train_per_sec_per_gpu": 61.08, "tokens/trainable": 317457} +{"epoch": 1.18359375, "grad_norm": 1.406029462814331, "learning_rate": 4.535077169046201e-05, "loss": 0.21905457973480225, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.03, "memory/max_allocated (GiB)": 38.03, "ppl": 1.2449, "step": 303, "tokens/total": 10558432, "tokens/train_per_sec_per_gpu": 57.1, "tokens/trainable": 318487} +{"epoch": 1.1875, "grad_norm": 2.0997066497802734, "learning_rate": 4.506743570677743e-05, "loss": 0.22649812698364258, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 36.35, "memory/max_allocated (GiB)": 36.35, "ppl": 1.2542, "step": 304, "tokens/total": 10588752, "tokens/train_per_sec_per_gpu": 72.08, "tokens/trainable": 319495} +{"epoch": 1.19140625, "grad_norm": 2.09303879737854, "learning_rate": 4.478451305763618e-05, "loss": 0.22999179363250732, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.25859, "step": 305, "tokens/total": 10622336, "tokens/train_per_sec_per_gpu": 64.14, "tokens/trainable": 320533} +{"epoch": 1.1953125, "grad_norm": 1.1098363399505615, "learning_rate": 4.450201551660454e-05, "loss": 0.19788196682929993, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 36.41, "memory/max_allocated (GiB)": 36.41, "ppl": 1.21882, "step": 306, "tokens/total": 10653520, "tokens/train_per_sec_per_gpu": 63.45, "tokens/trainable": 321538} +{"epoch": 1.19921875, "grad_norm": 1.1508101224899292, "learning_rate": 4.4219954839558276e-05, "loss": 0.21139946579933167, "memory/device_reserved (GiB)": 42.76, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.23541, "step": 307, "tokens/total": 10688144, "tokens/train_per_sec_per_gpu": 53.86, "tokens/trainable": 322550} +{"epoch": 1.203125, "grad_norm": 0.9898407459259033, "learning_rate": 4.393834276419352e-05, "loss": 0.19917482137680054, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 35.7, "memory/max_allocated (GiB)": 35.7, "ppl": 1.2204, "step": 308, "tokens/total": 10718032, "tokens/train_per_sec_per_gpu": 72.38, "tokens/trainable": 323588} +{"epoch": 1.20703125, "grad_norm": 1.080328345298767, "learning_rate": 4.36571910095382e-05, "loss": 0.24009756743907928, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.27137, "step": 309, "tokens/total": 10756464, "tokens/train_per_sec_per_gpu": 55.2, "tokens/trainable": 324662} +{"epoch": 1.2109375, "grad_norm": 1.1950353384017944, "learning_rate": 4.337651127546448e-05, "loss": 0.22875307500362396, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.25703, "step": 310, "tokens/total": 10792064, "tokens/train_per_sec_per_gpu": 63.55, "tokens/trainable": 325723} +{"epoch": 1.21484375, "grad_norm": 1.2674778699874878, "learning_rate": 4.3096315242201736e-05, "loss": 0.2636401057243347, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 36.51, "memory/max_allocated (GiB)": 36.51, "ppl": 1.30166, "step": 311, "tokens/total": 10824800, "tokens/train_per_sec_per_gpu": 59.06, "tokens/trainable": 326771} +{"epoch": 1.21875, "grad_norm": 1.225868582725525, "learning_rate": 4.2816614569850635e-05, "loss": 0.19644495844841003, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 38.66, "memory/max_allocated (GiB)": 38.66, "ppl": 1.21707, "step": 312, "tokens/total": 10860576, "tokens/train_per_sec_per_gpu": 58.31, "tokens/trainable": 327771} +{"epoch": 1.22265625, "grad_norm": 1.088719367980957, "learning_rate": 4.2537420897897864e-05, "loss": 0.21290838718414307, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 38.72, "memory/max_allocated (GiB)": 38.72, "ppl": 1.23727, "step": 313, "tokens/total": 10896224, "tokens/train_per_sec_per_gpu": 53.82, "tokens/trainable": 328807} +{"epoch": 1.2265625, "grad_norm": 1.2311557531356812, "learning_rate": 4.225874584473174e-05, "loss": 0.2182517945766449, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.2439, "step": 314, "tokens/total": 10928768, "tokens/train_per_sec_per_gpu": 64.0, "tokens/trainable": 329821} +{"epoch": 1.23046875, "grad_norm": 0.7491008043289185, "learning_rate": 4.19806010071587e-05, "loss": 0.17919045686721802, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.19625, "step": 315, "tokens/total": 10966848, "tokens/train_per_sec_per_gpu": 55.02, "tokens/trainable": 330917} +{"epoch": 1.234375, "grad_norm": 0.9948517680168152, "learning_rate": 4.170299795992081e-05, "loss": 0.22027146816253662, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 36.88, "memory/max_allocated (GiB)": 36.88, "ppl": 1.24642, "step": 316, "tokens/total": 11000416, "tokens/train_per_sec_per_gpu": 60.4, "tokens/trainable": 331979} +{"epoch": 1.23828125, "grad_norm": 0.7559939622879028, "learning_rate": 4.142594825521398e-05, "loss": 0.18330979347229004, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.20119, "step": 317, "tokens/total": 11039520, "tokens/train_per_sec_per_gpu": 51.39, "tokens/trainable": 333036} +{"epoch": 1.2421875, "grad_norm": 15.57944107055664, "learning_rate": 4.114946342220728e-05, "loss": 0.18756264448165894, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.20631, "step": 318, "tokens/total": 11076096, "tokens/train_per_sec_per_gpu": 58.35, "tokens/trainable": 334136} +{"epoch": 1.24609375, "grad_norm": 0.6292304396629333, "learning_rate": 4.087355496656321e-05, "loss": 0.17579813301563263, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.1922, "step": 319, "tokens/total": 11109728, "tokens/train_per_sec_per_gpu": 69.31, "tokens/trainable": 335262} +{"epoch": 1.25, "grad_norm": 0.7573001980781555, "learning_rate": 4.05982343699588e-05, "loss": 0.1817469447851181, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.19931, "step": 320, "tokens/total": 11148976, "tokens/train_per_sec_per_gpu": 59.53, "tokens/trainable": 336421} +{"epoch": 1.25390625, "grad_norm": 0.8439776301383972, "learning_rate": 4.0323513089607876e-05, "loss": 0.1882067769765854, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.20708, "step": 321, "tokens/total": 11183888, "tokens/train_per_sec_per_gpu": 57.72, "tokens/trainable": 337517} +{"epoch": 1.2578125, "grad_norm": 1.0337157249450684, "learning_rate": 4.004940255778431e-05, "loss": 0.18017208576202393, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 39.63, "memory/max_allocated (GiB)": 39.63, "ppl": 1.19742, "step": 322, "tokens/total": 11222608, "tokens/train_per_sec_per_gpu": 56.72, "tokens/trainable": 338574} +{"epoch": 1.26171875, "grad_norm": 0.606879472732544, "learning_rate": 3.977591418134619e-05, "loss": 0.1829857975244522, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.2008, "step": 323, "tokens/total": 11257568, "tokens/train_per_sec_per_gpu": 62.49, "tokens/trainable": 339635} +{"epoch": 1.265625, "grad_norm": 2.1646857261657715, "learning_rate": 3.95030593412612e-05, "loss": 0.2159503996372223, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.24104, "step": 324, "tokens/total": 11293472, "tokens/train_per_sec_per_gpu": 57.36, "tokens/trainable": 340651} +{"epoch": 1.26953125, "grad_norm": 1.2576650381088257, "learning_rate": 3.923084939213296e-05, "loss": 0.2508283853530884, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 35.75, "memory/max_allocated (GiB)": 35.75, "ppl": 1.28509, "step": 325, "tokens/total": 11324048, "tokens/train_per_sec_per_gpu": 68.32, "tokens/trainable": 341702} +{"epoch": 1.2734375, "grad_norm": 1.1154669523239136, "learning_rate": 3.895929566172861e-05, "loss": 0.16333332657814026, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.17743, "step": 326, "tokens/total": 11360064, "tokens/train_per_sec_per_gpu": 60.34, "tokens/trainable": 342806} +{"epoch": 1.27734375, "grad_norm": 0.9262186884880066, "learning_rate": 3.868840945050728e-05, "loss": 0.1616458296775818, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 39.19, "memory/max_allocated (GiB)": 39.19, "ppl": 1.17544, "step": 327, "tokens/total": 11399008, "tokens/train_per_sec_per_gpu": 56.16, "tokens/trainable": 343895} +{"epoch": 1.28125, "grad_norm": 0.9300323724746704, "learning_rate": 3.841820203114995e-05, "loss": 0.19175589084625244, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 36.78, "memory/max_allocated (GiB)": 36.78, "ppl": 1.21137, "step": 328, "tokens/total": 11430752, "tokens/train_per_sec_per_gpu": 67.56, "tokens/trainable": 344994} +{"epoch": 1.28515625, "grad_norm": 1.3427897691726685, "learning_rate": 3.814868464809027e-05, "loss": 0.1796874701976776, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 36.77, "memory/max_allocated (GiB)": 36.77, "ppl": 1.19684, "step": 329, "tokens/total": 11462688, "tokens/train_per_sec_per_gpu": 61.44, "tokens/trainable": 345990} +{"epoch": 1.2890625, "grad_norm": 1.1356314420700073, "learning_rate": 3.787986851704667e-05, "loss": 0.22746828198432922, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.25542, "step": 330, "tokens/total": 11496240, "tokens/train_per_sec_per_gpu": 67.48, "tokens/trainable": 347071} +{"epoch": 1.29296875, "grad_norm": 0.8881457448005676, "learning_rate": 3.7611764824555654e-05, "loss": 0.16204075515270233, "memory/device_reserved (GiB)": 42.77, "memory/max_active (GiB)": 37.84, "memory/max_allocated (GiB)": 37.84, "ppl": 1.17591, "step": 331, "tokens/total": 11527968, "tokens/train_per_sec_per_gpu": 76.26, "tokens/trainable": 348224} +{"epoch": 1.296875, "grad_norm": 1.5836899280548096, "learning_rate": 3.734438472750619e-05, "loss": 0.1985495239496231, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.21963, "step": 332, "tokens/total": 11567936, "tokens/train_per_sec_per_gpu": 53.5, "tokens/trainable": 349352} +{"epoch": 1.30078125, "grad_norm": 0.9414400458335876, "learning_rate": 3.707773935267552e-05, "loss": 0.18220938742160797, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.19987, "step": 333, "tokens/total": 11604992, "tokens/train_per_sec_per_gpu": 57.97, "tokens/trainable": 350479} +{"epoch": 1.3046875, "grad_norm": 1.7557284832000732, "learning_rate": 3.68118397962661e-05, "loss": 0.1994556337594986, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 38.49, "memory/max_allocated (GiB)": 38.49, "ppl": 1.22074, "step": 334, "tokens/total": 11639744, "tokens/train_per_sec_per_gpu": 55.87, "tokens/trainable": 351441} +{"epoch": 1.30859375, "grad_norm": 0.6129204034805298, "learning_rate": 3.654669712344384e-05, "loss": 0.1598445177078247, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 36.72, "memory/max_allocated (GiB)": 36.72, "ppl": 1.17333, "step": 335, "tokens/total": 11670944, "tokens/train_per_sec_per_gpu": 74.98, "tokens/trainable": 352497} +{"epoch": 1.3125, "grad_norm": 0.7944165468215942, "learning_rate": 3.628232236787763e-05, "loss": 0.16914255917072296, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 37.91, "memory/max_allocated (GiB)": 37.91, "ppl": 1.18429, "step": 336, "tokens/total": 11706544, "tokens/train_per_sec_per_gpu": 56.24, "tokens/trainable": 353610} +{"epoch": 1.31640625, "grad_norm": 1.1813009977340698, "learning_rate": 3.6018726531280144e-05, "loss": 0.18417993187904358, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.20223, "step": 337, "tokens/total": 11743952, "tokens/train_per_sec_per_gpu": 60.42, "tokens/trainable": 354725} +{"epoch": 1.3203125, "grad_norm": 0.9043789505958557, "learning_rate": 3.575592058295017e-05, "loss": 0.21998165547847748, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.24605, "step": 338, "tokens/total": 11777936, "tokens/train_per_sec_per_gpu": 63.81, "tokens/trainable": 355802} +{"epoch": 1.32421875, "grad_norm": 1.077480435371399, "learning_rate": 3.549391545931585e-05, "loss": 0.1778983324766159, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 38.93, "memory/max_allocated (GiB)": 38.93, "ppl": 1.1947, "step": 339, "tokens/total": 11812416, "tokens/train_per_sec_per_gpu": 62.46, "tokens/trainable": 356852} +{"epoch": 1.328125, "grad_norm": 1.0540121793746948, "learning_rate": 3.5232722063479914e-05, "loss": 0.18346677720546722, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.20138, "step": 340, "tokens/total": 11846704, "tokens/train_per_sec_per_gpu": 66.62, "tokens/trainable": 357927} +{"epoch": 1.33203125, "grad_norm": 0.9317489862442017, "learning_rate": 3.49723512647657e-05, "loss": 0.16997404396533966, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.18527, "step": 341, "tokens/total": 11885968, "tokens/train_per_sec_per_gpu": 54.2, "tokens/trainable": 359043} +{"epoch": 1.3359375, "grad_norm": 1.077460527420044, "learning_rate": 3.471281389826491e-05, "loss": 0.18902048468589783, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.20807, "step": 342, "tokens/total": 11917024, "tokens/train_per_sec_per_gpu": 63.42, "tokens/trainable": 359996} +{"epoch": 1.33984375, "grad_norm": 0.7893366813659668, "learning_rate": 3.4454120764386764e-05, "loss": 0.16768348217010498, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.18256, "step": 343, "tokens/total": 11955648, "tokens/train_per_sec_per_gpu": 56.7, "tokens/trainable": 361076} +{"epoch": 1.34375, "grad_norm": 0.6205090880393982, "learning_rate": 3.4196282628408526e-05, "loss": 0.18204015493392944, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 37.58, "memory/max_allocated (GiB)": 37.58, "ppl": 1.19966, "step": 344, "tokens/total": 11989936, "tokens/train_per_sec_per_gpu": 62.22, "tokens/trainable": 362134} +{"epoch": 1.34765625, "grad_norm": 0.997672438621521, "learning_rate": 3.3939310220027456e-05, "loss": 0.1742849349975586, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.19039, "step": 345, "tokens/total": 12023968, "tokens/train_per_sec_per_gpu": 60.83, "tokens/trainable": 363229} +{"epoch": 1.3515625, "grad_norm": 1.115218997001648, "learning_rate": 3.3683214232914404e-05, "loss": 0.1699899137020111, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.18529, "step": 346, "tokens/total": 12059632, "tokens/train_per_sec_per_gpu": 59.82, "tokens/trainable": 364309} +{"epoch": 1.35546875, "grad_norm": 1.8379502296447754, "learning_rate": 3.342800532426873e-05, "loss": 0.19880948960781097, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.21995, "step": 347, "tokens/total": 12095616, "tokens/train_per_sec_per_gpu": 60.06, "tokens/trainable": 365327} +{"epoch": 1.359375, "grad_norm": 0.9519286155700684, "learning_rate": 3.317369411437484e-05, "loss": 0.18861067295074463, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.20757, "step": 348, "tokens/total": 12131488, "tokens/train_per_sec_per_gpu": 56.04, "tokens/trainable": 366371} +{"epoch": 1.36328125, "grad_norm": 0.6635769605636597, "learning_rate": 3.292029118616024e-05, "loss": 0.18066199123859406, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.19801, "step": 349, "tokens/total": 12164208, "tokens/train_per_sec_per_gpu": 56.96, "tokens/trainable": 367353} +{"epoch": 1.3671875, "grad_norm": 1.0987298488616943, "learning_rate": 3.266780708475511e-05, "loss": 0.17497725784778595, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 39.12, "memory/max_allocated (GiB)": 39.12, "ppl": 1.19122, "step": 350, "tokens/total": 12199616, "tokens/train_per_sec_per_gpu": 58.34, "tokens/trainable": 368428} +{"epoch": 1.37109375, "grad_norm": 1.0639108419418335, "learning_rate": 3.241625231705354e-05, "loss": 0.1735270917415619, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.18949, "step": 351, "tokens/total": 12235664, "tokens/train_per_sec_per_gpu": 60.04, "tokens/trainable": 369498} +{"epoch": 1.375, "grad_norm": 104.6640853881836, "learning_rate": 3.216563735127618e-05, "loss": 0.16308240592479706, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 39.12, "memory/max_allocated (GiB)": 39.12, "ppl": 1.17713, "step": 352, "tokens/total": 12270800, "tokens/train_per_sec_per_gpu": 61.17, "tokens/trainable": 370631} +{"epoch": 1.37890625, "grad_norm": 1.590192437171936, "learning_rate": 3.191597261653475e-05, "loss": 0.24284638464450836, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.27487, "step": 353, "tokens/total": 12307568, "tokens/train_per_sec_per_gpu": 57.76, "tokens/trainable": 371717} +{"epoch": 1.3828125, "grad_norm": 0.84271240234375, "learning_rate": 3.166726850239794e-05, "loss": 0.16282817721366882, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 37.18, "memory/max_allocated (GiB)": 37.18, "ppl": 1.17683, "step": 354, "tokens/total": 12341200, "tokens/train_per_sec_per_gpu": 62.62, "tokens/trainable": 372795} +{"epoch": 1.38671875, "grad_norm": 0.8644141554832458, "learning_rate": 3.141953535845912e-05, "loss": 0.20625758171081543, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 36.16, "memory/max_allocated (GiB)": 36.16, "ppl": 1.22907, "step": 355, "tokens/total": 12372016, "tokens/train_per_sec_per_gpu": 63.09, "tokens/trainable": 373733} +{"epoch": 1.390625, "grad_norm": 0.9857526421546936, "learning_rate": 3.11727834939056e-05, "loss": 0.184464693069458, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 36.28, "memory/max_allocated (GiB)": 36.28, "ppl": 1.20257, "step": 356, "tokens/total": 12403856, "tokens/train_per_sec_per_gpu": 60.05, "tokens/trainable": 374767} +{"epoch": 1.39453125, "grad_norm": 0.9649003744125366, "learning_rate": 3.092702317708967e-05, "loss": 0.19495058059692383, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.21525, "step": 357, "tokens/total": 12437584, "tokens/train_per_sec_per_gpu": 62.51, "tokens/trainable": 375792} +{"epoch": 1.3984375, "grad_norm": 0.9195184111595154, "learning_rate": 3.0682264635101276e-05, "loss": 0.22355607151985168, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 37.44, "memory/max_allocated (GiB)": 37.44, "ppl": 1.25052, "step": 358, "tokens/total": 12470096, "tokens/train_per_sec_per_gpu": 62.37, "tokens/trainable": 376808} +{"epoch": 1.40234375, "grad_norm": 1.0233367681503296, "learning_rate": 3.0438518053342407e-05, "loss": 0.1649172008037567, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.1793, "step": 359, "tokens/total": 12509168, "tokens/train_per_sec_per_gpu": 56.17, "tokens/trainable": 377939} +{"epoch": 1.40625, "grad_norm": 0.846088707447052, "learning_rate": 3.0195793575103266e-05, "loss": 0.17649874091148376, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.19303, "step": 360, "tokens/total": 12548208, "tokens/train_per_sec_per_gpu": 54.97, "tokens/trainable": 379015} +{"epoch": 1.41015625, "grad_norm": 1.8010936975479126, "learning_rate": 2.9954101301140146e-05, "loss": 0.19461765885353088, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.21485, "step": 361, "tokens/total": 12583552, "tokens/train_per_sec_per_gpu": 58.93, "tokens/trainable": 380065} +{"epoch": 1.4140625, "grad_norm": 0.6648022532463074, "learning_rate": 2.9713451289255123e-05, "loss": 0.1824951171875, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 36.9, "memory/max_allocated (GiB)": 36.9, "ppl": 1.20021, "step": 362, "tokens/total": 12612944, "tokens/train_per_sec_per_gpu": 64.05, "tokens/trainable": 381109} +{"epoch": 1.41796875, "grad_norm": 0.7212105989456177, "learning_rate": 2.9473853553877484e-05, "loss": 0.16908316314220428, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.18422, "step": 363, "tokens/total": 12646880, "tokens/train_per_sec_per_gpu": 60.43, "tokens/trainable": 382148} +{"epoch": 1.421875, "grad_norm": 0.5806607604026794, "learning_rate": 2.9235318065647e-05, "loss": 0.18359032273292542, "memory/device_reserved (GiB)": 43.08, "memory/max_active (GiB)": 39.61, "memory/max_allocated (GiB)": 39.61, "ppl": 1.20152, "step": 364, "tokens/total": 12686896, "tokens/train_per_sec_per_gpu": 53.5, "tokens/trainable": 383216} +{"epoch": 1.42578125, "grad_norm": 0.9261683821678162, "learning_rate": 2.8997854750998964e-05, "loss": 0.19633831083774567, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.21694, "step": 365, "tokens/total": 12726416, "tokens/train_per_sec_per_gpu": 59.12, "tokens/trainable": 384335} +{"epoch": 1.4296875, "grad_norm": 0.7518418431282043, "learning_rate": 2.8761473491751258e-05, "loss": 0.17852084338665009, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.63, "memory/max_allocated (GiB)": 36.63, "ppl": 1.19545, "step": 366, "tokens/total": 12759248, "tokens/train_per_sec_per_gpu": 59.99, "tokens/trainable": 385333} +{"epoch": 1.43359375, "grad_norm": 0.4468555450439453, "learning_rate": 2.8526184124692883e-05, "loss": 0.14651629328727722, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.15779, "step": 367, "tokens/total": 12795904, "tokens/train_per_sec_per_gpu": 52.68, "tokens/trainable": 386436} +{"epoch": 1.4375, "grad_norm": 0.7725984454154968, "learning_rate": 2.829199644117484e-05, "loss": 0.1731327921152115, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.18902, "step": 368, "tokens/total": 12829616, "tokens/train_per_sec_per_gpu": 62.93, "tokens/trainable": 387399} +{"epoch": 1.44140625, "grad_norm": 0.6593184471130371, "learning_rate": 2.8058920186702553e-05, "loss": 0.16627919673919678, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.91, "memory/max_allocated (GiB)": 38.91, "ppl": 1.1809, "step": 369, "tokens/total": 12867744, "tokens/train_per_sec_per_gpu": 58.91, "tokens/trainable": 388527} +{"epoch": 1.4453125, "grad_norm": 0.7569136619567871, "learning_rate": 2.782696506053033e-05, "loss": 0.17940865457057953, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.19651, "step": 370, "tokens/total": 12905936, "tokens/train_per_sec_per_gpu": 52.68, "tokens/trainable": 389522} +{"epoch": 1.44921875, "grad_norm": 1.0188349485397339, "learning_rate": 2.7596140715257824e-05, "loss": 0.17367300391197205, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.18967, "step": 371, "tokens/total": 12941216, "tokens/train_per_sec_per_gpu": 61.56, "tokens/trainable": 390627} +{"epoch": 1.453125, "grad_norm": 0.5811536908149719, "learning_rate": 2.7366456756428184e-05, "loss": 0.1636316478252411, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.17778, "step": 372, "tokens/total": 12975584, "tokens/train_per_sec_per_gpu": 60.14, "tokens/trainable": 391663} +{"epoch": 1.45703125, "grad_norm": 0.8231342434883118, "learning_rate": 2.7137922742128486e-05, "loss": 0.17275479435920715, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.21, "memory/max_allocated (GiB)": 38.21, "ppl": 1.18857, "step": 373, "tokens/total": 13010464, "tokens/train_per_sec_per_gpu": 61.52, "tokens/trainable": 392730} +{"epoch": 1.4609375, "grad_norm": 0.5605610013008118, "learning_rate": 2.691054818259188e-05, "loss": 0.1630263328552246, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.4, "memory/max_allocated (GiB)": 37.4, "ppl": 1.17707, "step": 374, "tokens/total": 13045136, "tokens/train_per_sec_per_gpu": 62.29, "tokens/trainable": 393849} +{"epoch": 1.46484375, "grad_norm": 1.3121976852416992, "learning_rate": 2.6684342539801933e-05, "loss": 0.20261234045028687, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.49, "memory/max_allocated (GiB)": 36.49, "ppl": 1.2246, "step": 375, "tokens/total": 13076400, "tokens/train_per_sec_per_gpu": 61.5, "tokens/trainable": 394858} +{"epoch": 1.46875, "grad_norm": 0.6615918874740601, "learning_rate": 2.645931522709877e-05, "loss": 0.16307878494262695, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.54, "memory/max_allocated (GiB)": 39.54, "ppl": 1.17713, "step": 376, "tokens/total": 13112144, "tokens/train_per_sec_per_gpu": 73.92, "tokens/trainable": 396035} +{"epoch": 1.47265625, "grad_norm": 1.0763063430786133, "learning_rate": 2.6235475608787365e-05, "loss": 0.18629859387874603, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.6, "memory/max_allocated (GiB)": 38.6, "ppl": 1.20478, "step": 377, "tokens/total": 13147872, "tokens/train_per_sec_per_gpu": 59.13, "tokens/trainable": 397064} +{"epoch": 1.4765625, "grad_norm": 0.8863649964332581, "learning_rate": 2.6012832999747916e-05, "loss": 0.20046305656433105, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.22197, "step": 378, "tokens/total": 13178656, "tokens/train_per_sec_per_gpu": 64.08, "tokens/trainable": 398045} +{"epoch": 1.48046875, "grad_norm": 1.0935066938400269, "learning_rate": 2.579139666504821e-05, "loss": 0.20389772951602936, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.22617, "step": 379, "tokens/total": 13218720, "tokens/train_per_sec_per_gpu": 52.62, "tokens/trainable": 399060} +{"epoch": 1.484375, "grad_norm": 0.7659334540367126, "learning_rate": 2.557117581955798e-05, "loss": 0.185114324092865, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 1.20336, "step": 380, "tokens/total": 13252544, "tokens/train_per_sec_per_gpu": 59.61, "tokens/trainable": 400126} +{"epoch": 1.48828125, "grad_norm": 0.8252306580543518, "learning_rate": 2.5352179627565532e-05, "loss": 0.20071451365947723, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.06, "memory/max_allocated (GiB)": 38.06, "ppl": 1.22228, "step": 381, "tokens/total": 13286832, "tokens/train_per_sec_per_gpu": 55.46, "tokens/trainable": 401156} +{"epoch": 1.4921875, "grad_norm": 0.729312539100647, "learning_rate": 2.5134417202396277e-05, "loss": 0.18596689403057098, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.20438, "step": 382, "tokens/total": 13320432, "tokens/train_per_sec_per_gpu": 57.54, "tokens/trainable": 402125} +{"epoch": 1.49609375, "grad_norm": 0.8550353646278381, "learning_rate": 2.491789760603361e-05, "loss": 0.17323711514472961, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.18915, "step": 383, "tokens/total": 13356928, "tokens/train_per_sec_per_gpu": 53.4, "tokens/trainable": 403141} +{"epoch": 1.5, "grad_norm": 1.0160902738571167, "learning_rate": 2.4702629848741764e-05, "loss": 0.18381750583648682, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.2018, "step": 384, "tokens/total": 13391568, "tokens/train_per_sec_per_gpu": 62.19, "tokens/trainable": 404145} +{"epoch": 1.50390625, "grad_norm": 1.1208440065383911, "learning_rate": 2.4488622888690785e-05, "loss": 0.18245700001716614, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.20016, "step": 385, "tokens/total": 13425152, "tokens/train_per_sec_per_gpu": 54.99, "tokens/trainable": 405121} +{"epoch": 1.5078125, "grad_norm": 1.007479190826416, "learning_rate": 2.427588563158384e-05, "loss": 0.16682444512844086, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.11, "memory/max_allocated (GiB)": 39.11, "ppl": 1.18155, "step": 386, "tokens/total": 13459504, "tokens/train_per_sec_per_gpu": 62.19, "tokens/trainable": 406195} +{"epoch": 1.51171875, "grad_norm": 0.6951150298118591, "learning_rate": 2.406442693028651e-05, "loss": 0.21386213600635529, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.23845, "step": 387, "tokens/total": 13491136, "tokens/train_per_sec_per_gpu": 61.1, "tokens/trainable": 407149} +{"epoch": 1.515625, "grad_norm": 0.7531113624572754, "learning_rate": 2.3854255584458547e-05, "loss": 0.1733504831790924, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.18928, "step": 388, "tokens/total": 13530112, "tokens/train_per_sec_per_gpu": 51.16, "tokens/trainable": 408166} +{"epoch": 1.51953125, "grad_norm": 0.8678991794586182, "learning_rate": 2.3645380340187508e-05, "loss": 0.17441821098327637, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.14, "memory/max_allocated (GiB)": 39.14, "ppl": 1.19055, "step": 389, "tokens/total": 13565680, "tokens/train_per_sec_per_gpu": 62.45, "tokens/trainable": 409240} +{"epoch": 1.5234375, "grad_norm": 1.1129170656204224, "learning_rate": 2.3437809889624914e-05, "loss": 0.1941099911928177, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.52, "memory/max_allocated (GiB)": 36.52, "ppl": 1.21423, "step": 390, "tokens/total": 13596960, "tokens/train_per_sec_per_gpu": 74.74, "tokens/trainable": 410314} +{"epoch": 1.52734375, "grad_norm": 0.45491892099380493, "learning_rate": 2.3231552870624487e-05, "loss": 0.16188979148864746, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.17573, "step": 391, "tokens/total": 13627840, "tokens/train_per_sec_per_gpu": 70.48, "tokens/trainable": 411387} +{"epoch": 1.53125, "grad_norm": 1.3832134008407593, "learning_rate": 2.3026617866382657e-05, "loss": 0.15613554418087006, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.7, "memory/max_allocated (GiB)": 36.7, "ppl": 1.16898, "step": 392, "tokens/total": 13660848, "tokens/train_per_sec_per_gpu": 62.39, "tokens/trainable": 412473} +{"epoch": 1.53515625, "grad_norm": 0.761065661907196, "learning_rate": 2.2823013405081507e-05, "loss": 0.1548372209072113, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.16747, "step": 393, "tokens/total": 13699024, "tokens/train_per_sec_per_gpu": 53.43, "tokens/trainable": 413545} +{"epoch": 1.5390625, "grad_norm": 0.5161297917366028, "learning_rate": 2.2620747959533722e-05, "loss": 0.15624278783798218, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.16911, "step": 394, "tokens/total": 13732336, "tokens/train_per_sec_per_gpu": 65.62, "tokens/trainable": 414656} +{"epoch": 1.54296875, "grad_norm": 0.5947636365890503, "learning_rate": 2.2419829946830123e-05, "loss": 0.16779151558876038, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.18269, "step": 395, "tokens/total": 13768112, "tokens/train_per_sec_per_gpu": 60.03, "tokens/trainable": 415686} +{"epoch": 1.546875, "grad_norm": 0.7020169496536255, "learning_rate": 2.2220267727989325e-05, "loss": 0.15321987867355347, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.74, "memory/max_allocated (GiB)": 37.74, "ppl": 1.16558, "step": 396, "tokens/total": 13802480, "tokens/train_per_sec_per_gpu": 60.23, "tokens/trainable": 416747} +{"epoch": 1.55078125, "grad_norm": 1.269440770149231, "learning_rate": 2.202206960760984e-05, "loss": 0.15180446207523346, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.16393, "step": 397, "tokens/total": 13840736, "tokens/train_per_sec_per_gpu": 55.92, "tokens/trainable": 417894} +{"epoch": 1.5546875, "grad_norm": 1.2318506240844727, "learning_rate": 2.182524383352446e-05, "loss": 0.19033145904541016, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.20965, "step": 398, "tokens/total": 13876896, "tokens/train_per_sec_per_gpu": 58.57, "tokens/trainable": 418933} +{"epoch": 1.55859375, "grad_norm": 1.0273628234863281, "learning_rate": 2.1629798596457056e-05, "loss": 0.17264017462730408, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.34, "memory/max_allocated (GiB)": 36.34, "ppl": 1.18844, "step": 399, "tokens/total": 13908992, "tokens/train_per_sec_per_gpu": 62.83, "tokens/trainable": 419938} +{"epoch": 1.5625, "grad_norm": 0.5790666341781616, "learning_rate": 2.1435742029681725e-05, "loss": 0.18134216964244843, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.29, "memory/max_allocated (GiB)": 36.29, "ppl": 1.19883, "step": 400, "tokens/total": 13939456, "tokens/train_per_sec_per_gpu": 69.56, "tokens/trainable": 420907} +{"epoch": 1.56640625, "grad_norm": 0.5913949608802795, "learning_rate": 2.124308220868431e-05, "loss": 0.15405884385108948, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.16656, "step": 401, "tokens/total": 13975312, "tokens/train_per_sec_per_gpu": 56.73, "tokens/trainable": 421968} +{"epoch": 1.5703125, "grad_norm": 0.700890302658081, "learning_rate": 2.105182715082638e-05, "loss": 0.1624976247549057, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.92, "memory/max_allocated (GiB)": 36.92, "ppl": 1.17645, "step": 402, "tokens/total": 14009056, "tokens/train_per_sec_per_gpu": 63.3, "tokens/trainable": 423061} +{"epoch": 1.57421875, "grad_norm": 0.6488879323005676, "learning_rate": 2.0861984815011552e-05, "loss": 0.17902138829231262, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 1.19605, "step": 403, "tokens/total": 14044320, "tokens/train_per_sec_per_gpu": 55.44, "tokens/trainable": 424032} +{"epoch": 1.578125, "grad_norm": 0.7269997596740723, "learning_rate": 2.0673563101354323e-05, "loss": 0.17583036422729492, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.19224, "step": 404, "tokens/total": 14079600, "tokens/train_per_sec_per_gpu": 53.48, "tokens/trainable": 425056} +{"epoch": 1.58203125, "grad_norm": 0.7370675802230835, "learning_rate": 2.0486569850851317e-05, "loss": 0.1725407838821411, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.18832, "step": 405, "tokens/total": 14114368, "tokens/train_per_sec_per_gpu": 61.43, "tokens/trainable": 426072} +{"epoch": 1.5859375, "grad_norm": 0.6463019847869873, "learning_rate": 2.0301012845054956e-05, "loss": 0.16567739844322205, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.18019, "step": 406, "tokens/total": 14153248, "tokens/train_per_sec_per_gpu": 52.48, "tokens/trainable": 427130} +{"epoch": 1.58984375, "grad_norm": 0.8658880591392517, "learning_rate": 2.011689980574966e-05, "loss": 0.1781865358352661, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.18, "memory/max_allocated (GiB)": 38.18, "ppl": 1.19505, "step": 407, "tokens/total": 14189632, "tokens/train_per_sec_per_gpu": 56.36, "tokens/trainable": 428145} +{"epoch": 1.59375, "grad_norm": 0.8075724840164185, "learning_rate": 1.993423839463052e-05, "loss": 0.16969266533851624, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.18494, "step": 408, "tokens/total": 14223072, "tokens/train_per_sec_per_gpu": 71.4, "tokens/trainable": 429231} +{"epoch": 1.59765625, "grad_norm": 0.5561400055885315, "learning_rate": 1.975303621298445e-05, "loss": 0.15993544459342957, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.17344, "step": 409, "tokens/total": 14259824, "tokens/train_per_sec_per_gpu": 57.13, "tokens/trainable": 430248} +{"epoch": 1.6015625, "grad_norm": 0.725566029548645, "learning_rate": 1.957330080137385e-05, "loss": 0.17919856309890747, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.22, "memory/max_allocated (GiB)": 36.22, "ppl": 1.19626, "step": 410, "tokens/total": 14290400, "tokens/train_per_sec_per_gpu": 66.85, "tokens/trainable": 431273} +{"epoch": 1.60546875, "grad_norm": 1.124013900756836, "learning_rate": 1.9395039639322864e-05, "loss": 0.19226178526878357, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.21199, "step": 411, "tokens/total": 14326528, "tokens/train_per_sec_per_gpu": 60.61, "tokens/trainable": 432338} +{"epoch": 1.609375, "grad_norm": 0.824521005153656, "learning_rate": 1.9218260145006073e-05, "loss": 0.1587214171886444, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.17201, "step": 412, "tokens/total": 14362400, "tokens/train_per_sec_per_gpu": 64.48, "tokens/trainable": 433475} +{"epoch": 1.61328125, "grad_norm": 0.7866014838218689, "learning_rate": 1.904296967493982e-05, "loss": 0.1686495542526245, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.18371, "step": 413, "tokens/total": 14394864, "tokens/train_per_sec_per_gpu": 69.18, "tokens/trainable": 434520} +{"epoch": 1.6171875, "grad_norm": 1.4780277013778687, "learning_rate": 1.8869175523676064e-05, "loss": 0.1500750333070755, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.36, "memory/max_allocated (GiB)": 38.36, "ppl": 1.16192, "step": 414, "tokens/total": 14431648, "tokens/train_per_sec_per_gpu": 61.01, "tokens/trainable": 435655} +{"epoch": 1.62109375, "grad_norm": 0.5870451927185059, "learning_rate": 1.869688492349885e-05, "loss": 0.16718889772891998, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.45, "memory/max_allocated (GiB)": 37.45, "ppl": 1.18198, "step": 415, "tokens/total": 14464912, "tokens/train_per_sec_per_gpu": 63.59, "tokens/trainable": 436728} +{"epoch": 1.625, "grad_norm": 0.9416561722755432, "learning_rate": 1.85261050441233e-05, "loss": 0.164754256606102, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.1791, "step": 416, "tokens/total": 14501056, "tokens/train_per_sec_per_gpu": 58.29, "tokens/trainable": 437829} +{"epoch": 1.62890625, "grad_norm": 0.9659982919692993, "learning_rate": 1.8356842992397304e-05, "loss": 0.19062796235084534, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.23, "memory/max_allocated (GiB)": 37.23, "ppl": 1.21001, "step": 417, "tokens/total": 14534928, "tokens/train_per_sec_per_gpu": 58.83, "tokens/trainable": 438826} +{"epoch": 1.6328125, "grad_norm": 0.5987693667411804, "learning_rate": 1.8189105812005714e-05, "loss": 0.17426705360412598, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.19037, "step": 418, "tokens/total": 14571680, "tokens/train_per_sec_per_gpu": 55.1, "tokens/trainable": 439788} +{"epoch": 1.63671875, "grad_norm": 0.8847687840461731, "learning_rate": 1.802290048317732e-05, "loss": 0.1844608336687088, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.20257, "step": 419, "tokens/total": 14605280, "tokens/train_per_sec_per_gpu": 59.2, "tokens/trainable": 440804} +{"epoch": 1.640625, "grad_norm": 0.6342026591300964, "learning_rate": 1.785823392239424e-05, "loss": 0.1679200828075409, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.82, "memory/max_allocated (GiB)": 37.82, "ppl": 1.18284, "step": 420, "tokens/total": 14638576, "tokens/train_per_sec_per_gpu": 58.5, "tokens/trainable": 441783} +{"epoch": 1.64453125, "grad_norm": 0.5394296646118164, "learning_rate": 1.7695112982104225e-05, "loss": 0.15710507333278656, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.78, "memory/max_allocated (GiB)": 39.78, "ppl": 1.17012, "step": 421, "tokens/total": 14673824, "tokens/train_per_sec_per_gpu": 61.95, "tokens/trainable": 442836} +{"epoch": 1.6484375, "grad_norm": 1.4305680990219116, "learning_rate": 1.7533544450435433e-05, "loss": 0.1776951551437378, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 35.69, "memory/max_allocated (GiB)": 35.69, "ppl": 1.19446, "step": 422, "tokens/total": 14703248, "tokens/train_per_sec_per_gpu": 75.68, "tokens/trainable": 443874} +{"epoch": 1.65234375, "grad_norm": 0.8254686594009399, "learning_rate": 1.7373535050913946e-05, "loss": 0.17021915316581726, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.18556, "step": 423, "tokens/total": 14739712, "tokens/train_per_sec_per_gpu": 52.12, "tokens/trainable": 444898} +{"epoch": 1.65625, "grad_norm": 0.5427790880203247, "learning_rate": 1.721509144218405e-05, "loss": 0.16455471515655518, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 35.68, "memory/max_allocated (GiB)": 35.68, "ppl": 1.17887, "step": 424, "tokens/total": 14769696, "tokens/train_per_sec_per_gpu": 73.15, "tokens/trainable": 445915} +{"epoch": 1.66015625, "grad_norm": 0.7224888801574707, "learning_rate": 1.705822021773101e-05, "loss": 0.16887295246124268, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.18397, "step": 425, "tokens/total": 14804160, "tokens/train_per_sec_per_gpu": 63.63, "tokens/trainable": 446991} +{"epoch": 1.6640625, "grad_norm": 0.7536628246307373, "learning_rate": 1.69029279056068e-05, "loss": 0.17593154311180115, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.99, "memory/max_allocated (GiB)": 38.99, "ppl": 1.19236, "step": 426, "tokens/total": 14841600, "tokens/train_per_sec_per_gpu": 54.97, "tokens/trainable": 448044} +{"epoch": 1.66796875, "grad_norm": 0.4631378948688507, "learning_rate": 1.6749220968158415e-05, "loss": 0.1594540774822235, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.17287, "step": 427, "tokens/total": 14876720, "tokens/train_per_sec_per_gpu": 60.57, "tokens/trainable": 449143} +{"epoch": 1.671875, "grad_norm": 1.0665005445480347, "learning_rate": 1.659710580175893e-05, "loss": 0.18186214566230774, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 35.89, "memory/max_allocated (GiB)": 35.89, "ppl": 1.19945, "step": 428, "tokens/total": 14907584, "tokens/train_per_sec_per_gpu": 66.4, "tokens/trainable": 450205} +{"epoch": 1.67578125, "grad_norm": 1.8447798490524292, "learning_rate": 1.644658873654133e-05, "loss": 0.151585191488266, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.18, "memory/max_allocated (GiB)": 37.18, "ppl": 1.16368, "step": 429, "tokens/total": 14940352, "tokens/train_per_sec_per_gpu": 66.69, "tokens/trainable": 451290} +{"epoch": 1.6796875, "grad_norm": 0.8505832552909851, "learning_rate": 1.629767603613508e-05, "loss": 0.1853545606136322, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.09, "memory/max_allocated (GiB)": 36.09, "ppl": 1.20365, "step": 430, "tokens/total": 14971200, "tokens/train_per_sec_per_gpu": 62.52, "tokens/trainable": 452210} +{"epoch": 1.68359375, "grad_norm": 1.160605788230896, "learning_rate": 1.615037389740547e-05, "loss": 0.1656842827796936, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.1802, "step": 431, "tokens/total": 15006224, "tokens/train_per_sec_per_gpu": 57.76, "tokens/trainable": 453263} +{"epoch": 1.6875, "grad_norm": 0.7364547252655029, "learning_rate": 1.600468845019576e-05, "loss": 0.16397641599178314, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.17819, "step": 432, "tokens/total": 15044336, "tokens/train_per_sec_per_gpu": 52.13, "tokens/trainable": 454315} +{"epoch": 1.69140625, "grad_norm": 2.633389949798584, "learning_rate": 1.5860625757072092e-05, "loss": 0.19231624901294708, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.21205, "step": 433, "tokens/total": 15079408, "tokens/train_per_sec_per_gpu": 55.22, "tokens/trainable": 455304} +{"epoch": 1.6953125, "grad_norm": 0.9528771042823792, "learning_rate": 1.571819181307116e-05, "loss": 0.16547083854675293, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.17995, "step": 434, "tokens/total": 15116576, "tokens/train_per_sec_per_gpu": 63.85, "tokens/trainable": 456459} +{"epoch": 1.69921875, "grad_norm": 0.6126032471656799, "learning_rate": 1.557739254545075e-05, "loss": 0.1680813729763031, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.42, "memory/max_allocated (GiB)": 36.42, "ppl": 1.18303, "step": 435, "tokens/total": 15147760, "tokens/train_per_sec_per_gpu": 71.73, "tokens/trainable": 457539} +{"epoch": 1.703125, "grad_norm": 0.5834000110626221, "learning_rate": 1.543823381344311e-05, "loss": 0.17064006626605988, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.18606, "step": 436, "tokens/total": 15180416, "tokens/train_per_sec_per_gpu": 72.82, "tokens/trainable": 458648} +{"epoch": 1.70703125, "grad_norm": 0.6377969980239868, "learning_rate": 1.5300721408011114e-05, "loss": 0.1655963957309723, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.1801, "step": 437, "tokens/total": 15215856, "tokens/train_per_sec_per_gpu": 62.91, "tokens/trainable": 459729} +{"epoch": 1.7109375, "grad_norm": 1.012304663658142, "learning_rate": 1.5164861051607254e-05, "loss": 0.19974930584430695, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.04, "memory/max_allocated (GiB)": 37.04, "ppl": 1.2211, "step": 438, "tokens/total": 15249152, "tokens/train_per_sec_per_gpu": 54.04, "tokens/trainable": 460676} +{"epoch": 1.71484375, "grad_norm": 0.8652110695838928, "learning_rate": 1.5030658397935521e-05, "loss": 0.1655081808567047, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.17999, "step": 439, "tokens/total": 15284720, "tokens/train_per_sec_per_gpu": 57.21, "tokens/trainable": 461741} +{"epoch": 1.71875, "grad_norm": 0.4678455591201782, "learning_rate": 1.4898119031716104e-05, "loss": 0.15268364548683167, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.16496, "step": 440, "tokens/total": 15317712, "tokens/train_per_sec_per_gpu": 64.74, "tokens/trainable": 462799} +{"epoch": 1.72265625, "grad_norm": 0.5493190288543701, "learning_rate": 1.476724846845306e-05, "loss": 0.15301431715488434, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.16534, "step": 441, "tokens/total": 15356000, "tokens/train_per_sec_per_gpu": 54.1, "tokens/trainable": 463903} +{"epoch": 1.7265625, "grad_norm": 0.6552057266235352, "learning_rate": 1.463805215420471e-05, "loss": 0.15958333015441895, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.65, "memory/max_allocated (GiB)": 39.65, "ppl": 1.17302, "step": 442, "tokens/total": 15391568, "tokens/train_per_sec_per_gpu": 60.72, "tokens/trainable": 464965} +{"epoch": 1.73046875, "grad_norm": 0.6342446208000183, "learning_rate": 1.451053546535705e-05, "loss": 0.1787796914577484, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 1.19576, "step": 443, "tokens/total": 15428352, "tokens/train_per_sec_per_gpu": 53.99, "tokens/trainable": 465971} +{"epoch": 1.734375, "grad_norm": 0.966280460357666, "learning_rate": 1.438470370840001e-05, "loss": 0.14642876386642456, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 1.15769, "step": 444, "tokens/total": 15464672, "tokens/train_per_sec_per_gpu": 62.28, "tokens/trainable": 467117} +{"epoch": 1.73828125, "grad_norm": 0.8577820062637329, "learning_rate": 1.4260562119706606e-05, "loss": 0.16961753368377686, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.18485, "step": 445, "tokens/total": 15497072, "tokens/train_per_sec_per_gpu": 68.57, "tokens/trainable": 468190} +{"epoch": 1.7421875, "grad_norm": 0.6576082110404968, "learning_rate": 1.413811586531508e-05, "loss": 0.16434292495250702, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.17862, "step": 446, "tokens/total": 15530416, "tokens/train_per_sec_per_gpu": 67.06, "tokens/trainable": 469245} +{"epoch": 1.74609375, "grad_norm": 0.782889723777771, "learning_rate": 1.4017370040713884e-05, "loss": 0.17719468474388123, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 1.19386, "step": 447, "tokens/total": 15567632, "tokens/train_per_sec_per_gpu": 53.33, "tokens/trainable": 470246} +{"epoch": 1.75, "grad_norm": 0.769986629486084, "learning_rate": 1.3898329670629645e-05, "loss": 0.16775476932525635, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.73, "memory/max_allocated (GiB)": 39.73, "ppl": 1.18265, "step": 448, "tokens/total": 15608192, "tokens/train_per_sec_per_gpu": 48.83, "tokens/trainable": 471287} +{"epoch": 1.75390625, "grad_norm": 0.5462313294410706, "learning_rate": 1.3780999708818058e-05, "loss": 0.16454952955245972, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.27, "memory/max_allocated (GiB)": 36.27, "ppl": 1.17886, "step": 449, "tokens/total": 15639360, "tokens/train_per_sec_per_gpu": 63.87, "tokens/trainable": 472314} +{"epoch": 1.7578125, "grad_norm": 0.5747849941253662, "learning_rate": 1.3665385037857758e-05, "loss": 0.16216014325618744, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.92, "memory/max_allocated (GiB)": 36.92, "ppl": 1.17605, "step": 450, "tokens/total": 15672560, "tokens/train_per_sec_per_gpu": 67.18, "tokens/trainable": 473381} +{"epoch": 1.76171875, "grad_norm": 0.6525391340255737, "learning_rate": 1.3551490468947126e-05, "loss": 0.16406647861003876, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.4, "memory/max_allocated (GiB)": 36.4, "ppl": 1.17829, "step": 451, "tokens/total": 15704416, "tokens/train_per_sec_per_gpu": 65.55, "tokens/trainable": 474426} +{"epoch": 1.765625, "grad_norm": 0.9066736102104187, "learning_rate": 1.3439320741704075e-05, "loss": 0.19328467547893524, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.93, "memory/max_allocated (GiB)": 38.93, "ppl": 1.21323, "step": 452, "tokens/total": 15739152, "tokens/train_per_sec_per_gpu": 63.99, "tokens/trainable": 475465} +{"epoch": 1.76953125, "grad_norm": 0.5210320353507996, "learning_rate": 1.3328880523968808e-05, "loss": 0.16201400756835938, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.06, "memory/max_allocated (GiB)": 37.06, "ppl": 1.17588, "step": 453, "tokens/total": 15772592, "tokens/train_per_sec_per_gpu": 61.51, "tokens/trainable": 476488} +{"epoch": 1.7734375, "grad_norm": 0.5652540326118469, "learning_rate": 1.3220174411609587e-05, "loss": 0.16187503933906555, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.17571, "step": 454, "tokens/total": 15806320, "tokens/train_per_sec_per_gpu": 63.97, "tokens/trainable": 477572} +{"epoch": 1.77734375, "grad_norm": 0.5578881502151489, "learning_rate": 1.3113206928331471e-05, "loss": 0.16021785140037537, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 35.23, "memory/max_allocated (GiB)": 35.23, "ppl": 1.17377, "step": 455, "tokens/total": 15834704, "tokens/train_per_sec_per_gpu": 74.25, "tokens/trainable": 478644} +{"epoch": 1.78125, "grad_norm": 0.5955293774604797, "learning_rate": 1.300798252548806e-05, "loss": 0.15065529942512512, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.1626, "step": 456, "tokens/total": 15869440, "tokens/train_per_sec_per_gpu": 63.14, "tokens/trainable": 479717} +{"epoch": 1.78515625, "grad_norm": 0.7966031432151794, "learning_rate": 1.2904505581896265e-05, "loss": 0.1668873429298401, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.18162, "step": 457, "tokens/total": 15908816, "tokens/train_per_sec_per_gpu": 51.59, "tokens/trainable": 480818} +{"epoch": 1.7890625, "grad_norm": 0.5143404006958008, "learning_rate": 1.2802780403654082e-05, "loss": 0.16947811841964722, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.18469, "step": 458, "tokens/total": 15942320, "tokens/train_per_sec_per_gpu": 62.93, "tokens/trainable": 481834} +{"epoch": 1.79296875, "grad_norm": 0.5518168210983276, "learning_rate": 1.2702811223961408e-05, "loss": 0.1575004905462265, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.9, "memory/max_allocated (GiB)": 36.9, "ppl": 1.17058, "step": 459, "tokens/total": 15976048, "tokens/train_per_sec_per_gpu": 61.03, "tokens/trainable": 482869} +{"epoch": 1.796875, "grad_norm": 0.7054102420806885, "learning_rate": 1.2604602202943861e-05, "loss": 0.16028490662574768, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.98, "memory/max_allocated (GiB)": 36.98, "ppl": 1.17385, "step": 460, "tokens/total": 16007136, "tokens/train_per_sec_per_gpu": 73.36, "tokens/trainable": 483984} +{"epoch": 1.80078125, "grad_norm": 0.9691389203071594, "learning_rate": 1.2508157427479686e-05, "loss": 0.16967076063156128, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.18491, "step": 461, "tokens/total": 16039728, "tokens/train_per_sec_per_gpu": 62.02, "tokens/trainable": 484986} +{"epoch": 1.8046875, "grad_norm": 0.6593483686447144, "learning_rate": 1.2413480911029655e-05, "loss": 0.1810620129108429, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 35.9, "memory/max_allocated (GiB)": 35.9, "ppl": 1.19849, "step": 462, "tokens/total": 16070528, "tokens/train_per_sec_per_gpu": 60.21, "tokens/trainable": 486001} +{"epoch": 1.80859375, "grad_norm": 0.7465775609016418, "learning_rate": 1.2320576593470082e-05, "loss": 0.16215947270393372, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.17605, "step": 463, "tokens/total": 16105056, "tokens/train_per_sec_per_gpu": 58.04, "tokens/trainable": 487044} +{"epoch": 1.8125, "grad_norm": 0.4851621389389038, "learning_rate": 1.2229448340928828e-05, "loss": 0.1491844356060028, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.16089, "step": 464, "tokens/total": 16144656, "tokens/train_per_sec_per_gpu": 51.21, "tokens/trainable": 488162} +{"epoch": 1.81640625, "grad_norm": 0.5020430088043213, "learning_rate": 1.2140099945624458e-05, "loss": 0.1567198485136032, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.16967, "step": 465, "tokens/total": 16183584, "tokens/train_per_sec_per_gpu": 58.16, "tokens/trainable": 489295} +{"epoch": 1.8203125, "grad_norm": 0.6213752627372742, "learning_rate": 1.205253512570841e-05, "loss": 0.16439098119735718, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.83, "memory/max_allocated (GiB)": 37.83, "ppl": 1.17868, "step": 466, "tokens/total": 16216352, "tokens/train_per_sec_per_gpu": 64.03, "tokens/trainable": 490325} +{"epoch": 1.82421875, "grad_norm": 0.5990549325942993, "learning_rate": 1.1966757525110255e-05, "loss": 0.17922475934028625, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 35.38, "memory/max_allocated (GiB)": 35.38, "ppl": 1.19629, "step": 467, "tokens/total": 16245696, "tokens/train_per_sec_per_gpu": 66.21, "tokens/trainable": 491291} +{"epoch": 1.828125, "grad_norm": 0.4565768539905548, "learning_rate": 1.1882770713386095e-05, "loss": 0.1529589295387268, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.16528, "step": 468, "tokens/total": 16282592, "tokens/train_per_sec_per_gpu": 62.19, "tokens/trainable": 492408} +{"epoch": 1.83203125, "grad_norm": 1.1009799242019653, "learning_rate": 1.180057818556998e-05, "loss": 0.15028244256973267, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.16216, "step": 469, "tokens/total": 16318800, "tokens/train_per_sec_per_gpu": 60.8, "tokens/trainable": 493528} +{"epoch": 1.8359375, "grad_norm": 0.516129732131958, "learning_rate": 1.1720183362028494e-05, "loss": 0.16170798242092133, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.59, "memory/max_allocated (GiB)": 38.59, "ppl": 1.17552, "step": 470, "tokens/total": 16355232, "tokens/train_per_sec_per_gpu": 55.87, "tokens/trainable": 494559} +{"epoch": 1.83984375, "grad_norm": 0.5390494465827942, "learning_rate": 1.1641589588318387e-05, "loss": 0.16803887486457825, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.18298, "step": 471, "tokens/total": 16393216, "tokens/train_per_sec_per_gpu": 56.6, "tokens/trainable": 495613} +{"epoch": 1.84375, "grad_norm": 2.300156354904175, "learning_rate": 1.1564800135047418e-05, "loss": 0.14638881385326385, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 1.15765, "step": 472, "tokens/total": 16426784, "tokens/train_per_sec_per_gpu": 62.3, "tokens/trainable": 496700} +{"epoch": 1.84765625, "grad_norm": 1.5203320980072021, "learning_rate": 1.148981819773816e-05, "loss": 0.17036370933055878, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.18574, "step": 473, "tokens/total": 16459792, "tokens/train_per_sec_per_gpu": 54.93, "tokens/trainable": 497712} +{"epoch": 1.8515625, "grad_norm": 0.698899507522583, "learning_rate": 1.1416646896695086e-05, "loss": 0.16083595156669617, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.95, "memory/max_allocated (GiB)": 37.95, "ppl": 1.17449, "step": 474, "tokens/total": 16494272, "tokens/train_per_sec_per_gpu": 62.91, "tokens/trainable": 498772} +{"epoch": 1.85546875, "grad_norm": 0.6754190921783447, "learning_rate": 1.1345289276874717e-05, "loss": 0.16751797497272491, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.43, "memory/max_allocated (GiB)": 38.43, "ppl": 1.18237, "step": 475, "tokens/total": 16531472, "tokens/train_per_sec_per_gpu": 57.69, "tokens/trainable": 499870} +{"epoch": 1.859375, "grad_norm": 0.8561041951179504, "learning_rate": 1.1275748307758873e-05, "loss": 0.15674510598182678, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.1697, "step": 476, "tokens/total": 16567184, "tokens/train_per_sec_per_gpu": 63.71, "tokens/trainable": 500970} +{"epoch": 1.86328125, "grad_norm": 0.585196852684021, "learning_rate": 1.1208026883231147e-05, "loss": 0.179554283618927, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.42, "memory/max_allocated (GiB)": 36.42, "ppl": 1.19668, "step": 477, "tokens/total": 16598352, "tokens/train_per_sec_per_gpu": 64.85, "tokens/trainable": 501893} +{"epoch": 1.8671875, "grad_norm": 0.6023932099342346, "learning_rate": 1.1142127821456433e-05, "loss": 0.15721459686756134, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.17025, "step": 478, "tokens/total": 16631824, "tokens/train_per_sec_per_gpu": 61.63, "tokens/trainable": 502909} +{"epoch": 1.87109375, "grad_norm": 1.2248177528381348, "learning_rate": 1.1078053864763674e-05, "loss": 0.1651458740234375, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.81, "memory/max_allocated (GiB)": 37.81, "ppl": 1.17957, "step": 479, "tokens/total": 16667776, "tokens/train_per_sec_per_gpu": 58.72, "tokens/trainable": 503982} +{"epoch": 1.875, "grad_norm": 0.5959789752960205, "learning_rate": 1.1015807679531756e-05, "loss": 0.14950385689735413, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.39, "memory/max_allocated (GiB)": 37.39, "ppl": 1.16126, "step": 480, "tokens/total": 16699408, "tokens/train_per_sec_per_gpu": 67.36, "tokens/trainable": 505067} +{"epoch": 1.87890625, "grad_norm": 0.6786162257194519, "learning_rate": 1.0955391856078528e-05, "loss": 0.1712333858013153, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.53, "memory/max_allocated (GiB)": 38.53, "ppl": 1.18677, "step": 481, "tokens/total": 16735440, "tokens/train_per_sec_per_gpu": 60.88, "tokens/trainable": 506119} +{"epoch": 1.8828125, "grad_norm": 0.785007119178772, "learning_rate": 1.0896808908553007e-05, "loss": 0.19102424383163452, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.35, "memory/max_allocated (GiB)": 37.35, "ppl": 1.21049, "step": 482, "tokens/total": 16770400, "tokens/train_per_sec_per_gpu": 53.44, "tokens/trainable": 507105} +{"epoch": 1.88671875, "grad_norm": 0.663195788860321, "learning_rate": 1.0840061274830763e-05, "loss": 0.18216168880462646, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.2, "memory/max_allocated (GiB)": 36.2, "ppl": 1.19981, "step": 483, "tokens/total": 16801776, "tokens/train_per_sec_per_gpu": 67.77, "tokens/trainable": 508159} +{"epoch": 1.890625, "grad_norm": 0.7313765287399292, "learning_rate": 1.0785151316412473e-05, "loss": 0.15750598907470703, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 1.17059, "step": 484, "tokens/total": 16837936, "tokens/train_per_sec_per_gpu": 53.38, "tokens/trainable": 509258} +{"epoch": 1.89453125, "grad_norm": 0.626480758190155, "learning_rate": 1.0732081318325639e-05, "loss": 0.15716758370399475, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.17019, "step": 485, "tokens/total": 16877072, "tokens/train_per_sec_per_gpu": 62.33, "tokens/trainable": 510385} +{"epoch": 1.8984375, "grad_norm": 1.3666796684265137, "learning_rate": 1.0680853489029501e-05, "loss": 0.18187278509140015, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.19946, "step": 486, "tokens/total": 16913840, "tokens/train_per_sec_per_gpu": 49.46, "tokens/trainable": 511379} +{"epoch": 1.90234375, "grad_norm": 0.7996423244476318, "learning_rate": 1.0631469960323152e-05, "loss": 0.18444213271141052, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.20255, "step": 487, "tokens/total": 16953264, "tokens/train_per_sec_per_gpu": 47.16, "tokens/trainable": 512350} +{"epoch": 1.90625, "grad_norm": 0.507358729839325, "learning_rate": 1.0583932787256783e-05, "loss": 0.16637404263019562, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.5, "memory/max_allocated (GiB)": 37.5, "ppl": 1.18101, "step": 488, "tokens/total": 16986736, "tokens/train_per_sec_per_gpu": 57.32, "tokens/trainable": 513301} +{"epoch": 1.91015625, "grad_norm": 0.7145753502845764, "learning_rate": 1.0538243948046206e-05, "loss": 0.16778592765331268, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.36, "memory/max_allocated (GiB)": 38.36, "ppl": 1.18268, "step": 489, "tokens/total": 17021792, "tokens/train_per_sec_per_gpu": 58.32, "tokens/trainable": 514293} +{"epoch": 1.9140625, "grad_norm": 0.555317759513855, "learning_rate": 1.0494405343990523e-05, "loss": 0.1717289537191391, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.53, "memory/max_allocated (GiB)": 39.53, "ppl": 1.18736, "step": 490, "tokens/total": 17059248, "tokens/train_per_sec_per_gpu": 52.11, "tokens/trainable": 515237} +{"epoch": 1.91796875, "grad_norm": 0.5222824215888977, "learning_rate": 1.0452418799392985e-05, "loss": 0.16870653629302979, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.09, "memory/max_allocated (GiB)": 37.09, "ppl": 1.18377, "step": 491, "tokens/total": 17092144, "tokens/train_per_sec_per_gpu": 59.29, "tokens/trainable": 516219} +{"epoch": 1.921875, "grad_norm": 1.3384592533111572, "learning_rate": 1.0412286061485102e-05, "loss": 0.15895357728004456, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.17228, "step": 492, "tokens/total": 17126688, "tokens/train_per_sec_per_gpu": 55.97, "tokens/trainable": 517247} +{"epoch": 1.92578125, "grad_norm": 0.6079921126365662, "learning_rate": 1.03740088003539e-05, "loss": 0.1643618792295456, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 35.96, "memory/max_allocated (GiB)": 35.96, "ppl": 1.17864, "step": 493, "tokens/total": 17156864, "tokens/train_per_sec_per_gpu": 71.85, "tokens/trainable": 518273} +{"epoch": 1.9296875, "grad_norm": 0.9245745539665222, "learning_rate": 1.0337588608872463e-05, "loss": 0.1640438735485077, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 35.99, "memory/max_allocated (GiB)": 35.99, "ppl": 1.17827, "step": 494, "tokens/total": 17186960, "tokens/train_per_sec_per_gpu": 73.98, "tokens/trainable": 519348} +{"epoch": 1.93359375, "grad_norm": 0.9195313453674316, "learning_rate": 1.0303027002633622e-05, "loss": 0.15927690267562866, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.17266, "step": 495, "tokens/total": 17223184, "tokens/train_per_sec_per_gpu": 57.24, "tokens/trainable": 520410} +{"epoch": 1.9375, "grad_norm": 0.5116511583328247, "learning_rate": 1.0270325419886884e-05, "loss": 0.15772800147533417, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.17085, "step": 496, "tokens/total": 17255680, "tokens/train_per_sec_per_gpu": 65.1, "tokens/trainable": 521450} +{"epoch": 1.94140625, "grad_norm": 0.6875448822975159, "learning_rate": 1.0239485221478599e-05, "loss": 0.1772512048482895, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 1.19393, "step": 497, "tokens/total": 17292240, "tokens/train_per_sec_per_gpu": 56.59, "tokens/trainable": 522451} +{"epoch": 1.9453125, "grad_norm": 0.5369216203689575, "learning_rate": 1.0210507690795292e-05, "loss": 0.16875866055488586, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.18383, "step": 498, "tokens/total": 17328080, "tokens/train_per_sec_per_gpu": 50.18, "tokens/trainable": 523408} +{"epoch": 1.94921875, "grad_norm": 0.9663099050521851, "learning_rate": 1.0183394033710305e-05, "loss": 0.16317451000213623, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.22, "memory/max_allocated (GiB)": 37.22, "ppl": 1.17724, "step": 499, "tokens/total": 17360432, "tokens/train_per_sec_per_gpu": 62.06, "tokens/trainable": 524419} +{"epoch": 1.953125, "grad_norm": 0.850791871547699, "learning_rate": 1.0158145378533583e-05, "loss": 0.16616404056549072, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.39, "memory/max_allocated (GiB)": 36.39, "ppl": 1.18077, "step": 500, "tokens/total": 17391968, "tokens/train_per_sec_per_gpu": 64.1, "tokens/trainable": 525425} +{"epoch": 1.95703125, "grad_norm": 0.7725031971931458, "learning_rate": 1.0134762775964726e-05, "loss": 0.15814439952373505, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.17134, "step": 501, "tokens/total": 17425584, "tokens/train_per_sec_per_gpu": 65.43, "tokens/trainable": 526540} +{"epoch": 1.9609375, "grad_norm": 0.7563740611076355, "learning_rate": 1.0113247199049278e-05, "loss": 0.1651013195514679, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.17951, "step": 502, "tokens/total": 17462752, "tokens/train_per_sec_per_gpu": 57.52, "tokens/trainable": 527569} +{"epoch": 1.96484375, "grad_norm": 0.505028486251831, "learning_rate": 1.0093599543138205e-05, "loss": 0.14327207207679749, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.15404, "step": 503, "tokens/total": 17499376, "tokens/train_per_sec_per_gpu": 59.3, "tokens/trainable": 528685} +{"epoch": 1.96875, "grad_norm": 0.830734372138977, "learning_rate": 1.0075820625850675e-05, "loss": 0.1721242368221283, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.18783, "step": 504, "tokens/total": 17538384, "tokens/train_per_sec_per_gpu": 49.61, "tokens/trainable": 529678} +{"epoch": 1.97265625, "grad_norm": 0.5672411322593689, "learning_rate": 1.0059911187040013e-05, "loss": 0.16227026283740997, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.17618, "step": 505, "tokens/total": 17572320, "tokens/train_per_sec_per_gpu": 63.88, "tokens/trainable": 530718} +{"epoch": 1.9765625, "grad_norm": 0.4565388858318329, "learning_rate": 1.0045871888762893e-05, "loss": 0.14730453491210938, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.15871, "step": 506, "tokens/total": 17610048, "tokens/train_per_sec_per_gpu": 51.03, "tokens/trainable": 531784} +{"epoch": 1.98046875, "grad_norm": 0.4957602024078369, "learning_rate": 1.003370331525184e-05, "loss": 0.15039962530136108, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.1623, "step": 507, "tokens/total": 17647296, "tokens/train_per_sec_per_gpu": 57.1, "tokens/trainable": 532886} +{"epoch": 1.984375, "grad_norm": 1.2802350521087646, "learning_rate": 1.002340597289085e-05, "loss": 0.163801372051239, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.17798, "step": 508, "tokens/total": 17682464, "tokens/train_per_sec_per_gpu": 59.36, "tokens/trainable": 533939} +{"epoch": 1.98828125, "grad_norm": 0.698728621006012, "learning_rate": 1.0014980290194387e-05, "loss": 0.1819794476032257, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.23, "memory/max_allocated (GiB)": 37.23, "ppl": 1.19959, "step": 509, "tokens/total": 17715440, "tokens/train_per_sec_per_gpu": 63.05, "tokens/trainable": 534970} +{"epoch": 1.9921875, "grad_norm": 1.0997791290283203, "learning_rate": 1.0008426617789489e-05, "loss": 0.15801206231117249, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.17118, "step": 510, "tokens/total": 17748000, "tokens/train_per_sec_per_gpu": 66.32, "tokens/trainable": 536002} +{"epoch": 1.99609375, "grad_norm": 0.5389105081558228, "learning_rate": 1.0003745228401215e-05, "loss": 0.1604510098695755, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.15, "memory/max_allocated (GiB)": 39.15, "ppl": 1.17404, "step": 511, "tokens/total": 17782656, "tokens/train_per_sec_per_gpu": 64.12, "tokens/trainable": 537125} +{"epoch": 2.0, "grad_norm": 0.5220202207565308, "learning_rate": 1.0000936316841296e-05, "loss": 0.15555965900421143, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.16831, "step": 512, "tokens/total": 17819360, "tokens/train_per_sec_per_gpu": 57.19, "tokens/trainable": 538198}